From 20623813e1724c4b08c4bc998aaac137b69d1254 Mon Sep 17 00:00:00 2001 From: akshatgokul Date: Tue, 22 Jul 2025 23:57:24 +0530 Subject: [PATCH] add: mircograd --- .gitignore | 2 + notebooks/01a_mircograd.ipynb | 1975 +++++++++++++++++++++++++++ notebooks/01b_mircograd.ipynb | 2359 +++++++++++++++++++++++++++++++++ 3 files changed, 4336 insertions(+) create mode 100644 .gitignore create mode 100644 notebooks/01a_mircograd.ipynb create mode 100644 notebooks/01b_mircograd.ipynb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8a58e00 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.ipynb_checkpoints +*/.ipynb_checkpoints/* \ No newline at end of file diff --git a/notebooks/01a_mircograd.ipynb b/notebooks/01a_mircograd.ipynb new file mode 100644 index 0000000..707ae51 --- /dev/null +++ b/notebooks/01a_mircograd.ipynb @@ -0,0 +1,1975 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "672d74cf-2fad-4493-9e3d-7ac2286dbfee", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "e7295cd8-1f33-40d2-aca5-e226e5b945ba", + "metadata": {}, + "source": [ + "Let's define a function f" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "07a0fdd9-e9c4-4a28-9201-99429a638573", + "metadata": {}, + "outputs": [], + "source": [ + "def f(x):\n", + " return 3*x**2 - 4*x + 5" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "67093e12-40d4-4abd-a81d-2f7f63881e19", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "20.0" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "f(3.0)" + ] + }, + { + "cell_type": "markdown", + "id": "ec1297be-8d68-4d59-af98-39e45eb9857b", + "metadata": {}, + "source": [ + "We can also plot it for a range of values" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0a3f756f-886b-455d-b1f2-53060088dce9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABC30lEQVR4nO3deXhU5eH28e+ZmezLhADZSELCGvZ9ExfUFFRcUESpuKEFrWBFXAptxfanNW5VXzewtipaEMWKaFUsokKRsAVB9j0QCFkgMNnINjPvH8G0UVSWSc4s9+e6zqWcmUzujFyZ2+c853kMt9vtRkRERMSLWMwOICIiIvJ9KigiIiLidVRQRERExOuooIiIiIjXUUERERERr6OCIiIiIl5HBUVERES8jgqKiIiIeB2b2QHOhMvlIj8/n6ioKAzDMDuOiIiInAK3201ZWRlJSUlYLD89RuKTBSU/P5+UlBSzY4iIiMgZyMvLIzk5+Sef45MFJSoqCqj/AaOjo01OIyIiIqeitLSUlJSUhs/xn+KTBeW7yzrR0dEqKCIiIj7mVKZnaJKsiIiIeB0VFBEREfE6KigiIiLidVRQRERExOuooIiIiIjXUUERERERr6OCIiIiIl5HBUVERES8jgqKiIiIeJ3TLijLli3jiiuuICkpCcMw+OCDDxo97na7mTFjBomJiYSFhZGZmcnOnTsbPaekpIRx48YRHR1NTEwMt99+O+Xl5Wf1g4iIiIj/OO2CUlFRQa9evXjppZdO+viTTz7J888/z6xZs1i1ahURERGMGDGCqqqqhueMGzeOzZs3s3jxYv71r3+xbNkyJk6ceOY/hYiIiPgVw+12u8/4iw2DBQsWMGrUKKB+9CQpKYn77ruP+++/HwCHw0F8fDxvvPEGY8eOZevWrXTt2pU1a9bQv39/ABYtWsRll13GgQMHSEpK+tnvW1pait1ux+FwaC8eERERH3E6n98enYOyd+9eCgoKyMzMbDhnt9sZNGgQ2dnZAGRnZxMTE9NQTgAyMzOxWCysWrXqpK9bXV1NaWlpo6MpbCso5fcLNvLRhvwmeX0RERE5NR4tKAUFBQDEx8c3Oh8fH9/wWEFBAXFxcY0et9lsxMbGNjzn+7KysrDb7Q1HSkqKJ2M3WLK1iDmr9vPGitwmeX0RERE5NT5xF8/06dNxOBwNR15eXpN8nzH9k7FZDHL2HWV7QVmTfA8RERH5eR4tKAkJCQAUFhY2Ol9YWNjwWEJCAkVFRY0er6uro6SkpOE53xcSEkJ0dHSjoynERYWS2aV+9Oft1fub5HuIiIjIz/NoQUlPTychIYElS5Y0nCstLWXVqlUMGTIEgCFDhnDs2DFycnIanvPFF1/gcrkYNGiQJ+OckV8OSgXg/XUHqKp1mpxGREQkMNlO9wvKy8vZtWtXw5/37t3L+vXriY2NJTU1lSlTpvDoo4/SsWNH0tPTeeihh0hKSmq406dLly5ccsklTJgwgVmzZlFbW8vkyZMZO3bsKd3B09TO69CKNjFhHDx2nE82HuKavslmRxIREQk4pz2CsnbtWvr06UOfPn0AmDp1Kn369GHGjBkAPPjgg9x9991MnDiRAQMGUF5ezqJFiwgNDW14jTlz5pCRkcHFF1/MZZddxrnnnstf//pXD/1IZ8diMfjlwPpJuLrMIyIiYo6zWgfFLE29DkpRaRVDHv8Cp8vN4nvPp2N8lMe/h4iISKAxbR0UfxEXHUpml/pbod9e3TR3DImIiMiPU0H5Eb8cWD9Z9p+aLCsiItLsVFB+xHkdW9MmJgzH8Vo+3XTI7DgiIiIBRQXlR1gtBmMHnJgsu0qXeURERJqTCspPGNM/BavFYHVuCbuKtLKsiIhIc1FB+QkJ9lAuytBkWRERkeamgvIzbtBkWRERkWangvIzzu/UmiR7KMcqa/ls88l3WxYRERHPUkH5GVaLwfUD6kdR5q7SyrIiIiLNQQXlFFw3IBmLAav2lrC7uNzsOCIiIn5PBeUUJNrDGibLztP+PCIiIk1OBeUUfbey7Hs5B6iu02RZERGRpqSCcoou6NSaRHsoRytr+WxzodlxRERE/JoKyimyWS1c1/+7lWV1mUdERKQpqaCchusGpGAxIHvPEfZosqyIiEiTUUE5DW1iwhjW+cRk2TVaWVZERKSpqKCcJk2WFRERaXoqKKfpws6tiY8OoaSihn9rsqyIiEiTUEE5TTarheu/myyrNVFERESahArKGbhuQAqGASt2H2Hv4Qqz44iIiPgdFZQzkNwinGGdWgMwb41GUURERDxNBeUMNUyWXXuAmjqXyWlERET8iwrKGbooI464qBCOVNSwaHOB2XFERET8igrKGbJZLYw9MYryj+x9JqcRERHxLyooZ+GGgalYLQarc0vYVlBqdhwRERG/oYJyFhLsoYzoFg/AmxpFERER8RgVlLN00+A0AD745iClVbXmhhEREfETKihnaXC7WDrFR1JZ4+SfOQfMjiMiIuIXVFDOkmEY3DS4LQBvrdyH2+02OZGIiIjvU0HxgKv7JhMZYmNPcQVf7zpidhwRERGfp4LiAZEhNq7p2waAN7NzzQ0jIiLiB1RQPOS7yzyfby3k4LHjJqcRERHxbSooHtIxPorB7WJxueHtVdqfR0RE5GyooHjQzUPSgPoNBKvrnOaGERER8WEqKB70i67xxEeHcLi8hkWbtD+PiIjImVJB8aAgq4UbBtbPRdHKsiIiImdOBcXDfjkwBZvFIGffUTbnO8yOIyIi4pNUUDwsLjqUS7onAPCWRlFERETOiApKE/husuwH6w/iqNT+PCIiIqdLBaUJDEhrQUZCFFW1Lubn5JkdR0RExOeooDQBwzC4aUj9ZNk5q/bjcml/HhERkdOhgtJERvVuQ1SIjb2HK1i+67DZcURERHyKCkoTiQixMbpfMqBbjkVERE6XCkoTuvHE/jxfbCvkwNFKk9OIiIj4DhWUJtQhLpKhHVrictfPRREREZFTo4LSxG4anAbAO2vyqKrV/jwiIiKnQgWliWV2iSPJHkpJRQ2fbDxkdhwRERGfoILSxGxWCzcMSgU0WVZERORUqaA0g+sHpBJkNVifd4yNB7Q/j4iIyM9RQWkGraNCuKxHIgBvZueaG0ZERMQHqKA0k5tPrCy7cEM+R8qrTU4jIiLi3VRQmknf1Bb0SrZTU+dirm45FhER+UkqKM3EMAxuOzcdgDdX7qOmzmVyIhEREe+lgtKMLu2eSHx0CMVl1Xy8Md/sOCIiIl5LBaUZBdss3DwkDYC/L9+L261djkVERE5GBaWZ3TAwlRCbhU0HS1mTe9TsOCIiIl5JBaWZtYgI5pq+9bscv7Z8r8lpREREvJMKigluG5oGwL+3FJBXol2ORUREvk8FxQQd46M4r2MrXG6YvSLX7DgiIiJex+MFxel08tBDD5Genk5YWBjt27fnkUceaTQh1O12M2PGDBITEwkLCyMzM5OdO3d6OopX++6W43fW5FFeXWdyGhEREe/i8YLyxBNPMHPmTF588UW2bt3KE088wZNPPskLL7zQ8Jwnn3yS559/nlmzZrFq1SoiIiIYMWIEVVVVno7jtS7o2Jr2rSMoq65j/to8s+OIiIh4FY8XlBUrVnDVVVcxcuRI0tLSuPbaaxk+fDirV68G6kdPnnvuOf7whz9w1VVX0bNnT958803y8/P54IMPPB3Ha1ksBuOH1o+ivLEiF6dLtxyLiIh8x+MF5ZxzzmHJkiXs2LEDgA0bNrB8+XIuvfRSAPbu3UtBQQGZmZkNX2O32xk0aBDZ2dknfc3q6mpKS0sbHf7gmr5tsIcFse9IJV9sKzI7joiIiNfweEGZNm0aY8eOJSMjg6CgIPr06cOUKVMYN24cAAUFBQDEx8c3+rr4+PiGx74vKysLu93ecKSkpHg6tinCg238cmAqoFuORURE/pfHC8q7777LnDlzmDt3LuvWrWP27Nk8/fTTzJ49+4xfc/r06TgcjoYjL89/5mzcPKQtVotB9p4jbMn3j5EhERGRs+XxgvLAAw80jKL06NGDm266iXvvvZesrCwAEhISACgsLGz0dYWFhQ2PfV9ISAjR0dGNDn+RFBPGZT0SAXjta42iiIiIQBMUlMrKSiyWxi9rtVpxuep3701PTychIYElS5Y0PF5aWsqqVasYMmSIp+P4hO8WbvtwfT7FZdXmhhEREfECHi8oV1xxBX/+85/5+OOPyc3NZcGCBTzzzDNcffXVABiGwZQpU3j00Uf58MMP2bhxIzfffDNJSUmMGjXK03F8Qp/UFvRJjaHG6WLOqn1mxxERETGdzdMv+MILL/DQQw9x1113UVRURFJSEnfccQczZsxoeM6DDz5IRUUFEydO5NixY5x77rksWrSI0NBQT8fxGbcNTefu/d/wj5X7+PWw9oTYrGZHEhERMY3h/t8lXn1EaWkpdrsdh8PhN/NR6pwuzn/yS/IdVTw9phfX9ks2O5KIiIhHnc7nt/bi8RI2q4Wbz0kD4O/L9+KDvVFERMRjVFC8yNgBKYQFWdl6qJSVe0rMjiMiImIaFRQvEhMezOh+bQDdciwiIoFNBcXLfLc/z+dbC9l3pMLkNCIiIuZQQfEy7VtHcmHn1rjd9ZsIioiIBCIVFC9027n1oyjz1x6grKrW5DQiIiLNTwXFC53boRWd4iMpr67j7dX7zY4jIiLS7FRQvJBhGPzq3HYAvLY8l5o6l8mJREREmpcKipe6qk8ScVEhFJRW8eGGfLPjiIiINCsVFC8VYrM2zEX567LduFxauE1ERAKHCooXu2FQKpEhNnYUlvPVjiKz44iIiDQbFRQvFh0axLhBqQDMWrrH5DQiIiLNRwXFy40fmk6Q1WD13hLW7T9qdhwREZFmoYLi5RLsoYzqXb/8/V81iiIiIgFCBcUH3HFB/S3Hn20pYE9xuclpREREmp4Kig/oEBdFZpd43G549T/aRFBERPyfCoqPuPPEKMo/1x2gqKzK5DQiIiJNSwXFR/RPi6Vf2xbU1LmYrU0ERUTEz6mg+JA7zq8fRXkrex/l1XUmpxEREWk6Kig+JLNLPO1aR1BaVcc8bSIoIiJ+TAXFh1gsRsMoyt+X79UmgiIi4rdUUHzMqD5taB0VwiFHFR9pE0EREfFTKig+JsRm5bah9ZsIvrJsN263NhEUERH/o4LigxptIri92Ow4IiIiHqeC4oPsYUHc0LCJ4G6T04iIiHieCoqPGj80jSCrwaq9JXyjTQRFRMTPqKD4qER7GFd9t4ngMm0iKCIi/kUFxYdNPHHL8aLNBew9XGFyGhEREc9RQfFhneKjuDgj7sQmghpFERER/6GC4uPuuKA9AO/lHKC4rNrkNCIiIp6hguLjBqS1oE9qjDYRFBERv6KC4uMMw+CO8+tHUd7MzqW0qtbkRCIiImdPBcUPDO8aT4e4SEqr6ngre5/ZcURERM6aCoofsFgMJl1YP4ry9+V7qaypMzmRiIjI2VFB8RNX9EyibctwSipqmLtqv9lxREREzooKip+wWS3cNax+FOWVZXuoqnWanEhEROTMqaD4kav7JNMmJozismreXZtndhwREZEzpoLiR4JtFu68oH512Vlf7aamzmVyIhERkTOjguJnxvRPIS4qhHxHFe+vO2B2HBERkTOiguJnQoOsDXv0vPzVbuqcGkURERHfo4Lih24YlErLiGD2l1Ty4YZ8s+OIiIicNhUUPxQebOP289IBePHLXThdbpMTiYiInB4VFD910+C22MOC2FNcwaebDpkdR0RE5LSooPipqNAgxg9NA+DFL3bh0iiKiIj4EBUUPzb+nHQiQ2xsKyjj862FZscRERE5ZSoofsweHsTNQ9oC8MIXu3C7NYoiIiK+QQXFz91+bjphQVY2HnTw1Y5is+OIiIicEhUUP9cyMoRxg1IBeGHJTo2iiIiIT1BBCQATz29HsM3Cuv3HyN59xOw4IiIiP0sFJQDERYcydkAKUD8XRURExNupoASIOy5oT5DVIHvPEdbmlpgdR0RE5CepoASINjFhjO6bDGgURUREvJ8KSgC5a1gHrBaDpTuK2ZB3zOw4IiIiP0oFJYCktgznql5JQP0ePSIiIt5KBSXA3HVhBwwDFm8pZEt+qdlxRERETkoFJcB0iIvksh6JADy/ZKfJaURERE5OBSUATbm4I4YBizYXsPGAw+w4IiIiP6CCEoA6xkcxqncbAJ5ZvN3kNCIiIj+kghKg7rm4I1aLwZfbi8nZd9TsOCIiIo00SUE5ePAgN954Iy1btiQsLIwePXqwdu3ahsfdbjczZswgMTGRsLAwMjMz2blT8yGaU1qrCK49sS6KRlFERMTbeLygHD16lKFDhxIUFMSnn37Kli1b+Mtf/kKLFi0anvPkk0/y/PPPM2vWLFatWkVERAQjRoygqqrK03HkJ9x9cQeCrAZf7zrCit2HzY4jIiLSwHB7eHvbadOm8fXXX/Of//znpI+73W6SkpK47777uP/++wFwOBzEx8fzxhtvMHbs2J/9HqWlpdjtdhwOB9HR0Z6MH3BmLNzEm9n76N+2BfPvHIJhGGZHEhERP3U6n98eH0H58MMP6d+/P2PGjCEuLo4+ffrw6quvNjy+d+9eCgoKyMzMbDhnt9sZNGgQ2dnZJ33N6upqSktLGx3iGZMu7ECIzcLafUdZuqPY7DgiIiJAExSUPXv2MHPmTDp27Mhnn33Gr3/9a37zm98we/ZsAAoKCgCIj49v9HXx8fENj31fVlYWdru94UhJSfF07IAVHx3KTYPbAvDM4h14eEBNRETkjHi8oLhcLvr27ctjjz1Gnz59mDhxIhMmTGDWrFln/JrTp0/H4XA0HHl5eR5MLHcOa094sJVvDzhYvKXQ7DgiIiKeLyiJiYl07dq10bkuXbqwf/9+ABISEgAoLGz8QVhYWNjw2PeFhIQQHR3d6BDPaRUZwvihaUD9KIrLpVEUERExl8cLytChQ9m+vfFtqzt27KBt2/rLCOnp6SQkJLBkyZKGx0tLS1m1ahVDhgzxdBw5RRPPa09UqI1tBWV8vPGQ2XFERCTAebyg3HvvvaxcuZLHHnuMXbt2MXfuXP76178yadIkAAzDYMqUKTz66KN8+OGHbNy4kZtvvpmkpCRGjRrl6ThyiuzhQUw4rx0Az36+gzqny+REIiISyDxeUAYMGMCCBQt4++236d69O4888gjPPfcc48aNa3jOgw8+yN13383EiRMZMGAA5eXlLFq0iNDQUE/HkdMwfmgaMeFB7Cmu4IP1+WbHERGRAObxdVCag9ZBaTqzlu7m8U+3kRIbxhf3DSPIqt0QRETEM0xdB0V8281D2tIqMoS8kuPMX3vA7DgiIhKgVFCkkfBgG5MubA/AC1/spKrWaXIiEREJRCoo8gO/HJhKoj2UQ44q3l693+w4IiISgFRQ5AdCg6xMvqgDAC99uZvjNRpFERGR5qWCIic1pl8KKbFhHC6v5s3sXLPjiIhIgFFBkZMKtlm45+JOQP2dPWVVtSYnEhGRQKKCIj9qVO8k2rWO4GhlLa9/nWt2HBERCSAqKPKjbFYL92bWj6K8umwPRytqTE4kIiKBQgVFftLIHol0SYymrLqOF7/cZXYcEREJECoo8pMsFoPpl2YA8GZ2LnkllSYnEhGRQKCCIj/r/E6tOa9jK2qdbp7+9/af/wIREZGzpIIip+S3l2RgGLBwfT4bDzjMjiMiIn5OBUVOSfc2dq7u3QaAxz7Zig/uMSkiIj5EBUVO2dThnQi2Wcjec4SvdhSbHUdERJqA2+1mV1GZ2TFUUOTUJbcIZ/w5aQA8/sk2nC6NooiI+Jt/fXuIXzy7jIcXbjI1hwqKnJa7hnXAHhbE9sIy/rnugNlxRETEg6pqnTyxaBtuN8RGhJiaRQVFTos9PIjJF9ZvJPjMv3doI0ERET8ye0UuB44eJz46hAnnp5uaRQVFTttNQ9rSJiaMgtIqXvt6r9lxRETEA0oqahoW5Lx/eGfCg22m5lFBkdMWGmTlgRGdAZj51W6OlFebnEhERM7W//t8B2VVdXRNjGZ032Sz46igyJm5slcS3ZKiKa+u44UvtAS+iIgv211czpxV+wH4w8guWCyGyYlUUOQMWSwGv7usCwD/WLmP3MMVJicSEZEzlfXJNupcbi7OiOOcDq3MjgOooMhZGNqhFRd0ak2dy81TWgJfRMQnZe8+wudbC7FaDKaf+B9Pb6CCImdl2qX1S+B//O0hvtl/1Ow4IiJyGlwuN3/+ZAsANwxMpUNcpMmJ/ksFRc5Kl/+ZTJX16TYtgS8i4kMWfHOQTQdLiQyxMSWzo9lxGlFBkbM29RedCLFZWL23hC+2FZkdR0RETsHxGmfDDvV3XdielpHmLsz2fSooctaSYsK47dz6BX0e/3QbdU6XyYlEROTn/H35Hg45qmgTE8ZtQ81dlO1kVFDEI349rD0twoPYWVTOezlaAl9ExJsVlVUx86vdADx4SWdCg6wmJ/ohFRTxiOjQIO6+qP765TOLd1BZU2dyIhER+THPLt5JRY2TXsl2ruiZZHack1JBEY+5cXBbUmPDKSqr5tVlWgJfRMQbbS8o4501JxZlu7yrVyzKdjIqKOIxwTYLD15yYgn8pbvIP3bc5EQiIvJ9j32yFZcbLumWwIC0WLPj/CgVFPGokT0SGZgWS1Wti6xPt5kdR0RE/seyHcUs3VFMkNVg2qUZZsf5SSoo4lGGYfDwlV2xGPDRhnxW7y0xO5KIiABOl5vHPtkKwE2D00hrFWFyop+mgiIe1y3JztiBqQD88cPNOF1avE1ExGzz1+axraAMe1gQv7m4g9lxfpYKijSJ+4d3JjrUxpZDpcw7MRlLRETMUVFdx18W7wDg7os6EBMebHKin6eCIk0iNiKYe3/RCYCnP9uOo7LW5EQiIoHrlWV7KC6rJjU2nJuGtDU7zilRQZEmc+PgtnSMi+RoZS3Pfr7D7DgiIgHp4LHj/HVZ/aJs0y7NIMTmfYuynYwKijSZIKuFh6/oBsBbK/exo7DM5EQiIoHnzx9voarWxcC0WC7tnmB2nFOmgiJN6tyOrRjRLR6ny83/fbRFux2LiDSj5TsP88nGAqwWgz9d1Q3D8M5F2U5GBUWa3B9GdiXYZmH5rsP8e0uh2XFERAJCTZ2Lhz/cBMBNg9vSJTHa5ESnRwVFmlxKbDgTz2sHwKMfb6Gq1mlyIhER//f613vZXVxBq8j/3rTgS1RQpFncdWF7EqJDySs5zt/+s8fsOCIifq3AUcXzS3YC8NtLMrCHBZmc6PSpoEizCA+2Mf2y+mWVX/pyN4cc2qdHRKSpPPbJVipqnPRJjWF032Sz45wRFRRpNlf2SmJAWguO1zp5XPv0iIg0iZV7jvDhhnwMAx65qrvX7lb8c1RQpNkYhsHDV3TDMGDh+nzW5mqfHhERT6p1unh44WYAbhiYSvc2dpMTnTkVFGlW3dvYGTsgBYA/fqR9ekREPOmt7H1sLyyjRXgQD4zobHacs6KCIs3u/uGdiQq1selgKe+uzTM7joiIXygqq+LZE/vtPDAiwyf22/kpKijS7FpGhnBvZv0tb099th3Hce3TIyJytp74dDtl1XX0TLZz/YmRal+mgiKmuGlI/T49JRU1/L/Pd5odR0TEp+XsK+Gf6w4A8Kcru2H10Ymx/0sFRUwRZLUw44quALyZncv2Au3TIyJyJpwuNzNOTIy9vn8KfVJbmJzIM1RQxDTndWzNiG7x1Lnc/G7BRlyaMCsictrmrt7P5vxSokNtPHiJb0+M/V8qKGKqP17ZjYhgKzn7jvKOJsyKiJyWkooanv5sOwD3j+hMy8gQkxN5jgqKmCrRHsZ9w+sbf9YnWykuqzY5kYiI73jqs204jtfSJTGaGwammh3Ho1RQxHS3nJNGjzZ2SqvqePTjLWbHERHxCRvyjjFvTf3I8yNXdcNm9a+PdP/6acQnWS0Gj13dA8uJFWaX7Sg2O5KIiFdzudzMWLgJtxuu6dOG/mmxZkfyOBUU8Qo9ku3cck4aAA8t3ERVrdPcQCIiXuydtXlsOOAgMsTGtBMbsfobFRTxGvcN70xCdCj7jlTy4he7zI4jIuKVisqqyPpkKwBTMjsSFxVqcqKmoYIiXiMyxMYfr+wGwCvLdrOjUGujiIh8358+3EJpVR092ti59cTIsz9SQRGvMqJbPJld4ql1uvm91kYREWlk8ZZCPt54CKvFIOuaHn43MfZ/+e9PJj7JMAz+dFU3woOtrMk9yvwcrY0iIgJQVlXLQx9sAuBX56XTvY3d5ERNq8kLyuOPP45hGEyZMqXhXFVVFZMmTaJly5ZERkYyevRoCgsLmzqK+Ig2MWFM/UX9ZoKPfbKNw+VaG0VE5KnPtlNQWkXbluFMubiT2XGaXJMWlDVr1vDKK6/Qs2fPRufvvfdePvroI+bPn8/SpUvJz8/nmmuuacoo4mNuPSeNronROI7X8uePt5odR0TEVDn7Snhr5T4Asq7uQViw1eRETa/JCkp5eTnjxo3j1VdfpUWL/25c5HA4+Pvf/84zzzzDRRddRL9+/Xj99ddZsWIFK1eubKo44mNsVgtZ1/TAMGDBNwdZvvOw2ZFERExRXefkt//ciNsNY/olc06HVmZHahZNVlAmTZrEyJEjyczMbHQ+JyeH2traRuczMjJITU0lOzv7pK9VXV1NaWlpo0P8X6+UGG4e3BaAP3ywUWujiEhAmvnVbnYVldMqMpjfj+xidpxm0yQFZd68eaxbt46srKwfPFZQUEBwcDAxMTGNzsfHx1NQUHDS18vKysJutzccKSkpTRFbvNB9IzoTHx1C7pFKXv5Sa6OISGDZWVjGSyd+9z18RTdiwoNNTtR8PF5Q8vLyuOeee5gzZw6hoZ5ZPGb69Ok4HI6GIy9Pd3YEiujQIP54Rf3aKDOX7mZXkdZGEZHA4HK5mfb+Rmqdbi7OiOPynolmR2pWHi8oOTk5FBUV0bdvX2w2GzabjaVLl/L8889js9mIj4+npqaGY8eONfq6wsJCEhISTvqaISEhREdHNzokcFzSPYGLM+Kodbr53YJNuN1aG0VE/N+c1fvJ2XeUiGArj4zqjmEYZkdqVh4vKBdffDEbN25k/fr1DUf//v0ZN25cw78HBQWxZMmShq/Zvn07+/fvZ8iQIZ6OI37gu7VRwoKsrN5bwvy1B8yOJCLSpA45jvPEp9sAePCSDJJiwkxO1Pxsnn7BqKgounfv3uhcREQELVu2bDh/++23M3XqVGJjY4mOjubuu+9myJAhDB482NNxxE8ktwjn3l905LFPtvHox1s4v1NrEuz+uf+EiAQ2t9vNQx9spry6jj6pMdx44maBQGPKSrLPPvssl19+OaNHj+b8888nISGB999/34wo4kNuG5pOr2Q7pVV1TH//W13qERG/9OmmAj7fWkiQ1eCJ0T2xWgLr0s53DLcP/pYvLS3FbrfjcDg0HyXA7CwsY+Tzy6lxunjy2p5c1193dImI/3BU1pL57FKKy6r5zUUdmDq8s9mRPOp0Pr+1F4/4lI7xUUwdXr/E8yMfbSH/2HGTE4mIeE7Wp1spLqumfesIJl3Uwew4plJBEZ8z4bx29EmNoay6jmnvb9SlHhHxC9m7jzBvTf0yGo+P7kmIzf+Xs/8pKijic6wWg6fH9CLEZmHZjmLeWaN1cUTEt1XVOvndgo0AjBuUyoC0WJMTmU8FRXxS+9aRPDCi/trsox9v5cDRSpMTiYicuScWbWPv4Qrio0P47aUZZsfxCioo4rPGD02nf9sWlFfX8dt/6q4eEfFNK3Yd5vWvcwF4YnRPokODzA3kJVRQxGdZLQZPXtuT0CALX+86wpxV+82OJCJyWhzHa7l//gag/tLOsM5xJifyHioo4tPatY7kwRH1w6GPfbKVvBJd6hER3/GnjzaT76iibctwfndZ4OxUfCpUUMTn3XpOGgPTYqmscfLge9/iculSj4h4v0WbDvH+uoNYDHjmul5EhHh8cXefpoIiPs9iMXhqTE/Cgqxk7znCP1btMzuSiMhPKiqr4ncLNgFw5wXt6ddWd+18nwqK+IW2LSOYfln9pZ6sT7ax70iFyYlERE7O7Xbzu/c3UlJRQ5fEaKZkdjI7kldSQRG/ceOgtgxp15LjtU4e0KUeEfFS89ce4POtRQRbLTx7fS+CbfooPhm9K+I3LCfu6gkPtrJ6bwmzs3PNjiQi0kheSSV/+mgzAPcN70RGgvaT+zEqKOJXUmL/OxP+u4WPRES8gdPl5r53N1BR42RAWgt+dV47syN5NRUU8TvjBqVybodWVNW6eGD+Bpy61CMiXuC15XtZnVtCeLCVv4zpjdVimB3Jq6mgiN8xDIPHR/cgMsTG2n1HmbV0t9mRRCTAbS8o46nPtgPw0OVdSW0ZbnIi76eCIn4puUU4D1/RFYBnFu9g3f6jJicSkUBVU+fi3nfWU+N0cVFGHGMHpJgdySeooIjfurZfMlf2SsLpcvObt7+htKrW7EgiEoCeX7KTLYdKaREexOOje2AYurRzKlRQxG8ZhsGjV3cnJTaMA0eP87v3N2pDQRFpVjn7jvLyV7sA+PPVPYiLCjU5ke9QQRG/Fh0axPNj+2CzGPzr20PMzzlgdiQRCRCVNXXc9+56XG64uk8bLuuRaHYkn6KCIn6vT2oLpg6vX6nx4YWb2V1cbnIiEQkEj368ldwjlSREh/LHK7uZHcfnqKBIQLjz/PYM7VC/yuzdc7+hus5pdiQR8WP/+jafuav2A/D0mF7Yw4JMTuR7VFAkIFgsBs9c15vYiGC2HCrliU+3mx1JRPxU7uEKpv1zIwB3DWvPuR1bmZzIN6mgSMCIjw7l6TE9AXjt6718sa3Q5EQi4m+q65xMfnsd5dV1DEhrwdRfaCPAM6WCIgHloox4bj0nDYD7539LUWmVuYFExK889vFWNh2sv6X4+V/2wWbVx+yZ0jsnAWfapRl0SYympKKGe99dr12PRcQjPt14iNnZ+wB45rreJNrDTE7k21RQJOCEBll54Zd9CAuy8vWuI7yybI/ZkUTEx+0/UsmD730LwB0XtOPCjDiTE/k+FRQJSB3iIvnjlfVL4f/l39v5Rkvhi8gZ+m7eSVl1Hf3atuD+4Z3NjuQXVFAkYF3XP4WRPROpc7n5zTwthS8iZ+bxT7fx7QEH9rD6eSdBmnfiEXoXJWAZhsFjV/egTUwYeSXH+cOCTVoKX0ROy2ebC3j961wA/jKmF21iNO/EU1RQJKDV/x9Pb6wWgw835POelsIXkVOUV1LJA/M3ADDhvHQyu8abnMi/qKBIwOvXNpZ7MzsC8NDCTWw9VGpyIhHxdjV1Lu5++xtKq+ronRLDg5dkmB3J76igiAC/HtaB8zq2oqrWxR1v5XCsssbsSCLixZ5ctI31eceIDrXx4g2ad9IU9I6KAFaLwfNj+5DcIoz9JZXcM289Tq2PIiIn8fmWQv62fC9Qv89OcotwkxP5JxUUkRNaRATzyk39CLFZWLqjmOc+32F2JBHxMgePHee+E/NObhuazvBuCSYn8l8qKCL/o1uSncdH9wDghS928dnmApMTiYi3qHW6uHvuOhzHa+mVbGfapZp30pRUUES+5+o+yQ379dz37gZ2F5ebG0hEvMIj/9rCuv3HiAq18eINfQm26SO0KendFTmJ34/swsC0WMqr67jjrRzKq+vMjiQiJnp79X7ezN6HYcCz1/UmJVbzTpqaCorISQRZLbw4rg/x0SHsKirn/nc3aBE3kQC1JreEGQs3AXDfLzppvZNmooIi8iPiokKZeWM/gqwGizYXMHPpbrMjiUgzO3jsOHe+lUOt083InolMurCD2ZEChgqKyE/om9qCP17ZDYCnP9vOsh3FJicSkeZyvMbJxDfXcqSihq6J0Tx1bU8MwzA7VsBQQRH5GTcMTOX6/im43PCbed+QV1JpdiQRaWJut5sH3tvA5vxSWkYE8+ot/QkPtpkdK6CooIj8DMMw+NNV3eiVbOdYZS13vJXD8Rqn2bFEpAm9/NVu/vXtIWwWg5k39tMmgCZQQRE5BaFBVmbe2I+WEcFsOVTK7xds1KRZET/1+ZZCnv73dgD+dFU3BqbHmpwoMKmgiJyipJgwXrihD1aLwfvfHGT2ilyzI4mIh+0sLGPKO+txu+HGwamMG9TW7EgBSwVF5DSc074V00+sHvnox1vJ3n3E5EQi4imOylomvLmW8uo6BqXH8vAV3cyOFNBUUERO0+3npnNlryTqXG7ueGstu4rKzI4kImepzuli8tvryD1SSZuYMF4e11c7FJtM777IaTIMgyev7Unf1BhKq+q49fU1FJdVmx1LRM7C459u4z87DxMWZOXVm/vTMjLE7EgBTwVF5AyEnvgl1rZlOAeOHudXs9dQWaPl8EV80T9zDvC35XsB+Mt1veiaFG1yIgEVFJEz1jIyhDfGD6RFeBAbDjj4zdvrcbp0Z4+IL/lm/1GmL9gIwG8u6sBlPRJNTiTfUUEROQvprSJ49eb+BNssfL61kEf+tcXsSCJyivYfqWTCmznU1LkY3jWeKZmdzI4k/0MFReQs9U+L5ZnregHwxopc/n5iqFhEvNfh8mpufm0Vh8ur6ZIYzTPX98Zi0TL23kQFRcQDLu+ZxLSG24+3sGhTgcmJROTHVFTXcfsba8g9UklyizBmjx9AZIiWsfc2KigiHnLH+e0YNygVtxvumfcN3+w/anYkEfmeWqeLX89Zx4YDDmIjgnnztoHERYeaHUtOQgVFxEMMw+BPV3bjws6tqa5z8avZa9l/RBsLingLt9vNb9/7lmU7igkLsvL3W/rTrnWk2bHkR6igiHiQzWrhxRv60i0pmiMVNdz6xmqOVdaYHUtEgCcWbef9bw5itRi8PK4vfVJbmB1JfoIKioiHRYTYeO3WASTZQ9lTXMHEN3OortPuxyJmem35XmYt3Q3A49f04MKMOJMTyc9RQRFpAvHRobw+fiBRITZW55bwwPxvcWmNFBFTfLQhn0c+rl8C4IERnRnTP8XkRHIqVFBEmkjnhChm3tgPm8Xgww35PHVi+3YRaT4rdh3mvnc34HbDLUPactew9mZHklPk8YKSlZXFgAEDiIqKIi4ujlGjRrF9e+NfzFVVVUyaNImWLVsSGRnJ6NGjKSws9HQUEdOd27EVWdf0AGDmV7uZ+dVukxOJBI7N+Q4mvpVDjdPFZT0SmHFFNwxDa534Co8XlKVLlzJp0iRWrlzJ4sWLqa2tZfjw4VRUVDQ859577+Wjjz5i/vz5LF26lPz8fK655hpPRxHxCmP6p/DgJZ0BeGLRNl7TQm4iTS6vpJJbX19DeXUdg9Jjeea63li1EJtPMdxud5NeGC8uLiYuLo6lS5dy/vnn43A4aN26NXPnzuXaa68FYNu2bXTp0oXs7GwGDx78s69ZWlqK3W7H4XAQHa1NncQ3PPPv7Tz/xS4AHru6BzcMSjU5kYh/OlJezZhZ2ew5XEFGQhTv3jmE6NAgs2MJp/f53eRzUBwOBwCxsbEA5OTkUFtbS2ZmZsNzMjIySE1NJTs7u6njiJjm3l90YuL57QD4/QcbeX/dAZMTififypo6bpu9lj2HK2gTE8bs2waqnPioJl3b1+VyMWXKFIYOHUr37t0BKCgoIDg4mJiYmEbPjY+Pp6Dg5MuDV1dXU11d3fDn0tLSJsss0lQMw2D6pRlU1zqZnb2P++dvINhm4fKeSWZHE/ELlTV1jH99DRvyjhETHsTs2wYSr1VifVaTjqBMmjSJTZs2MW/evLN6naysLOx2e8ORkqJbxMQ3GYbBw1d0Y+yAFFxumDJvPf/erH17RM7Wd+Vk1d4SokJsvH7rADrEaZVYX9ZkBWXy5Mn861//4ssvvyQ5ObnhfEJCAjU1NRw7dqzR8wsLC0lISDjpa02fPh2Hw9Fw5OXlNVVskSZnsRj8+eoejOqdRJ3LzeS537B0R7HZsUR81vfLyezbB2qVWD/g8YLidruZPHkyCxYs4IsvviA9Pb3R4/369SMoKIglS5Y0nNu+fTv79+9nyJAhJ33NkJAQoqOjGx0ivsxqMXh6TC8u7Z5AjdPFxDfXkr37iNmxRHzOycpJX5UTv+DxgjJp0iT+8Y9/MHfuXKKioigoKKCgoIDjx48DYLfbuf3225k6dSpffvklOTk5jB8/niFDhpzSHTwi/sJmtfD/xvbh4ow4qutc3D57DTn7SsyOJeIzVE78m8dvM/6xRXBef/11br31VqB+obb77ruPt99+m+rqakaMGMHLL7/8o5d4vk+3GYs/qap1MuHNtfxn52GiQmzMmTCInskxZscS8WoqJ77pdD6/m3wdlKaggiL+5niNk1teX83qvSXYw4KYN3EwXRL1d1vkZFROfJdXrYMiIj8vLNjKa7cOoHdKDI7jtdz4t1XsLCwzO5aI11E5CRwqKCJeIjLExuzbBtItKZojFTVc90o26/OOmR1LxGuonAQWFRQRL2IPC+Iftw+iV0oMRytrueHVlXy967DZsURMp3ISeFRQRLxMi4hg5vxqEEM7tKSyxsn419ewaNMhs2OJmEblJDCpoIh4ocgQG6/dOqBhnZS75qxj3ur9ZscSaXaO47Xc+prKSSBSQRHxUiE2Ky/e0LdhWfxp729k1tLdZscSaTb5x44zZtYKVueqnAQiFRQRL2a1GGRd04M7L2gPwOOfbiPrk6344OoAIqdlW0Ep17y8gh2F5cRHh/DOHUNUTgKMCoqIlzMMg2mXZjD90gwAXlm2h2n/3Eid02VyMpGmsWLXYcbMzKagtIqOcZG8f9dQuiZpXaBAo4Ii4iPuuKA9T47uicWAd9bmMXnuN1TVOs2OJeJRC9cf5JbXV1NWXcfA9Fjeu/Mc2sSEmR1LTKCCIuJDrhuQwsvj+hFstbBocwG3vbGG8uo6s2OJnDW3280rS3dzz7z11DrdjOyZyJu3DcQeHmR2NDGJCoqIj7mkewJv3DaAiGArK3Yf4YZXV1JSUWN2LJEz5nS5+dNHW8j6dBsAt5+bzgtj+xAaZDU5mZhJBUXEB53TvhVvTxxMbEQw3x5wcO2sFew9XGF2LJHTVlXrZNKcdbyxIheAP4zswkOXd8ViOfnGsxI4VFBEfFTP5BjevWMISfZQ9hRXcNWLy1m6o9jsWCKn7GhFDTf+bRWLNhcQbLXw4g19+NV57cyOJV5CBUXEh3WIi+SDyUPp17YFpVV1jH99Na8s3a3bkMXr5ZVUMnrWCtbuO0p0qI03bx/I5T2TzI4lXkQFRcTHxUWFMnfCIK7vX7+gW9an25jyznrd4SNea0PeMa6ZuYI9xRUk2UN579fnMLhdS7NjiZdRQRHxAyE2K4+P7sH/XdUNm8Vg4fp8rp21goPHjpsdTaSB2+1mzqp9jJmVTXFZNRkJUbx/11A6xUeZHU28kAqKiJ8wDIObh6Tx1u2DiI0IZtPBUq56cTlrckvMjibC8Ron983fwO8XbKLG6WJ413jevXMICfZQs6OJl1JBEfEzQ9q35MPJQ+mSGM3h8hpueHUlc1btMzuWBLC9hyu4+uWveX/dQawWg+mXZvDKTf2IDtUaJ/LjVFBE/FByi3D++eshjOyZSK3Tze8XbOL3CzZSU6fl8aV5fba5gCtfWM62gjJaRYYw51eDuOOC9hiGbiOWn6aCIuKnwoNtvPjLPjwwojOGAXNW7Wfc31ZSXFZtdjQJAHVOF1mfbuWOt3Ioq65jQFoLPvnNuZoMK6dMBUXEjxmGwaQLO/D3W/oTFWJjTe5RrnxxOevzjpkdTfxYUVkV4/62ileW7gFgwnnpzJ0wmLhozTeRU6eCIhIALsqIZ8GkobRrFcEhRxWjZ67g/32+Uzsii8etyS3h8ueXs2pvCZEhNl4e15ffj+xKkFUfN3J69DdGJEB0iItkwaShjOyZiNPl5tnPdzDmlWxytUS+eIDb7eZv/9nD2L+upKismk7xkSycPJTLeiSaHU18lAqKSACxhwXx4i/78Nz1vYkKtfHN/mNc9vx/eHv1fq0+K2espKKGX/9jHY9+vBWny81VvZP4YNJQ2reONDua+DDD7YO/lUpLS7Hb7TgcDqKjo82OI+KTDh47zn3vrmflnvp1UjK7xJF1TU9aR4WYnEx8ycffHmLGwk0cqaghyGow4/Ku3Di4re7SkZM6nc9vFRSRAOZyufn78r089dl2apwuWkYE8/jonvyia7zZ0cTLFZdVM2PhJj7dVABA5/gonh7Tix7JdpOTiTdTQRGR07L1UCn3vrOebQVlAIwdkMJDl3clIsRmcjLxNm63m4Xr8/njR5s5VlmLzWJw14UdmHxhB4JtmjUgP00FRUROW1Wtk2cW7+DV/+zB7Ya2LcN55rre9Gvbwuxo4iWKSqv43YJNfL61EICuidE8NaYn3ZI0aiKnRgVFRM5Y9u4j3PfuevIdVVgMuGtYByZf1IHQIKvZ0cQkbrebf647yP99tJnSqjqCrAa/uagjdw5rr9uH5bSooIjIWXEcr+WPH25mwTcHAUiJDeMPI7syvGu8Jj8GmEOO40x/fyNfbS8GoGeynaeu7UXnBO1ALKdPBUVEPOKTjYf4v4+2UFBaBcC5HVrx8BVd6RivDyd/53a7mbcmj8c+3kpZdR3BNgv3ZnZiwnnp2DRqImdIBUVEPKaiuo6ZX+3mr8v2UON0YbUY3DIkjXsyO2IP0260/ihnXwmPfbKNnH1HAeiTGsNT1/akQ5yKqZwdFRQR8bh9Ryp49OOtLN5SP0GyZUQwD4zozJj+KVgtuuzjD/YUl/Pkou0s2lx/63BokIX7h3dm/NB0/TcWj1BBEZEms2xHMX/6aDO7i+uXyO/Rxs4fr+xKv7axJieTM1VcVs3zS3Yyd/V+nC43FgOu65/Cvb/oRLw2+BMPUkERkSZV63TxZvY+nlu8g7LqOgCu7tOGaZdm6APNh1TW1PG3/+zllaW7qahxAnBxRhy/vTSDTppnJE1ABUVEmsXh8mqeWrSdd3PycLshPNjKhPPaces5abSICDY7nvyIOqeL+TkHeHbxDorKqgHolWxn+mVdGNyupcnpxJ+poIhIs/r2wDH++OFm1u0/BtQXlV8OTOVX56WTaA8zN5w0cLvdLNlaxOOLtrGrqByA1NhwHrykMyN7JOoWcmlyKigi0uzcbjefbCzgpS93seVQKQBBVoNr+iRzxwXtaKedbU3jdLn5YlsRry7bw+rc+s0hW4QHcfdFHRk3OJUQmxbhk+ahgiIipnG73SzdUczMr3azam/9h6FhwKXdE/j1BR20mVwzKquq5d21B5i9Ipf9JZUAhNgs3HZuOnde0F63iUuzU0EREa+Qs+8oM7/axedbixrOndexFb8e1p4h7VrqkkIT2Xu4gtkrcpm/Nq9h8mt0qI1fDkzl1qFpuuwmplFBERGvsq2glFeW7uHDDfk4XfW/cnqnxHDnBe3J7BKnlUk9wO12s3zXYV7/Opcvtxfx3W/2DnGR3HpOGtf0bUN4sHanFnOpoIiIV8orqeSvy/bw7to8qutcQP2Cb1f0SmJUnzb0SrZrVOU0Ha9x8v43B3jj61x2npj4CnBh59aMH5rOeR1b6T0Vr6GCIiJerbismte/3ss7a/I4UlHTcD69VQRX9U5iVO82pLWKMDGhd6tzulidW8KiTQUsXJ+P43gtABHBVq7tl8wt56RpUrJ4JRUUEfEJtU4Xy3cd5oNvDvLZ5gKqal0Nj/VJjWFU7zZc3jORlpEhJqb0DtV1Tr7edZhFmwpYvKWQo5W1DY+lxIZxy5A0rhuQQnSoJr6K91JBERGfU15dx783F/DB+nyW7yzmxFQVrBaD8zu2YlSfNmR2iSciJHDmUVRU1/HV9mIWbS7gy21FlJ9YtRfqbxP+Rdd4Lu2eyPmdWmuvHPEJKigi4tOKyqr4aMMhFq4/yLcHHA3nbRaD7m3sDGoXy+D0lvRLa+F3IwbHKmtYsrWIRZsLWLajuGGuDkB8dAiXdEtgRPcEBqbFanKx+BwVFBHxG7uKylm4/iAfbshn35HKRo9ZDOiaFM2g9JYMSo9lYHosMeG+s8R+dZ2TrYfK+PbAMTbkOfj2wDF2FZfzv7+V27YM55LuCVzSLYFeyTFYNFIiPkwFRUT8Ul5JJav2lrB67xFW7S35QWEByEiIYlB6LAPSY2nfOpKU2HAiveCykNPlZldRORvyjrHhwDG+PeBgW0Eptc4f/gruHB/FJd0TuLRHAp3jo3QXjvgNFRQRCQgFjipWnSgrq/YcYXdxxUmfFxsRTEpsOKmx4aTGhpEaG05Ki3BSYsNJtId65FLJ8RonxWXVFJdX1f/zu6O8mt1FFWzKd1B5YtG072frmWynZ3IMvU78s3WUJgWLf1JBEZGAVFxWzeoTIyzr846xv6Sy0d0uJ2OzGLRpEUZsRDBBFgs2q4HNasFmMbBZDIKsJ85ZTpyzGlgMg5LKGorLqjl8ooiU/c8E1h8TEWylexs7vVJi6Jlsp1dyDMktwjRCIgFDBUVE5ITSqlrySirJKzlOXkkl+08ceSWVHDh6nBqn6+df5BSF2CzERYfQOjKE1lEnjshQkluE0TPZTrvWkbrbRgLa6Xx+m39hVkSkCUWHBtEtyU63pB9uUuhyuSksq2LfkUpKj9dS53JT63RR53RT53JR63Tj/O6cy02ds/6cy+2mRXjwf0vIiSMqxKbREBEPUUERkYBlsRgk2sO0eZ6IF9JN9CIiIuJ1VFBERETE66igiIiIiNdRQRERERGvo4IiIiIiXkcFRURERLyOqQXlpZdeIi0tjdDQUAYNGsTq1avNjCMiIiJewrSC8s477zB16lQefvhh1q1bR69evRgxYgRFRUVmRRIREREvYVpBeeaZZ5gwYQLjx4+na9euzJo1i/DwcF577TWzIomIiIiXMKWg1NTUkJOTQ2Zm5n+DWCxkZmaSnZ39g+dXV1dTWlra6BARERH/ZUpBOXz4ME6nk/j4+Ebn4+PjKSgo+MHzs7KysNvtDUdKSkpzRRURERET+MRdPNOnT8fhcDQceXl5ZkcSERGRJmTKZoGtWrXCarVSWFjY6HxhYSEJCQk/eH5ISAghISHNFU9ERERMZkpBCQ4Opl+/fixZsoRRo0YB4HK5WLJkCZMnT/7Zr3e73QCaiyIiIuJDvvvc/u5z/KeYUlAApk6dyi233EL//v0ZOHAgzz33HBUVFYwfP/5nv7asrAxAc1FERER8UFlZGXa7/SefY1pBuf766ykuLmbGjBkUFBTQu3dvFi1a9IOJsyeTlJREXl4eUVFRGIbRDGm9X2lpKSkpKeTl5REdHW12HL+n97v56T1vXnq/m18gvOdut5uysjKSkpJ+9rmG+1TGWcTrlZaWYrfbcTgcfvsX25vo/W5+es+bl97v5qf3vDGfuItHREREAosKioiIiHgdFRQ/ERISwsMPP6zbsZuJ3u/mp/e8een9bn56zxvTHBQRERHxOhpBEREREa+jgiIiIiJeRwVFREREvI4KioiIiHgdFRQ/Vl1dTe/evTEMg/Xr15sdx2/l5uZy++23k56eTlhYGO3bt+fhhx+mpqbG7Gh+46WXXiItLY3Q0FAGDRrE6tWrzY7kt7KyshgwYABRUVHExcUxatQotm/fbnasgPH4449jGAZTpkwxO4rpVFD82IMPPnhKywnL2dm2bRsul4tXXnmFzZs38+yzzzJr1ix+97vfmR3NL7zzzjtMnTqVhx9+mHXr1tGrVy9GjBhBUVGR2dH80tKlS5k0aRIrV65k8eLF1NbWMnz4cCoqKsyO5vfWrFnDK6+8Qs+ePc2O4h3c4pc++eQTd0ZGhnvz5s1uwP3NN9+YHSmgPPnkk+709HSzY/iFgQMHuidNmtTwZ6fT6U5KSnJnZWWZmCpwFBUVuQH30qVLzY7i18rKytwdO3Z0L1682H3BBRe477nnHrMjmU4jKH6osLCQCRMm8NZbbxEeHm52nIDkcDiIjY01O4bPq6mpIScnh8zMzIZzFouFzMxMsrOzTUwWOBwOB4D+PjexSZMmMXLkyEZ/1wOdabsZS9Nwu93ceuut3HnnnfTv35/c3FyzIwWcXbt28cILL/D000+bHcXnHT58GKfT+YNdzuPj49m2bZtJqQKHy+ViypQpDB06lO7du5sdx2/NmzePdevWsWbNGrOjeBWNoPiIadOmYRjGTx7btm3jhRdeoKysjOnTp5sd2eed6nv+vw4ePMgll1zCmDFjmDBhgknJRTxj0qRJbNq0iXnz5pkdxW/l5eVxzz33MGfOHEJDQ82O41W01L2PKC4u5siRIz/5nHbt2nHdddfx0UcfYRhGw3mn04nVamXcuHHMnj27qaP6jVN9z4ODgwHIz89n2LBhDB48mDfeeAOLRf3/bNXU1BAeHs57773HqFGjGs7fcsstHDt2jIULF5oXzs9NnjyZhQsXsmzZMtLT082O47c++OADrr76aqxWa8M5p9OJYRhYLBaqq6sbPRZIVFD8zP79+yktLW34c35+PiNGjOC9995j0KBBJCcnm5jOfx08eJALL7yQfv368Y9//CNgf6E0hUGDBjFw4EBeeOEFoP6yQ2pqKpMnT2batGkmp/M/brebu+++mwULFvDVV1/RsWNHsyP5tbKyMvbt29fo3Pjx48nIyOC3v/1tQF9a0xwUP5Oamtroz5GRkQC0b99e5aSJHDx4kGHDhtG2bVuefvppiouLGx5LSEgwMZl/mDp1Krfccgv9+/dn4MCBPPfcc1RUVDB+/Hizo/mlSZMmMXfuXBYuXEhUVBQFBQUA2O12wsLCTE7nf6Kion5QQiIiImjZsmVAlxNQQRE5a4sXL2bXrl3s2rXrByVQA5Rn7/rrr6e4uJgZM2ZQUFBA7969WbRo0Q8mzopnzJw5E4Bhw4Y1Ov/6669z6623Nn8gCVi6xCMiIiJeR7P4RERExOuooIiIiIjXUUERERERr6OCIiIiIl5HBUVERES8jgqKiIiIeB0VFBEREfE6KigiIiLidVRQRERExOuooIiIiIjXUUERERERr6OCIiIiIl7n/wOmIpCi+M1VdAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "xs = np.arange(-5, 5, 0.25)\n", + "ys = f(xs)\n", + "plt.plot(xs, ys)" + ] + }, + { + "cell_type": "markdown", + "id": "5c9cb820-159e-420d-b0d0-a8706e19f9d5", + "metadata": {}, + "source": [ + "Now, what's a derivate?\n", + "> It is sensitivity of the function to the change of the output with respect to the input.\n", + "\n", + "In simpler terms, (f(x+h) - f(x))/h, where h tends to zero. This gives us the slope." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7866b622-e10c-4003-844f-9044e7bdc080", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3.0000002482211127e-05" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "h = 0.00001\n", + "x = 2/3\n", + "(f(x+h) - f(x))/h" + ] + }, + { + "cell_type": "markdown", + "id": "4f027e6d-d889-4827-867e-335a96c20045", + "metadata": {}, + "source": [ + "Let's define a function with mupltiple inputs" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a3bcd24a-4a48-4d85-8d49-3fb751589b40", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4.0\n" + ] + } + ], + "source": [ + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d = a*b + c\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "78dcb378-190f-4421-8d95-af79cdf05b18", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "d1 4.0\n", + "d2 3.999699999999999\n", + "slope -3.000000000010772\n" + ] + } + ], + "source": [ + "h = 0.0001\n", + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d1 = a*b + c\n", + "a += h\n", + "d2 = a*b + c\n", + "\n", + "print('d1', d1)\n", + "print('d2', d2)\n", + "print('slope', (d2 - d1)/h) # By the good old derivation (wrt to a), we know that this will be b" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9fb1208f-1a1e-4fdb-a290-a7d0e8a5050a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "d1 4.0\n", + "d2 4.0002\n", + "slope 2.0000000000042206\n" + ] + } + ], + "source": [ + "h = 0.0001\n", + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d1 = a*b + c\n", + "b += h\n", + "d2 = a*b + c\n", + "\n", + "print('d1', d1)\n", + "print('d2', d2)\n", + "print('slope', (d2 - d1)/h) # By the good old derivation (wrt to b), we know that this will be a" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "e7bfb7ef-4c16-44c6-afce-ac20ab858b4f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "d1 4.0\n", + "d2 4.0001\n", + "slope 0.9999999999976694\n" + ] + } + ], + "source": [ + "h = 0.0001\n", + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d1 = a*b + c\n", + "c += h\n", + "d2 = a*b + c\n", + "\n", + "print('d1', d1)\n", + "print('d2', d2)\n", + "print('slope', (d2 - d1)/h) # By the good old derivation (wrt to c), we know that this will be 1" + ] + }, + { + "cell_type": "markdown", + "id": "ad332637-3dbe-4630-bb5a-d5a76e935a9b", + "metadata": {}, + "source": [ + "The NN will be mathematically very large expressions. We now start by building the data structures for this. Let's start by making the `Value` object from mircograd" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "id": "e11f7575-00be-44a4-9bde-ae9288b1922a", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=4.0)" + ] + }, + "execution_count": 131, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "class Value:\n", + "\n", + " def __init__(self, data, _children=(), _op='', label=''): # Define a empty tuple `children` to keep the pointers of other Value objects\n", + " # Define a empty set `op` to keep track of what created that Value object\n", + " self.data = data\n", + " self.grad = 0.0 # this is the derivative of Value wrt to its nodes; initalized as 0 (we assume at the beginning that every Value doesn't impact the output)\n", + " self._backward = lambda: None # for a leaf node, this should be nothing.\n", + " self._prev = set(_children) # This will be a empty set when we define a new Value object (a, b, c)\n", + " self._op = _op\n", + " self.label = label\n", + "\n", + " def __repr__(self):\n", + " return f\"Value(data={self.data})\"\n", + "\n", + " def __add__(self, other): # a.__add__(b)\n", + " out = Value(self.data + other.data, (self, other), '+') # Since self.data and other.data is python floating point number,\n", + " # the addition is according to whatever is defined in the python kernel\n", + " def _backward():\n", + " self.grad += 1.0 * out.grad\n", + " other.grad += 1.0 * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + " \n", + " def __mul__(self, other): # a.__mul__(b) # You can't name it mult or multi, because we are defining magic methods for the Value object\n", + " out = Value(self.data * other.data, (self, other), '*')\n", + "\n", + " def _backward():\n", + " self.grad += other.data * out.grad\n", + " other.grad += self.data * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + "\n", + " def tanh(self):\n", + " x = self.data\n", + " t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n", + " out = Value(t, (self, ), 'tanh')\n", + "\n", + " def _backward():\n", + " self.grad += (1 - t**2) * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + "\n", + " def backward(self):\n", + " \n", + " topo = []\n", + " visited = set()\n", + " def build_topo(v):\n", + " if v not in visited:\n", + " visited.add(v)\n", + " for child in v._prev:\n", + " build_topo(child)\n", + " topo.append(v)\n", + " build_topo(self)\n", + " \n", + " self.grad = 1.0\n", + " \n", + " for node in reversed(topo):\n", + " node._backward()\n", + "\n", + "a = Value(2.0, label='a')\n", + "b = Value(-3.0, label='b')\n", + "c = Value(10.0, label='c')\n", + "e = a*b; e.label = 'e'\n", + "d = e + c; d.label = 'd' #(a.__mul__(b)).__add__(c); you can also call this manually.\n", + "f = Value(-2.0, label='f')\n", + "L = d * f; L.label = 'L'\n", + "d" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "aa1c0bd9-5c01-4ff6-b397-bf8b37ca0a70", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{Value(data=-6.0), Value(data=10.0)}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d._prev # This will gives us the Value objects that made d (the children of d, ik sounds wrong but it is what it is)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "7d88f630-07a0-4e12-a0e6-177094ed4999", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "set()" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c._prev # Has no children" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a24046f3-53a4-4bea-ae0b-756bb9a80dd3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'+'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d._op" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "cdf24235-ee48-4675-bb65-bd78d1ae201e", + "metadata": {}, + "outputs": [], + "source": [ + "# imported from mircograd's codebase\n", + "\n", + "from graphviz import Digraph\n", + "\n", + "def trace(root):\n", + " # builds a set of all nodes and edges in a graph\n", + " nodes, edges = set(), set()\n", + " def build(v):\n", + " if v not in nodes:\n", + " nodes.add(v)\n", + " for child in v._prev:\n", + " edges.add((child, v))\n", + " build(child)\n", + " build(root)\n", + " return nodes, edges\n", + "\n", + "def draw_dot(root):\n", + " dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n", + " \n", + " nodes, edges = trace(root)\n", + " for n in nodes:\n", + " uid = str(id(n))\n", + " # for any value in the graph, create a rectangular ('record') node for it\n", + " dot.node(name = uid, label = \"{%s | data %.4f | grad %.4f}\" % (n.label, n.data, n.grad), shape='record')\n", + " if n._op:\n", + " # if this value is a result of some operation, create an op node for it\n", + " dot.node(name = uid + n._op, label = n._op)\n", + " # and connect this node to it\n", + " dot.edge(uid + n._op, uid)\n", + "\n", + " for n1, n2 in edges:\n", + " # connect n1 to the op node of n2\n", + " dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n", + "\n", + " return dot" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "68a93a67-c9f0-462b-b61d-3442e3e8523a", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653933648\n", + "\n", + "e\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "2377653934672+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2377653933648->2377653934672+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653933648*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2377653933648*->2377653933648\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653934672\n", + "\n", + "d\n", + "\n", + "data 4.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "2377653934672+->2377653934672\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653955280\n", + "\n", + "a\n", + "\n", + "data 2.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "2377653955280->2377653933648*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410649872\n", + "\n", + "b\n", + "\n", + "data -3.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "2379410649872->2377653933648*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379407120272\n", + "\n", + "c\n", + "\n", + "data 10.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "2379407120272->2377653934672+\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "draw_dot(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "c580a0dd-f363-4fd8-b221-4c80f78cc1c1", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653933648\n", + "\n", + "e\n", + "\n", + "data -6.0000\n", + "\n", + "grad -2.0000\n", + "\n", + "\n", + "\n", + "2377653934672+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2377653933648->2377653934672+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653933648*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2377653933648*->2377653933648\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653997136\n", + "\n", + "L\n", + "\n", + "data -8.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377653997136*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2377653997136*->2377653997136\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653934672\n", + "\n", + "d\n", + "\n", + "data 4.0000\n", + "\n", + "grad -2.0000\n", + "\n", + "\n", + "\n", + "2377653934672->2377653997136*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653934672+->2377653934672\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653955280\n", + "\n", + "a\n", + "\n", + "data 2.0000\n", + "\n", + "grad 6.0000\n", + "\n", + "\n", + "\n", + "2377653955280->2377653933648*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653995280\n", + "\n", + "f\n", + "\n", + "data -2.0000\n", + "\n", + "grad 4.0000\n", + "\n", + "\n", + "\n", + "2377653995280->2377653997136*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410649872\n", + "\n", + "b\n", + "\n", + "data -3.0000\n", + "\n", + "grad -4.0000\n", + "\n", + "\n", + "\n", + "2379410649872->2377653933648*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379407120272\n", + "\n", + "c\n", + "\n", + "data 10.0000\n", + "\n", + "grad -2.0000\n", + "\n", + "\n", + "\n", + "2379407120272->2377653934672+\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "draw_dot(L)" + ] + }, + { + "cell_type": "markdown", + "id": "9886d9da-645c-489e-af3a-60d48facf40b", + "metadata": {}, + "source": [ + "The value of the forward pass is 8. We would now like to run backpropagation. We will start at the end and reverse and calculate the gradient along all these intermediate values. For every single value here, we are going to calculate the derivative of value (L) wrt every node. \n", + "\n", + "In a NN, we will be very interested in this derivative of the loss function wrt the weights of the NN. There will be data and weights, but the data is fixed, so our focus will be the only the derivatives wrt the weights." + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "f1dd3efe-8498-4c25-9fb6-2f7e71447f6b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "a.grad 6.000000000000227\n", + "b.grad -3.9999999999995595\n", + "c.grad -1.9999999999988916\n", + "e.grad -2.000000000000668\n", + "f.grad 3.9999999999995595\n", + "d.grad -2.000000000000668\n", + "L.grad 1.000000000000334\n" + ] + } + ], + "source": [ + "def local_staging(): # just for our local testing\n", + " h = 0.001\n", + " \n", + " a = Value(2.0, label='a')\n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0, label='f')\n", + " L = d * f; L.label = 'L'\n", + " L1 = L.data\n", + " \n", + " a = Value(2.0 + h, label='a') # changing a by h\n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0, label='f')\n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data\n", + "\n", + " print('a.grad',(L2 - L1)/h)\n", + "\n", + " a = Value(2.0, label='a') \n", + " b = Value(-3.0 + h, label='b') # changing b by h\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0, label='f')\n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data\n", + "\n", + " print('b.grad',(L2 - L1)/h) \n", + "\n", + " a = Value(2.0, label='a') \n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0 + h, label='c') # changing c by h\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0, label='f')\n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data\n", + "\n", + " print('c.grad',(L2 - L1)/h) \n", + "\n", + " a = Value(2.0, label='a') \n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " e.data += h # changing e by h\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0, label='f')\n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data\n", + "\n", + " print('e.grad',(L2 - L1)/h) \n", + " \n", + " a = Value(2.0, label='a')\n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0 + h, label='f') # changing f by h\n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data\n", + "\n", + " print('f.grad',(L2 - L1)/h) # should be d\n", + "\n", + " a = Value(2.0, label='a')\n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " d.data += h # changing d by h\n", + " f = Value(-2.0, label='f') \n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data \n", + "\n", + " print('d.grad',(L2 - L1)/h) # should be f\n", + "\n", + " a = Value(2.0, label='a')\n", + " b = Value(-3.0, label='b')\n", + " c = Value(10.0, label='c')\n", + " e = a*b; e.label = 'e'\n", + " d = e + c; d.label = 'd'\n", + " f = Value(-2.0, label='f')\n", + " L = d * f; L.label = 'L'\n", + " L2 = L.data + h # changing L by h\n", + "\n", + " print('L.grad',(L2 - L1)/h) # should be 1\n", + "\n", + "local_staging()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "78e46a2f-7ba5-458f-beb3-133ebca6a839", + "metadata": {}, + "outputs": [], + "source": [ + "L.grad = 1.0" + ] + }, + { + "cell_type": "markdown", + "id": "90580c47-bf74-4599-81c9-8bccb8f3c69d", + "metadata": {}, + "source": [ + "L = d * f\n", + "\n", + "dL/dd = ?\n", + "By calculus, we know that this will be f" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "bbd2d2c2-f136-4244-a901-ff7c0165efe2", + "metadata": {}, + "outputs": [], + "source": [ + "f.grad = 4.0\n", + "d.grad = -2.0" + ] + }, + { + "attachments": { + "8211c8be-62e8-4102-aaca-a5bd17838677.png": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAngAAACwCAYAAACcsxPKAAAgAElEQVR4Xu2dC5geVXnHX1TUglq8S4jiEtFErVUCgkETrKhNBLQKiQkISVtYogWtl2ajIMGI2bTeKirZoAiUbCRegTR4oQobgaAEFS+bimGFLomoxViFlmK1856ZM3Ouc87MN99+s5v/PI+PZL+Zc/md23/e9z1n9vljchEuEAABEAABEAABEACBKUNgHwi8KdOWqAgIgAAIgAAIgAAICAIQeOgIIAACIAACIAACIDDFCEDgTbEGRXVAAARAAARAAARAAAIPfQAEQAAEQAAEQAAEphgBCLwp1qCoDgiAAAiAAAiAAAhA4KEPgAAIgAAIgAAIgMAUIwCBN8UaFNUBARAAARAAARAAAQg89AEQAAEQAAEQAAEQmGIEIPCmWIOiOiAAAiAAAiAAAiAAgYc+AAIgAAIgAAIgAAJTjECpwNu2dgYtHiqv8dzBG+iyhffQ4CFLiIZ30sBRE0lonK5cOo+2LOAyTO8g41t6VH4uclN16KD6E/3otrXUl3SXjXeuoG51F9F3R9fQ1ksXUic9Y6LRTNr8xjfRaXNX0izvHNDEGLuPrjp/jFZPeyrdenoHrXrbKB2ezGtDQ7NodjL+PtZ/L91xfB9deNwTOsCflu262THpNFSPDkqbP7p7J521+VF0YSWeCbPzH6QTz5tB0zghjWcThYpPQ4xzGqaxFUfmD41vWkYvGxgR/+6f8DUpvuzpndXHRXhua8uakpZjYJbePlUJTej9Ym3aQYMjn6FFHUwxE1rmkswqWPDKGqt6J20GQFs6cie1mQp1qFh/CLyKwCbB7UGB10QduiGMmhJ4VerXjXpUyV/eW68c2y/eTv27DqCrpcCrk3VDz1gCb0L6YUOFr5kMBF5NcDGPQeC51DgEXkzfcd8DgVefnf/J8CTYjVz34jQnZGGtJ0jKW6UXAo9IiCTq0BLZcXerxxMCr2PwHSUQntvasqbAgtdRQzfwcMMWvGGiJUso9+r222JQd/vOjTKFqiZ3rnPqFmb7qezIw7RgyxLKrPLJDbZrznI3a/foAlXkt2U+bVxwLS0uEg2UldNYR32DM2lgYD2XMrl/NdE5tgtZH6CuwZiWJ+foqI/W9mJxvZYWDM+nLUtWUuqcyDjNuUm4ztK/OXiLNxYur7yKe1LuMw1XasSgzRZ7WQ5OWXOVmBY86/4zLPetvw+4R4FkPDhrJQ1IkC6OZt6OPmvnUNY+2W9aXhmzkbRe0539y6gzM1rXR3n5lfRCY8js68V4SWsS+l26jUr7n8Gtf3ANjQ5UcNHKPpBMGYuV/lfuUosXJEKE3Cpbbl86d9UL6LUHZv92umgPoEOv2UOXZ7e8RHPZSvfrU+nY7ffS6l3ZTdNUK5bqoqXUlSzvUztQ9gxtvp3WJMED0i2sl5dIz9/sgTHlSZ8x0z21fzadfRj/kgpbWV+i/TKXdfKT4POAkqnkl/HP65X9fbfq8k4f25XU74RrHsrTqM7TPa7Nv2oWPHMuK5s3S+Y9K2ffy4ucd6U7L2bes8b0wXSFGd4UKFt4bnMLvNC8YdbbnHPl7/4xKue57M7+NTQ4msy/hou2rBzutddeD4JzVOz8orFO1j6xfqsuWqNO5lrm6abmHOtc/yrNfXHjwbyrUYE3pAqIDJxasbTSSmM57nEOYPWZLGZhVIg8Sn38iZLI88kGGeUiMFvQtHisbBHOF3OHwEsTzWI7skZOpmR/TJdc9NXOWDLQ8vKY95hlkwuyq5NntPKJpbin6GDm30z+eryB1kauiS1kqXH8nk4UVAhkVeA57jffUNMyKeLU0cbufiNGo78NrXQqtLMiBK1+bdTJrH8+cVqizR4bRfnTGobGkPV2b5Ql9Hs+cUbUrxhjheD1T/6GlV9OrGUMrDktTuClosYWLLm4cQg8Fjr57xyXtmoPUS7yCmETuscbg5eJpkJgFZWzLGJZ/ofmYswEEV+e1btKOJCDpyjnQ5ogNnla5TVi8NL7FVFdi2fcglbLRetww1njSsve/VIrxYhYE6LnveRlWnuJdI2Lkjk5nwPK5jZ73QnNGzbtpFxriQby2MY0zbEzfXH29tyZr0Flc4mhAaLmxkwDDEWkqxp7ggzydVSuNY52D7pw9Zd5EWNu6pxac1/ceOiqwEtFl4xMNOB4hEG5uTnk+nUPPD1Ndxpl9zgtV8GGTfNxMTA3gZRZ8NxWM1faSlO6BI9LPBt1sCbHvDPKCcbm6y5fURZt0pN/NtteFXghrp5+EyqHc8IOiR0ub0DAxrZPft9IH12UWFDVFw5L8ApORhu7uATH0By6sTSoOWx9jamfc8wGX9bCC1mIPbkEiTmjecSRJkpcAu9w3V2aWqD2zaxabmGpC52yTRZlbuA40apXM6I8DqHGabjKrG5acbqOjbRKBZ6Hf3WecQtaHYEXnvfsvO1xoYuo+HnPDODXx0VM2cJzmyHwgvNGeCMa1+89tNq/mdE5jxvGiohykGkMcMyNMXNUKqoM1lr+JaI9N0YE1l1XF/WsZ9qcGSxbXN+PuatRC56+i1YH6F2QywLuQ4u/ZweqXzT6XGtuF61mrQuWxSUkq1rwMoukY9eRc+D7BFQuVBK3rbobqKQOPrO53m5hgaB2OtPEn1t3tDYv2sR0JYq0fGUOCDE3L7U90p3fuiAX4Ep2Zvvrb+en9DXDVeRcDERVlR2BjnERM4aE+1eEFbgtvkWbuH6Pqd9Bbj4hy665W9A17oNphMWQLiSU3qiKOofAI9NaJoTK/XSscO26xVucwMvKTP5NCYU7U7G2lc7e4fKwC7gQqCYHaaEr5+lzc5cKPI+wJE34hcsvdudGXHUEnpqsb96zsna+pLp3W8bNezIHvxHDV7bw3JauI9KwEDNvlJ5mwGM1cS2XnUjgntOqawAZvmLmVdQ5nX9cO3ND86f+Aunh7jKCcKxKKEwqa85y1ll/GXecIhGc+yIGg+OWCRZ4alSWWhqP+7FM/InHYwSUEXOUNdS4doxGywSeD5MvPszrSg0IPGkqFiwzs7TofOrEpbCZHjoKg5skvUdWQYg68zmrXUviHHx9oCGB5zsFyCk2s/6Wh2WaA8poHznRm27LzgRexBjS2tUxOXl/t9tBq6KoX2KMdArjkLXdZcEzjsoJTnKxAq+I/dKbKBNQEyjwvILT7Dtm3JsW32feHBZIQuBt39/e6aoJMJ+LVsbfqTF2hds2LPDkETRKudsk8KLmPZO5LlZ8oQ7V5j3OwxO6UDIn1xN4EfOGS0OIMTlGywNHWvmMKmpZfTF9abZ6fHJQ4IXWyNALpG8tE/XV102r3CVx2r65XTNUTF2BZwbsB1Rp0GoWIfAizMLTjUHmbKRgWbprwSslVUvgeaw1jnrKQbq1b51j04VeMudAL3PRWhWTIsMnOPNXJWsgqkm5J0HV5O6z4JWRrmDB1ISu/gIT87abTgi6AAq5pZ0ll4uZb1LSfve/HRdpeyycQXE2kQJPulY9bRkj8BxCyIyvC1rwSuLuSsezFHuG27h4JlLg5S5mJTet7qbA8wjoKi7aUgteFYtoYG3Ifq5uwYuf9yyJl286O4auN15y6s976riIK1t4bou04AURh+LulFkh2zimC7NIC55SjvDcGDNHJQmGBN5RcRY8dx8YUTZ66ndEWUuDZQs2TPQNE2bB87navIqXqxBcNCIEntMClLnPcrNrdy14ujnZDEbV6+B+EypzG3o4Od5E9DYoiU1UNzRwO2QM+/vXJzt7yw6tdJfTsmQFLbNhq2FI7IQnXJ9YK7dExbWP2sbpLjk1ILgsBi8Pc3Ax8rxklI4h0T3SXeE+F4v6u27ZlvNIRB/NhGK1TRbNW/DSHaD6JgGuhbCkSYtWxEHHrpixagKvs+NXtPJa03lY4E2rFYOXltl0V5ubJpqKwSvnGbeGVRd4FeY9e3VPD/PuPyOJzbR3Wpqx1nHznlqeuLLFzm15eWrOG5zPRX2RHxFw5mHE4EWUoywGT86NUXNwUETFxOC5+2BpuJSnjnYMXtW5L248mHdNnMATOsHYRRsUcK4dpGrDuOOBNJiODQhFbIO0rHRL4Nnlz829ubg0RZExKOQCbR1Xor322F8RCAo8U2hKIZcemaIv1EU8WehkeF8bs0U9d3uqg88xIEzxlqY5EbtoXTuuzSET0T5mnQzxY/cBR74eEVw+hlyTlmq1dMV4moHEEfWzxlRM/5gYCx63lrWL1gz8r7mLNl7gZceklMTdFb3KZTULicMIgSc3pFTaReuIF1Rcx3IHcFO7aHsj8KrMe5bCy09tMGOyKs97edIOC556WoPiTpZzb75+eU8IsF+0K6+9nK8Rdze+aS3dOGeF5wsPNle7nGENEDU3OnbRWi/8QYFXrHfmCRyJfS478cGcG+ONTgPZkVj+XbRTUOClxiD982chwcDP+H3gERa83AKlnPWWDIzU3SiP7uiewJNxgnnclsw7t6i46lAsmOk8UHJESgqohsDjBx35OHZ9Fu0WKIcoqxnHxYPlTBqbq1ixzMFnxoQ5zuvzBi6bc3D2b/mWtZGWFJ/bc7kpjZjBuGDakvaR6Rl5qSL16JvS8wUHB3ckZy5lwSRmEG+JlbN8DDni6LSyhH739AszBseMtax7Dp6abvCFzzyHTW989QgS//lvyTNOF+1Tk8Cx4lw4/TiTeEElRMu0e41z5NRyujZSOOrldc9yWjHlSfMsPQ+Qb8gFnDzWxHE+3qpH0qXqsTGZYL45eVxwovA5eNV5hi3P+bykehWCfajavGdOL3IesmN0a8x7InHTaheek8NzW8m6qAQde9deaz7OKATPCDUYRJ2DpxsTpFArnRsVbkV1HOeImp/DdPUNra6uc/DM9lDP4fUsPsmf9TnaOH82Rnz6k670SwULXqV0cfOUIlAh9mxK1bs7lQm5l7uTK1IFARAAgXYTwNzYbPtA4DXLc2qm5nL3Ts2aTkitMIlNCGZkAgIgMMkIYG5stsEg8JrlOcVSi4mtmmJVnoDqYBKbAMjIAgRAYNIRwNzYbJNB4DXLE6mBAAiAAAiAAAiAQM8JQOD1vAlQABAAARAAARAAARBolgAEXrM8kRoIgAAIgAAIgAAI9JwABF7PmwAFAAEQAAEQAAEQAIFmCUDgNcsTqYEACIAACIAACIBAzwlA4PW8CVAAEAABEAABEAABEGiWAAReszyRGgiAAAiAAAiAAAj0nAAEXs+bAAUAARAAARAAARAAgWYJQOA1yxOpgQAIgAAIgAAIgEDPCUDg9bwJUAAQAAEQAAEQAAEQaJYABF6zPJEaCIAACIAACIAACPScQITAS75HuvQuOuXShTSdi7ttLfUtIdp45wo6qufFjynAOF25dB5tWXADXbZQ1CB4xXwPT9yzZT5tlVyCqbbhhhgWMfcodanVH9Jv3NLwThrIOtG2tcvorpM/Q4vimigCpp1H2q4j4tl+Je/SxET9dtDgSJNliyh+7VvS9huYNUxjK46snUq3HtTb2W6jbuXbVLqVx/34Jjpt7kqaVdbfao2hpmqUpCPKOEbLszmd63jFMz6Tjc2K80FZsTifDQfTZd5+Ofn6Q7VWaPfYrFaX7O7dO+msVXvo0P7ZdPZhtVKY8Ie2X7yd+ncdQFefN4OmNZk7s9j8KLrw9MYWsY5LFxR429bOoMWjawoh0+vJqOMqhxPYuwVemI92RxP9oSsiylgsYhZaV9W7UraKjCvd3uJFxGI51Rd0KZ7aLPDS/jJ2ZvayZY2TpgRei/tlpfHVyc1TkAEEXtYh7qOrzh+j1dOeSrdC4HUySLr/LAReBcYQeBVgTcStLV5EIPDcHaCJMVSza1kWSQi8miRjHmvx2IwpvuseCLzJKvCyzph6tJJrbuqmGs9ctMNEi5esz5vccnllE0X+eH+Zy8jX8c23R7NMSfZqujxRruujwVkraWCIi8yWxzl0o+miNcsmkinchVLgDQ7uoIHMpUd0huaWdrlqhLWT81V5lVprUwvGkMYm/dvoYIRL2WVdcvxNlIuY/0GZu3qYFmxZQnnVBKfMBU+uN/asnHlrKyzk4hTqD9rkUFhuTrm7cJuKXibrLdIt+lfalrKMaWLlvBXrEIXTyoun5Zv0+cGZSR/QXbSqq5ef0/u+zDcBsiRpW9kbHO0ZV349Ha2/i7SNMdG/hgZHk/6vuGj1fBTGJZO6Wce8XZQ8tywo60d64u707slc9aE6Gv3P0RfU3CyvQ/ZjMQ5S17XJRe9jnOc66hPtz/0wnf+OvskMzQjMSVIwDa6h0YGVJOdDrc84BF55H3M3nPlimqfh5WWMdXPMiXkpZs5wjUc5//KYLZk/PPNCGrphs40OrcjTrTfHlvcNyS3U/8Nj02pJx9ok7ilZP2PKOrag6H85w0prdFbSTNDdnP3z1OMPoDuuMV204/Sx/nvpclm5abo7dNfm2+mE7fvT0Oz7qf+ah7K79qOhoVk0WwVi5EWH69Yxdzr70rmrXkCvPbBISLhkb83+nZTl3Gl7aLXhohVp5WUhOlVzOafWuetmP5WO3X5v8myRVurmNepLjrqUzLXd/KmmizaZ8JRJI+1gyqKfdRzKF7Wso5O9SMvKOa1mIp1raYGIf3KIQDMfOUFpg8GYxByuujRvyuOsXBOjWUdT4FkMsrJ4JyRtMlXFY5W3PFcsWyoyi3zV+lMan5WsMuYgN9sqj1m02pJDdpjXzFTwynqU9QerBxvlNkWpJVJtJmHeNVy0ZpvlE2D2cpOIdbOvpPFLK5POIwV5sZj5GUtxYQrl9Uq7yXSKvNP4V/Uee1zlk302BiyxE+GqTtNQ8vWN59J+5Ji2PBa8IfnyyC9DVh3tBdpqezMrp1td7w+2CDTzkfzLXuwi5iRHH7LqaAi8cB8zK6wLCe53/OL0HlpdHnesza9Zmh4LXvmckfVnNZRHirp8Lo6Z19Q2ctxfNVyi5hwb7hsF79A8OqCseebYdC3s29auJVpRxLeLvjB2pjeeNr6syngW2tmct8JrNGWCi47vowuPe4IQ4FLIFYIo+5sixlKBVYieXEwpws+8x84rc4FSIRbzdPK87Hti8k7ToUIYWvXM0k2EXV5P3z1Tw0VrBJ0bk4LzDTq0sPiEl9zI4JqM5BuetFY4JwBd4DmDpI28rQlWjETdsqal46mbz5KgDmz3YnULXbnpIFoU3BRiToKp1WF0brLyyk0lIZEs1lQ1zjKCl14BexNCqK0zlvkmC6Pdgi7yKN5VBZ57AdL7gtu6qpfXZTEwRHGF8uuWXKOMJUImtQqn1pdKGy48ZdPr6GYV7O8egVdWR3dfCFm5Hb9rIsod+6eXv6Stq8xJjhckHj5aXo6ymRb84JhQ5ijhvIjwAPhfqtWYwZi2juFZVeCF2jje7lFtjq1fF7tNzc1Z7rnBWxPuO+cQXeDdyNdQWQsrS+mGIOfmhNtG6fChB3Lhk4qlfQ1rXCr67siEoSWoRP76Pc68DHewMy9RnodSsUauDSCmCNTzlSj0tN3xdXoZp3IMnrYo+Aamy/Wndm2XWPG4Kn2mZWcsiz9fnxvEt1NOdfGo95Bq0bLET/mOY5eYHN+0icYXLozapWwJzWRCWL7gWlqcLUKibPluXzcLv8DLLH5lOzJdzDsUePLtkt1ZLguod7ErW8Srlkm2oypKRHiCY0etJqLdk25ap9QaLdx80gLq7S+udPQx4u6njnuEL163RHkXFZ+VROMX048cOXgEnrqbOnfLiT7nF6imu9U24tkvLW6h63P/utvRNzeofVaURVqunC+mQuEVpxFY/x3qY35xEycE0+fjXsSrtrWPZ1WBp7jQAy75kNSrP8eW18U8mUHlqc+7uYKq8MKVvqz3Re/er1LWOmu0dFNK611WJ010+UWOEEOUulila9XcxVrcQ5ogLNpXL4MzHVXg7WaxR5brVxNm6v2KWze1IN5Pxwp3r7vue7XAy0PRjNFX9mZpu/7Uic6MZ3AsADECzxCHQkBM148yqCfw8mhDo8aBhdVc9JLyDd40hwaC1rssG0M4CLfMnJvyYw8osc5d1CfdhzGTte3SLT1iphsCT1TNmLAUN54pzHXgkndFC55PAKoLtBB4SlyglrF0gcQKvFB/CQu82Fgz6RLMi1u2YDrHUPJkLwWeD1VZXK86rsg+qkSLW8p4jGuW7BiBFzEn1RJ4oT5WIml8+Tn1tnFCAt/jcdGWiZnUs67EHzt5Vhd4aXH0GN2yWDQvlYpzbGxdypjofakoWejFJL0zZZXvbC5p7nplNedWPQP3Gu22dEnLG4mYtcKV6Sxy5kqNFXh5DJ+R2EtUS2ASy6cJRUWwHbHdZU0kXWBmFkg3YhnPB4GnmHY7MK0rk8sxN7iOZzHfbF0uK9NipouamLfWaMuIaiVzWWRCr5aOCXVbYr2bnljv4k/SKRYiZnb9PN4sIv+WBq8XFpJ6Aq/Uxdc1gafCkxNSKt6m+yymGu+KAs90G8u0Yix4Zfk60vFa8ILpVLfgWV1QxiX5xFGpBU+Phw0t+u681TEcErE1XMx5psU8dAGdq1tMK7jIdetiJjikRbwkJCQfMx7BZb/MZvNW1TgzA/K2ZGxcxBbbCBdtzFwoxUZpW0fxrCfw1OpJsRfjftawGOUrnWMr1KVbFrxQ3F1et9plrbNGd2bBU9vDLfBU65/Pgqd39q5a8LSsIPAUgecbyB7Lhj4SMxP2GdQ/tF7bSVoWOzciF6ugBS91OZoDU04c0h0YE/ejlcczKXvdOfqslbPjwOjigNEYdZjek74VMjOiY8RhpZmQS6xeIyPZZghxZ1WBN93txlGLNiECj4uubLjxuEp13lUFXkQMnsMSlDWAcvi3O86men8JiZ9kJ2gwBs990HFpv6wQg9d9gXekp/+Fwj2UcTGajAtaT0NqmIFznsjaLbduhi14Tjec9BBoLlr7HDxvDJ7PkuyzrBpjcZBWkHg5Frvmyw+6rhKDV9rWUTw7F3jFXBeum0/gBefYCnUJM6kRg8f5JydBRB2gX7us9dbomBg89yHCDtequqkhW5s4Tk+zBFobFlIrYnqPYYmTDV41Bs93zItIR7p3IfD04ExHYHEwADtroMIkb+z6ySwP5u5QEV4kJ+WgwMsEi2PHr0gme+vNy1CyM9RcJK1A3mDMl+yR2dtU/xk0q+/keNesJbL0nc15Hcp2FGdplG2ysHdbCUVZxKLJY3PUL5sE624snoZQcS08Zv8J864q8GS9lF2quTu/ZBetsflGdS2bO+zUrxlULr8i0AuLajZRe3fquSxg4bf3tGzhXbQTIfBynko/jo4zk9bKhJ0Wy+mbn0RcicfFr85PmgXPsatZnZMcfajyLlqrj6mDX8ZqJTstNxANJKKu4HMy3bV0Ax18qeerQy7rYh0XbRTPqgLP0U+D84rvpbjCHFuhLuX9PzQ2XeLbEXdXFrLTQVld83pwja65i9bcDOHdRaseXWLlRWSKx6AFL4mpq7WL1tjwQTSFBZ4V8C7OFDPcoK6Bl09s2aCLDZSVzznuN+MxWJAtH5tXHNESIfDss5V4IUsmx7nFeXRSvG3kzQrysDijPC4riBYPYS4qvrlHxpo5+cRYPaV1Sz2qwyFWalrw0mL74+Gcn64LTsRmvZRYpmwht+Ps7FjGct41BF5qIlDi7OLOwdNdRnJhWkOzknPPZCyqa7NIpfI7BR7/0YwDM8/BM37nR0rPpExb3OSvlz/GEuzq8GY7J55E45N1+iYLuQia/S9yw0jebx33a+2cMtnat045LilsweMwiuCcJEXU8HzaskSeg+d6edXnVFe67k8t6uEL6fFxaczhiHr8jHP+KWnHsnNExTBxhdAosYMWT3VsGfXPy2Yyt2PFtLEWHW9oWme9k3H6Q7BvxPb/0NhUy2HXNf01Joa7jHuJtbvOGl3nHDzjXDgp+M49/iFaLc+eM87KE1U3z8HznKfni8GTZ+HVOQdPxvmlbRAj8JLb8ng++yy+QI/r2s/Bc/C6ljMSzggkA3DtTXT0Ck/cXTLZsNtFfrMV2NpOIGwha3sNUL69h0BUGEmLcYxvWks3zlkR+IZ1YI5tcf2mYtHcR6lMxZr2vk4QeD1uA/3j63ZhQr/3uPjI3iIAgYdOMZkIxO/YbF+tuOwlLuiswJhD29VyEHgT1x4QeBPHOstJxlmsJjontBU+uXfpXXSK96DLCS88MgwSgMALIsIN7SIgXHVjtFyNo21XCd2l4XJvOJguszaTVJljJ0NFp1YZIfAmrj0h8CaOdZpTdHzMRBcM+YEACIDAFCCAOXYKNCKq0AQBCLwmKCINEAABEAABEAABEGgRAQi8FjUGigICIAACIAACIAACTRCAwGuCItIAARAAARAAARAAgRYRgMBrUWOgKCAAAiAAAiAAAiDQBAEIvCYoIg0QAAEQAAEQAAEQaBEBCLwWNQaKAgIgAAIgAAIgAAJNEIDAa4Ii0gABEAABEAABEACBFhGAwGtRY6AoIAACIAACIAACINAEAQi8JigiDRAAARAAARAAARBoEQEIvBY1BooCAiAAAiAAAiAAAk0Q6JnAE9+j274/XX3eDJoWU5PdO+msVXvo0P7ZdPZhngduG6XDh4iGhmbR7Jg0g/d09iFu/SPX6TdKaXgnDRwVzLiFN6QsBmYN05j17ccWFpeLtG0t9S3ZQYMjn6FF01taxqlWLMGcaKP4rmnaZ7YsuIEuW9hQA2jp14c3MWOzXv3HNy2jK57xmck1Txhjjevwsi3zaWuV72h7vy1bv52be7L6/L1t7QxaPLqmhEG9/tFcnZSUxOfdrqUFHc+VxvfTY8ZrzD1dqXRvEtXmni7XvWcCrzLangi8pJR1P8RtiYvqE0RlRl19AAKvq3inSuLdFnhNcGrz2My+ozprsr0IdvwyNQnnl0Bf3BsFnlXnLguYJqaDCU3DHCdd5gOBF9G63Gkv6qtohWjzIhJRZ/uWSTgBd7zo1AK1dz8EgddZ+0PgTR4PAQSeRQACL0gr+qUAACAASURBVNgpdK9SLwXe9ou3U/+uAyw3qvg7PZVuPT11u4h/36pUbJr6zH101fljdNfsA+iOa/bQzcltpyZu1hN3mS7a9L7Vu5R0Di/yIGnBO75Ih+/ktHKXrctFmz3H+YpLTZP/bf5O+9ku3oqNINwTAyN5ReYOsji8J3PRDhMtWUKJJzm9+k2XZ2rpy3+fa5j4ZVmSZBYvWZ/n0R944xcDL09UPja3xH2ZCTpZjf41NDi60nDRlpdVumk2LriWFuc8zsjcd0o7Z4taTsxg4k7HUXbBRjJJfh+cSQMDiovWzIccZbHGp1FH85ma7ZFahldSbqnJy1bSJlb5ufvoLn+zndO+53KPugS742+W6yayf9Zx0frEjVqGcdUFnDaW1bfNMaN1tU7GpplX2fjhTA0XXKivaP1XnxvMOUVvd26TddQn+jv3/7RcB29IxjwN00Zaooz9tM+TOh9UnYNS6KVjzXbRGvOJNveVjLHA3KAN15j+w0MhNI64buv6aHBWMt/xnCn608F0hRliY7ZXxl2Gg0ixk6fDhdX6pttFq/fnUB8r2kLrE1Vfbj3jnIZD65VsAbN9s3LL8Vq2Xpnrazfm6VCbZ2NVWbatudXqN9a4sRYPo3s65p5nbEhDWoLreWDe9WRdbsETgukhOnfVC+i1BxYN+bH+e5OVJRVWtggcJ/798lxISeG2r5aOHoOX3TPNFnR0fB9deNwTFCGmpCPK90Ah8kyBl4m3PA3K8qFMgDrcvm5RW8O96rHgDamTQDZBFAMza0Sl46SDXREiclJRJgrrHrOxk849eNMcGsgX+mxB8MZbZIOVCnGZTzp52cJlzRelsrJmA49yIWLnnaeT523fIxecnKUpmMgQVFIYlMXIyDQabg9tsea0T76LTjuH6IKyeCXH4pWmQ7lIt96eA9YgaxGW9VUXIW3yDbd52g51Y/DcVmKtnMZiYLvB7DJac1+tsSnFnT0W/S9XLoGXCLCI8aC6aM12lgtNMWbk5K+/sMgxm4t8ZZEz/2alFdHnvWMtEVF63/K9OKxMOq98AalwjzIv6W0b0X8ixlEuXrUF3FgDHALKnIftOdOct2yB55vvfX1ME4Nqeatagr0vcorAtNYrW1W4LXiBPq+OaUe5g65uaw3J+t/AzNSYEGxzR78x2zdinSqXd9mvThdtgA9FzLu1BF6ialms3SFFFieiiaj0dyn2ZB66SHKIt+RGTeAJoXU/HasJSeM5S6yluWl5GQLPKdZUUUcuAesiVSMY1rOIjGoWFb1jpRN51inzYqSNmz/nejOrOJh5wFw/r2Szh/PtT+9kMWW1FiZRJ70+zsFr1MeZl1bGkoldCiDxJlltw4V7YmmuPfLJucTilL9WuYLWNU41XOhGO6eLcqI/RmbScmGBy0RNYgXijTUxbd6ZwDMmZlF5jxVMlM/94hVcEGqMTcvims/XZYH0rrIbfdBjyS0EntHfsnz1tnDfY7/4OV6MpOUi2zwVbmNybrYyx7om8CzxULRrsWnL7r8xc4P9LmvOoXobODd/mG1QMv/JTXKibNm4yMtgPOd88XaM2XwTkmceD/ZnhwAQZdq2ia6cvjBug5lH4JWtV66V0i3wAn1eFXhVLY88QwQ29ITb3D1+1PrV6YtOzeUUeOV8wmPSv4EtGIMXI9bSimSWO1mr3E2bCrXrZmeWuOx37y5an0vVKQKTxFRR5xCfmjgVeavleSC1NiZ/fYkqYh0t4xzQzhbMZ35DULgWI3VSO8i7S1XL2zRnC/S2dcpXNNFZxs4sjXNxD5jqZfUNvKI+yQu8Kl7zQkdMytpA8VhYHfcIj4vXbalS8wsmbbB30h4V2k0rmRECIN/uC8tgjOuZU9S5pcI/dcmkC5n6e4CHXOw6suA5+rJzQpQWQpVKBReGR+DpO9xjXr54EbVdxkWpysRpdldI4PkWPG1Bjhe69jxWdVynY9Y6DcD5suDYRWu6ykyrfL5L37foBl62o8RatmJ5xpG7Tf1eHJ9L1b1mqOVPxbIUeO6FPNTH0rVXnHCgWTaTv226hxYtPLJslVK6qrmLNrReudN1CzxjvDrbSN5TjONO52lXxX2hDuUv2zX7oqsAMfNZ5It7jCYJCrxUQGVuWrKPKtHi7zJRt1uL3YsReEb8nXDv7pfG5Em3bU2Bx+LNdRWCzo790+L6cr0W2vJu5FJ5EckEXhG6pycoJ8JOBAU/m8SWhI4u8L0xFh0qrqyxAs8KDcxqLge4Mx2Vr8P9KpKw3kqziVBhHO1eU1qjMYFX5RgRY2EU5Z7uEPaZGyUvbsA6qIvt6+mYLD5LbCqac1MSJziWWfNsdloHdfbPGpZvw6IUWjC0xTWr63joeIrKY1NaL32D0yeomxJ4RaytPilI91nDAq9sDuJwAjV2VBbIGGtOF20ez8sWYfOF1nyBMOPyjJpHxpZa/SdmHDlFu8tFq8T7criL4SWoJ/Cq9rGUi1nP8U1r6cY5K+Ksd865slcCTxGsHc7TeY+JaXOxXOgxckV8fN2+6FAedQVeSBd4dE5Y4Clu2pXJf59wzb7FJgTP0SUuq1+pBc8Z6+dy0Zpu3MzVK8sUZcEre6FxxwvmA8g0yZclVXkR8VvwtGxqC7xQ3F2RSycWPLWs4XR8FjwdbFDgTY+x4JmNJQWLL4i5jsUqyyPWMpe4UE5bt5JGvDFFRZlruQik2CsJBs7Zvp/oPVkcIGUuD7E5Jj/LLNIF3KkFL59oOVThGLretPCq6dd1aVUemz73dNkEUCxUuQsuZuxWsEAVuTcs8ErPuowba3bcpBkeYfYnt8DTXYQh3unvhSXM7j9R4ygo8DxjweWitdYM1RoUacGLqLbpsr9y00GJ9a7C2ZMeF22ZVdtVrNALWdZA+ktChBV8YKT6PC3LF9XmRmWk2FM3SNbpixajugKv5vmzEQJPxrntR6fSA3S5uhHCebBw5qqt4KIl16HH0lUrN2vEiEmtPO7YP+lKNuMGFXnjiCusYYmosYi4LWc1rABWr0rTGDsz8pBlp1tIj8GLKWtZDF46cfhEg76IhAWeOx13/iocv9slF/XWJgxXDF7ABeGcoJN01hINzLte2ZTgm8nd/U9OQmU7qEPxKdLKOaufaLRvdbrjNmv/uXOT10blkOKYNu80Bk9bAPrPSF4mDWGgLgYlC/FQmeWyxtj0HZpdzrfG2A25bItVS+k3TQm8Iy1LUCaZFFdi9Rg8+cKgeQ6kVcXroo2bG5wjRjK0+k/kOAoKvBLeQ4UQCQsLV/+w44SDY1gTtSfTXUs30MGXpjG00VdrBR7XoM48LWse2eYOUJbHyhJZ5eVysq8s8GzrrD0mO4jBE4llu1X5PzX3pWPjQ+GylceNRLhozd2wcrcrH5kihWIem9fJLlrXpgx9l7CIDVStlAJAAw3pTMPjllAsLlZcRowVwPU2YsTdbVubqIsVvknAjunIXWElu2jNsuYmb3PXoCqarN1JLneD41R850BZX2xtz83y6nZ9ffL0xrzkc0PqAh0J7iiMF3jMkd2fy8c2pPyVBf2Uu/1fMEj5K65Axe2QvmW6Ft5w8HAev5NoOWs3t3HsgxwHQ9H9s8aLUarw0pgidkv4jglSdscVOzHVY0xK4hDrCLykKL428B9K3IDAyxfvYre0uVHJNz+5BEZ5DB7HVekvcqI1zM1fmWU4eheteb+zfR2CLmJucAsYf/8JjyP5gmPGeTriUVXLuxIaIbnYc6Y5r9rjo3ofywhkfbq/fyb1nVzBNavNdeqXLHrkonUYF2LnaXUeUF/IxHFBpXNneoSZZqHzvGhZc03pl0qcqjF8Dp7lmYgYkx4lH2XBKzZQOM6IU8SfyCOxuF194H2JSKLsWJQIgZc8lgqrh/Jicozc0t1jyfl6WZ4yBq9/f7puKD1Pj0g/ekXf4ZslZW7a0M7o08Vr+oSRJv/JFFXW246LrjLJiAXRFZzseks1/f3GQlVR4FlxBVlRwwGsRsxVzDl4xhlxcmAODu5IzufKgghclhUz8Nq4J2zBUye5knPwlEk4fSLijCm54MkmLhMc2mSpnHGndA856RdiquBc3iZGe4iyJ5tl5iZnJppWEDVeI+KsJmtRcQZty0pU6Z/mAhb/oqS7SJQ3VLP/m22a1Hdr3zrt+Bh7dNYdm/aZe+XnT9YQeGK6yc6sVPqaOY71vtKcBS9lFWjjfE70jzVzzLrKv3xsnvsIKHVcBuYG18zLf/P2H+u8M8c4ClrwPIxG+uii5IVQigApprVzCLXx6H4B0uJKk6xCZ5wKBtk4cM8hEeOuIQueelacKDc5NiGVbrIo6lK0bY15WpvbY+ZOO87OYhnqi3V0gcuD4ww9iRiTjsEQKfB8w2jv+Lu0uqgHxlYOYt07UGm1DL557YVM9voqJwvRIK2YXN9Z3esbDQBaTyARBfpZp60v8ZQsYNt0AQReqJsJNS13Esqb+Y2gRpxDKK8p9jsE3hRr0Aaqo31ou4H0kAQIgEBi3Vl6F51SdlA6IE0AgfbpAgi80mb3bE5g0bfhYLosOfwVl58ABB56h04ACxF6BAg0QoBdsnzkFe9+twwQjeSARKoSaKEugMCr2oi4HwRAAARAAAR6SMC1ga2HxUHWLSUAgdfShkGxQAAEQAAEQAAEQKAuAQi8uuTwHAiAAAiAAAiAAAi0lAAEXksbBsUCARAAARAAARAAgboEIPDqksNzIAACIAACIAACINBSAhB4LW0YFAsEQAAEQAAEQAAE6hKAwKtLDs+BAAiAAAiAAAiAQEsJQOC1tGFQLBAAARAAARAAARCoSwACry45PAcCIAACIAACIAACLSUAgdfShkGxQAAEQAAEQAAEQKAuAQi8uuTwHAiAAAiAAAiAAAi0lECHAi/9VuuWBTfQZQund1DF5BuVhywhGt5JA0d1kEytR5uoQy/LX6vSk+ShtG0GZg3TGL77O0naDMUEARAAARBoA4GWCLxeomhC4PWy/FM5bwi8qdy6qBsIgAAIgED3CEDgEQRe97pXpylD4HVKEM+DAAiAAAjsnQSiBN74pmX0soGRnNDcQemSleJomBZsWUL5LXPX0NZLF5LqtN22dgYtHlIga/foLk6R35b5tHHBtbS4SJQGRz5Di7yeYE5jHfUNzqSBgfVJRnOT+1cTnWO7kEVZRmUZXQIvLU9eXEd99O5iu2jN+hbM/B1Nf4bLX9Q3/U3/m2yXfnZtT99Ep829lhYMz6ctS1aSbC3xW+72djFK8zDbWH8uKfO2tdS3hLlml8kk9LtIQu0Del3SVDNBlxd+DQ2Orgy6aMv7FifLbAomRGfQxjtX0IRHA+ydcwxqDQIgAAIg0AMCQYGXLp7qgpiKmVEh8iiNkUoW5FwQZIsp5SIwW9hzQcW1zARUv4ytcgi8NNEs9ipb+MkWjgUzKcrUsrqtc+UCzyybFCZlokAvv55+ITBmlcQYWpwzwVQILcOaZXLORYwinKw0XIykuKNCUJppi3R2KILTKEvo91zcKQx99VPaOBdueT+wR4jF2uxbWV1U9vYzPRh5yBIEQAAEQAAEukggIPBCmwfcLjR9AXWnUXZPak2aqVtZLBFhUlGFpzTzVRd4zrwz0ZCKWpcJUa1jDbeiQ4Rw7fxC8QbqW5cIa1XwOoS1nYaLketvUvSlbTDd1R4Kfjcz7QZhQTMFrlY/Z/vaYltv9Yi+Few3XRxdSBoEQAAEQAAEekSgXOAFF8cYAaXWzOf6dLtoNTdvsCyuxT6mfOo9mUXSsWtTiBHy7eb0WCAjXYFegSTqTJrQLdyRhotTCLzERWu6sbU0HIx8XNX0qHBxWq5bbl7FBer6PaZ+QkQmbnndtV9FLJf3LXa3x7jJezQOkS0IgAAIgAAINEogQuDpAkPPPUZAGbFXWezWuBYH1zKBV4Qb6tX1ugp9wqkkZs2ygPkyNVzDUkyZZelI4Cnl1GqsikhDQIkYRzUm0v+7Gd+nQ03rR1p/KO4oF9YxfUso0DyUQKbsFKqNDi0kBgIgAAIgAAK9IxAh8NTYK7OgEQIvyv3YMoFX+dy1gCtbbkDwCMSgizPHrgsVTaR4BJ6edgULXmmfdMfyFY/ov4dcvKkRsIYFL6pvufvswIhrk0fvBiJyBgEQAAEQAIEmCZQLPM8CWhQgQuA53Iz5Jot8J2Z3BZ5+UK65YUOvgzsAP3SUSihW0SdgMpIeN6kpelKxlm6GOHiDsfmlLAYvdy07yulrY2e7aWZHt0tY3qIKznFzk0Z6k1a/OjF4UX3LNVzC7dXkIENaIAACIAACIDDRBGrsolXjog5yfslCE0m+XbXiDBLpfuyWwLN3wObuwlxcmuLNDuwPW9hCmyzcGxnUxrZ20VrCy0zDKKd3F61qgXULG1U4psfQ6Hm56q+2MTk2YZhCOVw/e6d0cBdtTN9yCMdwe070MER+IAACIAACINAsgaDAyy0tyjl45vEl5qfKLCuYeUZa4qrc2rcut0Ytmt49gWefrZblnQf0R5yDF9wsYQonO+arYOZvQPM8N+uIlBEjHi/jKjYPzLkp3ak6uIZGB+SZb+44Odcn4fxnHSrWNrUPGEzsODv7WBl//XKznx4r1x9xDl6wbyVpm/dY8YPNDiqkBgIgAAIgAAK9JhAl8HpdSOQfQSDoTo9IA7eAAAiAAAiAAAhMCQIQeFOiGZNKQOBNlZZEPUAABEAABECgYwIQeB0jbEkCEHgtaQgUAwRAAARAAAR6TwACr/dtgBKAAAiAAAiAAAiAQKMEIPAaxYnEQAAEQAAEQAAEQKD3BCDwet8GKAEIgAAIgAAIgAAINEoAAq9RnEgMBEAABEAABEAABHpPAAKv922AEoAACIAACIAACIBAowQg8BrFicRAAARAAARAAARAoPcEIPB63wYoAQiAAAiAAAiAAAg0SgACr1GcSAwEQAAEQAAEQAAEek8AAq/3bYASgAAIgAAIgAAIgECjBCDwGsWJxEAABEAABEAABECg9wQg8HrfBigBCIAACIAACIAACDRKAAKvUZxIDARAAARAAARAAAR6TwACr/dtgBKAAAiAAAiAAAiAQKMEIPAaxYnEQAAEQAAEQAAEQKD3BCDwet8GKAEIgAAIgAAIgAAINEoAAq9RnEgMBEAABEAABEAABHpPAAKv923Q+hJ8+9vfpn/4h38Q5TzwwANp48aNrS8zCggCIAACIAACezOBSSfwzj77bLr99tvpBz/4AQ0ODtLpp58+qdvvwQcfpFe/+tX00EMP0W233Uajo6P0zGc+s1V1uu+++0TZuJyLFi2i4eHhVpUPhQEBEAABEAABENAJTDqB973vfY8+/vGP06c//Wn64Q9/SM973vMmdZv+8Y9/pBtuuIHe+ta30q9+9Su65557Wlmfb33rW/Syl72MhoaG6IwzzmhlGVEoEAABEAABEACBlMCkE3hc6AULFtD27dvp5z//Oe2zzz6Tvi1///vf02Mf+1h6/etfTxs2bGhlfT784Q/TO97xDvr3f/93evazn93KMqJQIAACIAACIAACk1jgPelJT6JXvOIVdOWVV06JdmSxevjhh9P69etb53JmCyOL6IULF9LWrVtp165duaiWv02JRkAlQAAEQAAEQGAKEZg0FjwWE3zt3LmTDj30ULrwwgvp7/7u76ymmIyig13OZ511VmusY//3f/8nXLEXXHCBYH3kkUfSunXr6C//8i9zUf2d73yHPvjBD9KNN94o/nb00UeLtmBrJMdJzp8/n44//vgpNFRQFRAAARAAARCYPARaL/A4oJ83Uxx11FG077770v/+7//Spz71KbHR4s/+7M9y0ldccQW9973vFRYmvv9tb3sbLV26VPz/C1/4wta0CJd/1apVdPXVV9Nf/dVfCTfzHXfcIcSdah1jkbVixQohtJ74xCfS5ZdfTs9//vNp8eLF4tlHPepRXanTH/7wB3rd615Hv/3tb+nLX/4yPe5xjxP/5jw/9rGPCSH6i1/8QnDltmHLI28K+fznPy/K87WvfU1sxti8eTO95jWv6UoZkSgIgAAIgAAIgEA5gVYLvIsvvlgE9LO4YGvQnj176OlPf7pwEf7617+mhz/84aJ2P/vZz+jlL385nXrqqfSb3/xGiA225LHAYytUmy52dX7zm9+k7373uzR9+nTasmWLEEInnXQSbdq0KS8q111axthqyWLrgAMOoEsvvZSOPfbYrlVpzZo1dO655wrBOWPGDJHPJz/5SXrLW96Si2o+MoUtdCw4+diUd73rXcTP8cWi9EMf+pDYMMLlxQUCIAACIAACIDDxBFor8O6++2561rOeRW94wxvyc9dYtD35yU8W7sCrrroqp8XHePzud7+jZzzjGeJvX/rSl4gteiz02rQJg8v0pje9iS655BJatmyZKCu7Ol/84hfTJz7xCXrzm9+c1+mnP/0pPe1pT6PHPOYx4m8ssGbOnCksaN26WDQfdNBBwiXLIlRefDTKN77xDfrlL3+pZX3RRReJMm/btk08w9ecOXOIrY+33HJLt4qJdEEABEAABEAABAIEWivw3v3udwurELv8XvnKV4pq8Nl3L3jBC4h3dP793/+9s2ocK8YH8/L/P/KRj2xVBzjiiCPoxz/+sRBK++23nygbW7ve+c530o9+9CN67nOfa5WXz8djMcgualfMYZMVZEHMlkTmPjAwIJJmUf2UpzyF/uIv/sLa1HLMMccIF/OOHTvEvWw9ZXcyi9CPfOQjTRYNaYEACIAACIAACFQg0FqBx3FzLBw4Foxj7/hiKxeLnO9///tC6JnXBz7wAeG2ZTdh2657771XWOT4iJd//dd/zYt3wgknCEG6e/duy9rIhyCffPLJQjDNmzev61WSfL/yla+IODq+pKjmHb58jMvNN99Mxx13nLDSPfrRjxYbKlik8nXNNdcQ1+dzn/scnXjiiV0vLzIAARAAARAAARBwE2itwGN3K7tj+QgRebGrcGRkRIghjldjsSRdmCzuDjnkEHrjG9+Y38/xahyH14aLxeqsWbOEZUzGq7FIYosXxxf+y7/8i9g88rd/+7eiuP/93/9Ny5cvF58Ik5Y9tpCx65Q3PXTjYtc2izh2G/PmCb64DGwN5Zg8FqK8S5aZclk4xo7F3dvf/nZxL7uf2Q3NmzC47XCBAAiAAAiAAAj0hkBrBR67/x7xiEfQddddJ8jceeedNHv2bOGu/exnP0vs7mQh8rCHPUzsSuX7+vv7hUDhHaZ89AhvvvjoRz/aG7JGrg888IA4zJjFnfyuK9eDd8WysGNBxbtU+Qsd//M//yOsZBwPx6KJ3aNsyeQNJ3/913+dW9earhiLSt6s8qIXvUiU8X3ve58Qciz8eMcs/4+F9Z/8yZ+IrHlzCD/zxS9+UbQJi0He2cw7nHGBAAiAAAiAAAj0jkBrBR4LNt5gcdlll4nPd7Gw4J2Z7II95ZRThCWJ4/T4+BB2CXKw/9q1a8Vnv/hiAXj99dfnsW69Q1zkzLtT2T0ry8mfWvvqV78qRN79998vdqOy0OPdwNOmTRN1YBcoH5/CF39Jgs+e6+bFx6TwrmUW1Gwh5Y0dvIOXPxHH5+BxLKC8/vM//1NY937yk58IEcrWRxaGbdu53E1eSBsEQAAEQAAE2kigtQKPYY2NjYndsnw0Crsx2YrF58KxyGOxwxY+tnix65ZdtbyTlg9A5uM72LrE1r22XRzDxt915V2nc+fOFdauL3zhC8J9y+5lec6f3FHLrlG2kHFM3GGHHdaK6vDGi7/5m78Rlj65q5ctprxZhIUhi1NcIAACIAACIAACvSPQaoHXOyzIuYyA/LQaC9KNGzeKHcAvfelLaeXKlbn7GQRBAARAAARAAAR6RwACr3fsJ23OHJfHZxSy5ZTj8diNzrub+UsiuEAABEAABEAABHpPAAKv920wKUvA7vJbb71VlJ2/eCEPmZ6UlUGhQQAEQAAEQGCKEYDAm2INiuqAAAiAAAiAAAiAAAQe+gAIgAAIgAAIgAAITDECEHhKg/LxHl//+tejm5g/N7Zly5bo+3EjCIAACIAACIAACEwEgVYJPD40l7+92o2Lv0vLn9YKXXwESJVrn332Kb2dv1bBZ9w1fYXqw4clL1myRJxdxwckl10cT1e13rH14U0Y8lNzsc/gPhAAARAAARAAgc4ItErg8Vl3mzdv7qxGnqdZ5PAZehN98cHL/FWOpq9ly5bRJZdc4k2WOTLPJzzhCcQHEpddj3/842nPnj1NF1GkxwdR89c4cIEACIAACIAACEwcgVYJvF//+tfiM13duPbff3963OMeF0y6qiUrZMHjg4tDAitYKMcN7B7+0z/9U++jnC9/9/YlL3kJnXTSSaVZ3HvvvcRfsOjGxd+rlZ8260b6SBMEQAAEQAAEQMAm0CqB1+sG+sAHPkBf+9rXoovBwuXaa6+Nvh83ggAIgAAIgAAIgMBEEIDAmwjKyCMnwN+r5YORb7rpJvGtXVwgAAIgAAIgAALNE4DAa54pUiwhwN/g/fa3vy2+fMHfDcYFAiAAAiAAAiDQPIFJJfA4VuzHP/6xoHDooYfS9OnTmyfSgxS/9a1vid3DvMuXY+badv3kJz+hXbt2iZ22hx9+OD32sY+tXcRf/epXdOONN9Lw8DBdeeWVtdPBgyAAAiAAAiAAAn4Ck0rgLV++nD73uc+JTQtXX3212CU62S/eZfu2t72Nvve979HcuXOJ/92mi4Xnq171Ktq6dSvxkS+//OUv6UlPelJHRfzqV79K27Zto/POO6+jdPAwCIAACIAACICAm8CkEnhchXPPPZd4M8R9991Xuot0MjX4f/3Xf4m6vPe976Xzzz+/lUV/+ctfTrzLmYVop9drXvMacWTNVLHAdsoDz4MACIAACIBA0wQmncCbM2cO8REg3/nOdyh0REnTsLqV3lVXXUWve93r6Bvf+AaxkGrbxQdQs1v2LW95C/3zP/9zR8X78pe/TOymJM6cbQAACC5JREFU5c0WuEAABEAABEAABLpDYFIJvAceeECcZXfWWWfRRz7yke4Q6UGqb33rW8WGA/7iRRvPjGPh+YpXvEK4x0888cQeEEKWIAACIAACIAACVQi0XuB997vfJbZwHXbYYcSf55o/f74Izl+4cGFez+9///vi/Lq+vj4hQH72s5/R5z//eTrkkEPo9a9/fRUeE3IvH+Z88cUX0+9//3t64xvfSK9+9avFBgveXcoXb2bYsGED/fznPxfxeY94xCPyct1999104IEHdv3zX2xp++EPfyg+dcb83//+99M999xD06ZNy8vyla98hW699VYRo/fiF79YWFW5Hbg+vBkDFwiAAAiAAAiAQG8ItFbgcUD/m9/8ZmKBd/bZZxPvNP3iF78ogvxZ5Dz96U8XxNjyxZalt7/97cKqxxsVWCixO5Gf5zPX2PrUloutYVyuN7zhDUIEvetd76KdO3dqx4acdtpp4vNmH/7wh4VAlXF5d955J82cOZM+/elPd+3zX8yXxfMTn/hEWrp0KV1wwQV0xx13EH85g7nLi79zy8KT733HO95Bp556Kt1+++3E8XXcXvzMk5/85LZgRzlAAARAAARAYK8i0FqBxyKIBR0f0cFuWbZqseXq4Q9/OI2Pj4v4ux/84AfE35gdGRkRFi3efPGe97xHCDzeqckbMtr0LVS2iPE5cP/0T/8kRB5f//iP/0grVqzIrZJ8fAhbxfgeFki8U5hFKl8sYFnIcjrPe97zGu+oHNvI5ePv1/7bv/2bSJ/5P+c5z6HFixeLo034Wr9+vRBzH//4x8W/jzvuOCHAd+/eLQQpW/ZYjLJFFRcIgAAIgAAIgMDEE2ilwGOBxkLj3e9+t7Ag8cXHdfBOU3bBsmjj67Of/azYifnSl75U/JutSOwiZKExOjoqPiPG8W2qi3PiERc5slWOLZJ8nh+7ZPliccci7xe/+IUQdIODg8JyxiLula98JW3cuFG4cfliqx+7QVVLWpP1kQJyy5YtwhXO180330y8sYV3vbKY5otF9Dvf+U56/OMfL/598MEHC2vkF77wBcGcdziffPLJTRYNaYEACIAACIAACFQg0EqBd/rpp9OnPvUpuu222+hFL3qRqA4fjstC7pJLLqFly5Y5q8gWPhZRLIradklLGLs2Ob5OXkcddRT97ne/E4JOvVhMseWOxeABBxwgfmIByKJPWtKarCNbSJ///OeLODsWmxzvyJe0inL5+XBp8/rpT38q/s6ucv46BS4QAAEQAAEQAIHeE2idwGOhwS5BPnONhYY8CmXNmjXCosfu2YMOOsgixxstXvjCF9Kll15KHMPWtostYGeeeaYmUFnYsRWsv78/d3fKcj/lKU8RR6bIrz3s2LGDZs2a1TUhxULyaU97mjiu5Utf+lKOj2PqmO1//Md/OI+l+cQnPiGEHQu9GTNmtA07ygMCIAACIAACeyWBVgo83kjArlcZB8Yts2DBArEZgYUOu2nZkscfrOf/Z5ctW5DY3cn38O5Zvo4++mgRi/eYxzym543Lu2bZKqdaJb/+9a+LHaj8VQ4Wetdddx2tWrWK9uzZI/79wQ9+UGxg4IstmmzZ5IOG//zP/7zx+vDmChaVHLf4vve9T6TPu3w5Hu+kk04S+R977LGiTdilzG5YdiezILzlllvEp8xYjPPf+W8cF4kLBEAABEAABECgNwRaJ/AYw6JFi4SrkAP3+dq8ebMI3mehsW7dOnFILlu2jjjiCPrRj34kxAVvRuD7+Sw53vHJoonjwS666KLekDVylXGFLEr5e7O//e1vRZwbu565/CyaWACylY5/Y2HF7lHeZfub3/yG5s2bJ6xobNXkjSbduNgy+qY3vUkIN77OOeccEQP5yU9+kp773OcKQc1nEHIsJMfc8WfVeLctW1xZuPLFsXksQDkdXCAAAiAAAiAAAr0h0EqBxzsw2TXIZ7Cx0OMYNBYfUmh86EMfEufiDQwM0De/+U0hNlhUbNq0SYiPZz/72UJw8C7c/fffvzdkHblyedmyxQKPY+7YtcnxhHyGHFsseXeqvNh6xzF4LGx54wgLRLZcSpdtNyrFu19ZwLGQZksh74695pprxJcneFMI/ze3BcdCsjv3YQ97mHCH845gttrx+X5sxWMRjgsEQAAEQAAEQKB3BFop8BjHH/7wB3EUB593x1YivtiCxBsN1GD/sbExISqe+cxnimdYmLDFi92zcqNA7/DaObNY4l2wHC/IAomtd7yBgc/v43/zxffwsS/y8GN22Z5wwgkTEl/44IMPCvHJFjv+qgb/m3fS8r/ZhSsv3g3Mx6Cw4ON7pAWPxSsuEAABEAABEACB3hJorcDrLZbe5c5uWLZAcrwbW8z44vhDPvOP4wvbKFp7Rws5gwAIgAAIgAAIuAhA4LWsX7BljN3PfMYfxyJyDOF5550nNovII2NaVmQUBwRAAARAAARAoGUEIPBa1iB8TAxvbOBNInyuH/9v9erV+OxXy9oJxQEBEAABEACBNhOAwGtz66BsIAACIAACIAACIFCDAAReDWh4BARAAARAAARAAATaTAACr82tg7KBAAiAAAiAAAiAQA0CEHg1oOEREAABEAABEAABEGgzAQi8NrcOygYCIAACIAACIAACNQhA4NWAhkdAAARAAARAAARAoM0EIPDa3DooGwiAAAiAAAiAAAjUIACBVwMaHgEBEAABEAABEACBNhOAwGtz66BsIAACIAACIAACIFCDAAReDWh4BARAAARAAARAAATaTAACr82tg7KBAAiAAAiAAAiAQA0C/w/2J9xHN52aXAAAAABJRU5ErkJggg==" + } + }, + "cell_type": "markdown", + "id": "57e6a7b9-30af-4273-95f1-906bc3da4756", + "metadata": {}, + "source": [ + "to find dL/dc, let's start with dd/dc \n", + "\n", + "d = c + e;\n", + "so, dd/dc = 1 (and dd/de = 1)\n", + "\n", + "This is a local derivative to study how c & e affects d. But we have to do this for L, and in a NN, there will be 1000s of such local derivatives.\n", + "\n", + "So, let's use chain rule\n", + "\n", + "![image.png](attachment:8211c8be-62e8-4102-aaca-a5bd17838677.png)\n", + "\n", + "> Intuitively, the chain rule states that knowing the instantaneous rate of change of z relative to y and that of y relative to x allows one to calculate the instantaneous rate of change of z relative to x as the product of the two rates of change.\n", + "\n", + "so,\n", + "\n", + "dL/dc = dL/dd * dd/dc = -2.0 * 1 = -2.0" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "416bb5f7-8e8a-49d6-b6b8-784e0bf26447", + "metadata": {}, + "outputs": [], + "source": [ + "c.grad = -2.0\n", + "e.grad = -2.0" + ] + }, + { + "cell_type": "markdown", + "id": "f5989699-809e-4b6b-bba6-7af1d836fe95", + "metadata": {}, + "source": [ + "Now, to find dL/da and dL/db,\n", + "\n", + "dL/da = (dL/dd * dd/de) * de/da = (dL/de) * de/da\n", + "\n", + "dL/db = (dL/dd * dd/de) * de/db = (dL/de) * de/db\n", + "\n", + "de/da = d(a*b)/da = b\n", + "\n", + "de/db = d(a*b)/db = a\n", + "\n", + "dL/da = -2.0 * -3.0 = 6.0\n", + "\n", + "dL/db = -2.0 * 2.0 = -4.0" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "9c66472a-bcd8-42c7-93c0-2c44c8941469", + "metadata": {}, + "outputs": [], + "source": [ + "a.grad = 6.0\n", + "b.grad = -4.0" + ] + }, + { + "cell_type": "markdown", + "id": "bbab9a31-8408-475a-98e3-58796dbe78cd", + "metadata": {}, + "source": [ + "This was manual backpropagation! Let's nudge our inputs towards their gradients to increase L (single, manual optimization step) " + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "15926195-7075-48e8-bf02-d8b06ad904a4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-7.713030143999999\n" + ] + } + ], + "source": [ + "a.data += 0.001 * a.grad # 0.001 is the step size\n", + "b.data += 0.001 * b.grad\n", + "c.data += 0.001 * c.grad\n", + "f.data += 0.001 * f.grad\n", + "\n", + "e = a * b\n", + "d = e + c\n", + "L = d * f\n", + "\n", + "print(L.data)" + ] + }, + { + "attachments": { + "34f0ebc5-cb10-4db9-b7a9-9f501a3732da.webp": { + "image/webp": "UklGRqA9AABXRUJQVlA4WAoAAAAIAAAAkgIAdwEAVlA4IMA8AACQ6wCdASqTAngBPnU4mEkkoyKhI9KZuJAOiWVu/HyZcejLHpANyu8/6D+1/jt75nKfcj6UyidrHYflo8+/937tPnH/lP+f/mfcn+m/+57gH6r+ev6kf26/WX/S/AP+lf5r9uvdf/2v/f/2Hua/q3+s/KP5Bv69/jf/H7YP+59iD/A/8b/2e4h/OP8R/9PXW/bz4S/7J/wP3H+BL9h////tPcA/+vqAdS/1//wv4++B/9V/uH7P/2X0//Efln7X+TP9s/bL47s3fYZ/c+hX8a+xn3v+5ft1/g/3p+8/8J/qv6/4p/Fj+e/vv7qf1v5BfyX+Xf4b+6/uX/gfVH/vu2Zs//tvUF9m/qX+y/u/7w/5v0fP6v0J+v//G/vHwAfy3+l/5/83P8r////b9996190/4P7U/AF/LP7T/1/8r/k/3d+mb+f/6f+U/1n7Se1/9C/xf/d/z/+w+Qn+Yf1n/hf4P/P//H/Tf///3/ez///cV+43//92H9iP/qOsuHRJyFkY/e0Z+Pab6crGqCJe2/fhZEbyFf32s7BkXoX06BbJ9XFGiuDIvKdUElTI1MS2A+iwRZBLdG5+DD5/4Z94nImfkroNl5TJw4ws306BbJ9XFGiuDIvL9Qu/QmsRLZR0oKlvM6cgop3bxZTc7CTIXxiCIPZe2N4NNWRW3tFTWaar3BVET8cqGS1BSEC3OuzbautSQWRehfToFsn1ba5Dx8ZKCrDh1lkOSjcNFOiPqpZ9QoDLuKsRPRmVOus8dJ01jfN4GXnpws3P7595+noqZLhSviE9z9ybnDuPGTejv3n76t9zYP9qpYkm7kckzjdA8X8EGAI0VwZF6F9OgWycK/Lc1+DyDvrqhC9Xa6/Q2Vhc05qrWqP570fvYgjHbpwmyUkAXyd+6euJ04n+LM2x0SPBzgu2TrZALeFCaeQ8GmfYlzeRTR90Mhn0YUj8W7yYYrijRXBkXoX06BtbgyL0L6eyNre/HV/JtR7lqgX2FlXXAKFG37IfhPeygAw2oJf8IijRXBkXoX06BbJ9XFGipYxyKuQr8TbvlJPCrO6Dcai2BrikHb+JFm+nQLZPq4o0VwZF6C1Bq8CQHJu+AHUsBbotBFFRqoU/lt9noEUeNYkBA3gjAcF6uKd2Z1UsYe7vlX/3Cw2RJMs5ZbKx/AW8h2/7a5Asi9C+nQLZPq4o0VwnpKJc3bzW7rqiL8RAF6G8ef0NYXCJ0jjfLo9MWYFOVa3oLHZ+BLlDLXWReX1IuxIiAo0VwZF6F9OgTBrKYzOZbkD4Tf9/AXwssDWmyIAz/QEIS5EI2xneZkg7sNUWhp7peGFh6MxWzWtdsh8TnjLyCHbWPwt8b1YY8dGC+hql6Z+6jwHPMwEPLla8Hy3pK2Go5lkmrk3gg3XXcUaK4MhoPHAxMs7OdCY4IxBlD9rbp1DGTzT9AEgwkISyCwYBJnYaUjmN3km0FXlcBb6YZWprLEkP12HUgTZ32Nf2fcOb4hKcQx33kEaqrRUSLN9OfpN/Z9ghhiQDCXaodZlPENDkqu8w1VCWRPZi3DGvQFwBqRVOI9U3ZaYsJ8bjVeJVIlK7iBCLfQ2KIdsCF5iRYyN2ZlebiMzws/1A8/G2R32vPJILFJHIGmfVaeBwlIp1rJc4wphdNfbhKdTFemjmBO0UWEvKNQDGu6byuGfoJSSp6HmrEv0Rp82abnsq9VPACAzDLJMA/xiCFzoElmC//puPvVv/vlgyUFJFji04l8pQhzcqbntKi6qYZF6GufKZeTmdc+B1/EeDGqs3h7h4SUj06VEQkZwscI1KEyq3+7kUGT6OMkx7/KgMZvySCi5/OuiYuZOvEbE/CnZP65MTwyoyOSA4JkTTyEy6URKljU3hbLHtuSOXX0lti2T6uKNFcEC6/fB5R4VU/tNWgJQnP0cfGOxYB9j5iOxuFLHBzID/5nKz1VnoHPSjYk2m0Uqc57/yztMgjxl283Vd1tT+k6tXjdtYvQvp0C2T5AYSQazj1XFGh24d3Jcv29f+yjZnf5kYPvary1GHx5y1M94cs7f6QhyEoQw2ItOgWyfVxRorgyLcep/0G2CYVRC+nIy4x60A2AOCVmyoIFnG6zjrwNulgUJy6rPmgmFIb3COFI14SQWRehfToE2pV5sBi18gwhgadjkeGybpmrspKGKLUIw2uIJXH+DId12Gu0aoTDMykj7uLXs3yI6cmdzC7/hRJ3/KydAtk+rijRXBhwAv2GVhXvANsjUdLXOmjlPpKkzca2Jc05tusR5JEKGafO6uPzgsIqrkybR6W1Kmos/clGntbor/f+wH5/qsC/85VdQqy64GWLxyiIvu/6kMoueSQWRehfToFHQIcrqo4JAOuWsvJoal/Em1ZPRotFp0C2T6uKNFcGRehfH2wCKKo6MIkgsi9C+nQLZPq4o0VwZF6F9OfkCpVhnjZkoFGiuDIvQvp0C2T6uKNFcGRehcbbS04xLU6K4Mi9C+nQLZPq4o0VwZF6F9Ofp9KMqsWyfVxRorgyL0L6dAtEAA/v+dzjZP8lj9Apmy6V7P6Y2n89UCzstoO1HuzUPoKtZ0a73r8corm+Px+gfOLr8yQDrsJ9lo+z2ZxO3xFWzj+UyY9AW3MKv2SAmiq1rmkLMNMOozdbKNaarQm6KuWguV5E9C2mBUByK4iCzUJkHnu5aoHh//gP18C3dotv3BbDvI/2qDUOGusfY8VDSCtB5utqRZxJ+A+XJ/6KrSs9Ln2zTahYGbSvIvX7eAZzE1fD8If6mIMa9nZtrAncTBR0OPbH6F+mCc4YN0N+wmdONhUCJ+OIltXfCTNTN8mGJrjHSU/JEeL+WWUbpcTiXi6RG6lduUsA0+0pQ2HsgBCR35m7UslJ1QQSmjvEw2IkjmZKcOFOP1ddsZcg6yGQYSOYSDNtq3fNm6ElqdpbrCP4RY75Kw6xMnmkZ6h1qa1DUhK8/y+LgDm1VnHMZsJRzDE+18XHYws47eB0rmxV9tPKKLRgtXRmYhZx6KJcj95/xHCiyu2Qux14u2nsX6RJqYF6OtJKtslpQHY7vWI+ii/MAHYG/baoEKM6Cp+kr0ciS/ezSdI/XWsyoMH5FgG22ttl6VfXyrSc2u3DUX6p7DMPGct600W9ElseW+r7xKo6sithV4VfUXEo3SQVZa3tlYdnALyz7sx4ngvZxbcpY1J18YvfgWrS2X6R072nMOCmw4n1pnHm4zZlJnFQFXN4KDTIzCGgf9DDdwAP3yC2Ihm0C48DYoMh2BROzvrSsg7fmyVnTRCA4x8vGOH1TyQEVSElwhnxFEmbYt8avd5pYizJj+9X28STe2Fj/zfbjL4v/hZWseqjuitxAj5eyqZbIAUQRLSIzu6e8YUSRHPy4zA30o/FEDfm+GZuENWLOddnr82Sfwnq0XVR38VFUrcYdpvjCY1j5U2SYGswbwAsaYHI6PET4dONRB/9acV/4z3929EfimyCtRx/F4CFNkkBurRhjA2ULD+grZQ8YYeS4WkrNe681zm5fJTi4Lh7ho+7ytv65Hx54bhe35suHMqPEjUZferE36VyUvSHjYw/kTFjzV4Wu7aQii2d8m0LYU89GkBlsmw5PcDt6Wzp7fAyiIP9erD//lW/J9BmYV9vQsR6RBklhvjbZeZsQu4z1kxka68q/Zum+AMg6GMdPhd+OTS42tvjRXGgfrdsP56XyMKIS+ns6+2GIAtQg++3Jgz5BYf3OqcdvsH146rMzEO4xWBGwjz1+NgGl+SeiAgNrWutJvvAwXkn/42zaTLWltpyln4ZYjXZnSS4BDQOoE4GvCpolcDdW968PufZ781OXo1oZAX89Vp6pr7VIrYP66VV4Z3PlLJnup64bmEFaI6ns/ExxIdsULHTGP/DE+LcIx7fvMO00IqasdK8YptROQu2/M4i/i85QWFF8Xt/F5e7joSoZQAg68pHlmxcsRUMxGh/RkoG4Bzrg6CFQMk0y1SYEcvj4WWBV6dFiGPN1SOdSOFBF6KUscvhO45Fr944aw/WFj+tEqPI36v+V118dK29PcokGl/mLpBWnwzvMtRb3G08Mkuo0S6+DhKeDFE2giiWIiZ5uQQqUQwAbZAiB3dLr9c3MmvxCw2+HnfAf41Y3KyFxs16dXWzNAUQijE7xeJ1gLtRcv5IEBrV/Vm4IgxsYndqTR9JGQVXb4wQK21ogBLuJ1/bSv6bTd/poSQL8VPlrqexzPkFP1/NvZZfMgew8CF3t4hLIqH39dx6VK9TTS7XspQITaFpUZdF4G5rbm6dUpXogeHdccnb7rvLX2b1ahyswstPgLgmxtGQilScyFns0VH/lwgCQIg2o8yJgFo2DwEWgLanSobK6BE0HBRN947EVJs4aFNZXW+bJAr3LX0k9iPPqcVeXB8kMzO4v4IO/bqnSF6wKd7SdaLW3SuIKgLODJFL1ZggrJSg63hkDyzDe/+C+05gQIPPYIOR3VtntibDuWSjeu19kM0uy72VGScNWudYnI/wtCRCMJsarchgUlnaNhq5dpvqYiRQZG1HFO6E8kCW1gMJD5XI7en0QbuneLmu+pZ6WSNq1XmciVsHhljn1bSgwiKjd+nkSd5Jj3XTAxCZfzDq6wpMGLSdQRukVLsfjl+Z0DVYc2fZZTmAlGbGIyIQE3WdcO3QOod6K97y/iXKuK4pM/lqkmE6730P7KIm0vWy/KwziT6g3dCByNl9qq6No9F1lAJp/Hg5e4TeKFdowN61lf8YgfZQvFSFpaH7d5Q73ewPFuWU4a+q8C4KtcZ7E3vu5CpKIKwgP0uH50io+7zxuAc7zcWxHvNuBTRQUDwHmV+kRXAg3OQ/2l9gn0kKC8glcuhNc5TObYuvz/fTO6uvGZXWVV9zX9YrhGHWX1XUedebz3I6TvpL/XYrh/+z7b8g2RVNKk4QOmj5+JmAelxMFpHTbZP5+V+xzR1YB83FDc+y1Zq3KgGFtIOjOyA08qBBSUKWbuo/sfNaZgjAMKLHslK24aDaRgMP4O6m1MN1OWFcmrIOEnLmRznbzgnjzEuCn/qjQz/QOPzzkBfbjv43L8y7EMDZkNWj+4nh4+rLUh4X6D3wOqwmpnp9ET9071J5Rdh+rou/aEAN3gA6BWofwQED4vTLFNRLmuLzjN0jicOfAXyfrPGr6dhjHTHjxbxnVCuEPQ75kSubMAfRbwU5kGP1HLdB43yUx7QVBo4YVg0648QnNmWNi8qkoZ5NIRy9kR9f5Gb5vSRrddRhbZf0PsjtrV6XctReQuKltNx63gkTRG9x5VlyVPMa/XnKc1QMsyPrkP28mJZjbOR8dzTXgGGob3UONYdReqkQHkG1WyhGcN5t3XE0wMXLooYHB17LC+rEzhkFzSvSZVhTYiR4L2lO6P2TX8ZeqnYjvg0DA5JoaB7OroqPhXW+2OTlNW9ZE2/MG9yl2l5t7hNLi8MZFlwE9dGJHWMnDb0sxGaLp6DY3+lhSzHcLaXWCNSiy83iiHNa5vCqa0dd8tvOz97s6CvTptCoiGRuwDgLFkeqFKO//4BIQgWt87ewiE84HDw6+EPXFqgapXylcBn1hmXsbYUI3VZj91hadUrlCGNIV8fbTVgbKvABb+bM9Tg6rc97QOqJG/NxXOZMjyXCYeWA93jEhYxtR92KK8VJHemVhk8KgD17sohRP4msOgmiU8Kq3LKyxkalLolxr92M196cjmJCM4QDPR6EJNKuL/aXY/ToMTqLlrbAoUL2uoqjbaFgSI9nqDy5pz2/68z3iy7sQacXACYTbpBF/km6lOs/pVisQzrFRbCcSG0VcJnzBrBYGN+vkVifYRrylGg4Ui5wryHdswAYsSAlsWFcDZ+je3FjKDJaaUoN6SIHgFN4eHOQriDWFLP8LXVhchXv8Q1Wx79idlsHKjiAW4h2Ph2jErjiVVy7DH3KDYk2Jmm/WImVTS/lQvnq5EyNP929sNb43aXZjky/bVguLFRTnqCR4L2vLDN7p3xmqJ08ZrAyyI8sw0+HppAkkDhc6dAQBs/oG+fSBlW0hDIeSLkUL9B+F8ZL28HpNeLWfJz4TQ2GnW9vdVDFKrMZPbJxON0NQdtFlVK0einRZtYJs5EdyIjNg7Y/aac0NsvdGaSZgo8NleH6QV/vUe2To0hJr6KrjpEcl/pYEmJ3qWJwpemNUk/J6qdBbZbPQwIEZATaSiy+yynLX7ulKRQvCwHu2FcEpj+1T+hFY00JxKvkUB3XtsLHtveeOGJqxiDW4zkDQVN2UcVZCxNUjXbjLobUa/ocqH2BfSXI08HOWIkEAGtbUIUDco7tHlADrwGJRqotzFGDZMS0ydMpDP2Fzo1PLytTA1PZa1ox0FwzvbEIjtm2R/POzkEYiVHv/TLbEnAv57teXsv4adWZX3RF36x96zjK6hJwqK3Z/7agHiNaIY5Bjo4tbzWmc2DpXp/pl2t/Npf9xzf49Ijmu3h7wKalgCaYBQO+U+EC0xdLTTZ4HW1RIcWh72Am25RWIo/7+8bSsAQlrnSPwAAw5Hia1dOs1D8g4EscNlMx3G9YAAyJTpLZcPWmzweCeEKeqAAAAAAAEci5vXK0cTje0Zm56kTX8WPMLrKdHHtMl/Z/CEKf/mqRJJ3fhAsxgMwxcMo8v/STKvKGM/fQDYkq8VcFDy6F0PQBRxU3Yh8roPzMj5LJnz63942WfnP33GvA9OS9xWp/WG85KGs9TbKPnByKNrVOulK4wxr95oe7zpuau6boUhaHQ2yKSRSlu0N0RLfcNRyLOD8Yx7QJP+bHYICS0K93JT+E6F/9lOCijffil/APHlRrPXz71a7LVmYjhn9bOm7Qdmfis4BB8riaZDQidYviD/s5u9l7WHPoznvEiOl9AUGa2do5XUCH25O5CWLfDjd4AASrUnE2TZlr3PWL9xF/dYEJXr2FjRH8eKkBQ1O8FcypUju1eYa0wJnxLl2amfCXXwR6HQu4AK4Vxvksfd+kRtNkjqRtc3j6r0SfbaSkunpgeUIwYcEuwxRMZFDcvABfnauXNe6wb6NZkeVSRUZuEGNfJNovvyjdYbx+TT3Nb8BLjLnY9hB21pJa/4JdH2+vRZOHuHABGLBpVK7xxy/IiISdZfmYYkuxcvrLbDu5oVHkeoRowJUhAb1Lt0Jt7gwPVg6UsK3G8z8d37h0pUNk3GieelTCz1VsqF3rtQOqs+9JR5VuIUaMSY5/kMH2Bb8hcR5D2LsO0TErpOhVkIgP5okiYYujd45QB5WyJoI/wz7/cOeyTJYQwmZeswEzI5Hlp+BY+n7E/tG/RXlO6VR7LInaBqjHP9Nyoyz9VsqWsKoztC0MGBxpXUUil/+7CZUYAQy0rvug+GTuG2XbyrhsXCd6eF4YC69NVLgTTmawBjhQAMYQuq/xAwYlnD6+eWjGGaob1s22dWtDeQNKNvmoqBoWhFoCsq+0WRfh2/jp8tUp7q/QfP4JcYNW7+zC+z1azWjibCty2wB3Antc6Rrm7ggAcgIOPvTILmR7Eu5og8zzyFLF3MSjT/ev8/YL3Kis03cEnmeZXIsuapbAB4UA4sD0DdArInHZEaEoccLU8GaC8E6oDkQRIuz4Wa80bXryqORco6hIGF/iFTwQRHryU8ja1ufEM36X9UnI1PmV9PsIvh07Qk+p4/Fp+CdhDAOZjm8hBGE0SvXJ6X91T5ZFVwlf+1rptcq0aove8EFFB9kxWOa8itZ+otplGrn1+ecSZuXY/uKiQ8L9OiNcRww7JFVehS28kYg5J6QNTOoslz3P8mwfewT5XEZOPVnJoLeKgC5M8iCRJPoNnUISd6SwmXzOsMo2O4HSRQDKs/Tn4Z/lSN3nb1wzrx0rQA/57AFKeEDlfH7kFdaI4bYJwcdllhvpxAbYxm1Z3RxLO/cRdgnFPWj4IkCxkZIuhXKrjG7NG5xWcDWBz2s5/J0osGT+XU4/4oFEfSCM9xzy5g6bfR/W1FjyqOaP5aI/Z/YEfMgU64gr+2QpV9iBw/rSpmdfsbAdf9RdRRTnmDv2mkOaQ8UULDbXtauXDAUoa+XZlrb9Dejt8kbIIbsUtA0gjs9t4Crr+1Md6d5O8H3KTT1cwBP+Tj+OEaiJQsDkkEUKxZaFVAl/s/s3fAnoXd8JD5voMg0G9Fjwb1izop1NncJqr1wO6QPQw7WLVB1vhgavm7BTSmRF6CIp6tYkPMKHI1voWsSXWxi95FxvErjnWxnKWFBwpbKG6YfNyaSBjmtc6+DFg3WybfGp5xtPsVIdbrTOt5l/auszqjXrn0Sg0e1UHb17OLQRyCoEgWh8XxVg//ULQNq43+WzJ97MFjW7nVMEkypEH9gU9jq+qQ0GGszLOi8Z6Jlb3g4kHfXcbbxGPyIvZ+8ZD7+Gt4IGgbR3S0eO8YMdBiokbOI1kq9evmaulcLTM1CBQGUOdbdlSDKnj7T4LKJAffM5yrBR1ZB3hLToClrXA7cixLLaKuuI6+Ok2M0iJph7jkgOkAkmlpHSpCEB8DH6p1JbwAAAAAFqI3ufVq6V0ix9SQgSiACkLUPi7uKh2KEfk2G8Jsu0+O6yys5KZeMX501ZBWWeNxvCH7a/GyPgVQUAQhJj/+MwIjQj3P9eNgdWbLMR3a8QlMZ75C0A8KqblejIp7WyYQhdN1KJgiK/5u3DoqY7biylZ80EMZ3A+d0mHcx3k29q77+jYcz4sId1AitjB+Tv0QPTwdy08yVJRaBCw/2ueEYPFAex7141tK4dYIK/DaDKIrFsZdm1X7Me2R3uW/ZWaG6qLp7XABKTKg3zTgGi9FTd9wqrAf4uWiXMBg2rvU8y0T04S3l1wUDFBm+2ufI1lWjFWV1rFd6DTuCMGXk/JpSqUGpFaK5abiVg4620OzZSJBBa5kDq7jRIxxSd5MyZ6CClvC8g7GrBnOrxHoEPbb/vl8Vw22Y00Y4aGAOYku4bjS5LXkHSTL+XribqoVHyKLP45RZ5MKycLw5Uy7+wTaleEDXIaItyUIeN4rlKU9ZqvoYkF4ZlzFB2bwwNG6mMi3JnRXnHukzQah2GgdMBogeAICVKOAfavGAJpC5KwucKjkVTvAZLMwP8xekWBbtCQ6v443pzIX+47TRNqNjEUwwyAWtC5k2UCpDsTOYILfP4suHWHn4MdiJQmjK21UFgW/euyQOIofGLvhIUv6vxPiFaiuo9xrZbi2rd+bdSTpdpZVET8XCR8VxTxCI6qk+IzIIzGNri1Hj7q6ubcLp0ffepAqLPrWRIu33c3YmKMTBw9TSHXhZvuSI5QMBqogmKQ1gA8KnO8cjdMMjsu7NHY/zSjmW3qn/amRP859VdFlHVxX8Rfo1Th8B0MJDdi2K5pb00gAm8LFV4/V1pd8qgmQiLGVthm3pAs4/cNhErHYaMjcW9x74SbdKVemTmLJJ4dMGifRbNN9jJyP13joRhIXB1p63z5ypfzY4CJZ2Z8I3e/9tp+JnWuRnPs/5JXCXjiD6TUrftIIjgY2R/PISamhKQ8hJdu7vo3AMlDCEmR3zPWoovULeTEaqTkshJ3ZoufrpVA7NB7gtrYjfYPX77b+k4dHuIKvjPPqnUxYMfT5f7EI+qFxTV1KTXTuTOZ5yB2p3GmZP+B2NeXRyTOX5hs/jE5ks3ZM+7XTZ3giltv+lx2gmMGXIME2NPPUgmFRK4eyZlOJ6QeiTl1HIXGQIfPZW1AdUQygUGZ8/7/tDA7IMpEA0N43KN5H0Cpzq6ewKf1zlq7vaLujEzM9mTLJ2ZFQ03h3C/BxYLc82tcktWNHDoftkracBfhXnP1pd/wETUwQx40/2JIo27o6BpCOS74Po+OfsYgXjMljfz0l4zWhkW+0iapL6/7Qa0IFHjEMjv+T7KCczY6W2XC4Xprwtsgq/15Bw6wXopn0c3+m7DgHXsxY0bPrig3W1wq58Lr0Dd44LV9xMzr3jea/32f4A4SoiW2PCODcmuh8Fd+glgNLD15UZI2i+4XOCGyP2rh+d+m+y/JCAB1iQz3i14XE0K0uJczs1CwgbJGEPr/No4Thxcow2vg1HqHGfBvGH8Ie+7Wivu7QWGs8W5tWEl5b+YAqBT4jUcgz/Vf/AbDkLO10GuwX6XbdrJ5NbM9g07LQIjjeeshL1vO0xmL7LhhPVQwP2/m6cdO13HVogXLmQ7o++dNpUZB7+MGVsQObCrk5jSl+peSfkh61tQuHC1/H1v8/4HwpB4KlxF0+nm2rA6swDe1p0/fp0RzZr9aklyfwSB4AGwuBlMqjLjn+mbnh1Zb18t6cdetw2mROtuvAdiF0bIasTCOq/9QLKtGguUrinqUiQMV4QuF+I3ivP7LQSX9ip4xJQ0V2unFwLxzSDMrUj2JKUktQUQktJ2QmXkSvAEA+JNcyJE932GKvH+6eXaicXIULAbdHcZZ+pHKX804zN1I4zOG+paGBIMglpHDWo2c7yhN2A3jX4GS+9Sf+v+YCk3Ohd8KGiTRMVMA+CtOt1uHV66RSa40nqhT8hRNOU4e5Gdo08fqaJEDurFIJWsyRXp1QnrYK3Au3xj4BGYcr+K3XR3gHsJEvwzXvrD1jNAowhu7EUPf5LTyS5MtfC4N8zDND2h2AAk2ASvnSCH6DsiC/+9S2ryw3xt0O/WCuMif79i8B1Sexy+Fvh1nXv4RXwqsw5UD/4WxdfH2ygXNRqN+Z45C+06cNC0afVPf98IPULF9HCpKKxyjsX8M+hEJYsehIHX+M6WSp1H3Bp4cI4boTeRtjkAuO6MB4IHPMQv14KuEBM+ckKE4YLd8Gt/7B2ZnJhR80klaetgPmMVUQ/SqsDNs7+6mm7ItVx/fq/H+9OtHGUJnBKvL3q/4Xh95poXMO4unu8tXEd+Gzcu80nQOiqljaTLmKKNOOwVNnOerpcdR2Q2vpindY2zLPq19H2b0Zr95tKQy6YwFduxLao04eYMBO1nzFZq62NEdZBephLseGaw9sHboc4STQF2hn4bhL+mf2REEjRZRir+OVfV7Cdf93Djhtyhj0S6JsKX1MlpK+8rjWjjzYbjHuMzSUAfMJIITuJ7XWssC8VwJv0MSM44NvuzLSQCaMbxajrou2lbf6UQ4aF3CSvSrg7sSiWTuxkXUpUSWY2ZePJ/Mqk7fh+xF7lCB8q8culLZDkJ00D/wuKKneQKOI4HJBcaQelhMk7nVuuFJ3mVduMhZDZxopImc7rPSMXqOLP5yR6MimiweKLKEO9+/swGwnlkbEB24iOcxfnubp0OBmZUky6ggjtSPsXTJP+FpHn9uk7qrXqybRgCX74CHvE2jIAz5B3bh6+A/JObbIZZfXulAgWv1hRTAADvStpELodVgugME4bywQtSQmlHQWmFLUgr/bPbAx+uxtT8v5/1nbQTVfmm2A5K6O6H379pOo51OwNx278QdSthNuEkd1OJL1STFX/jAQPci3+9N9ski1S9OUv/fNnBJreYQ5yFM10uHcKiB2Fxaaalul8rFBChLsp5/Irr4ChM1w2BbPHdIQDBUnr+KgjjUyKFVu7eCdmwJ76tKoaZJUS/z36chbChWBtJtzOvjqTWfy/zerV6UtUIaOp+T1Omc9pMdK8e/2p7aozvm2ZUp5McySq2EORSYeLt5bFkr6Eyh2gQsAK3VYc17pZotIhuPXLybeBd9a+l9QoeG/7nulODwSRYZVyPSJkd1xcO+UxHRM7JMoAKKi7gP4g3X3/rJ+iFyHSqJXTrztQ1znb3iFk1SVAMTaeKhY3O8vO18u1ZEBNiGkujcycEi5+bExEcoG6jJ99yHyGITnYxNFv0jCaKOzBNcLsvbaQd7uLfPNxytXnADt0oX0PeyDtMgNXh538mq76FFnQwRZyPkYsqfsbD/U9RMBX0UEswP1Wxre1966jXbGiqU69UQJK3MDhP15MCNH3J/Yojvk+gcV4SAZXgcfPRD1CYBJsAtoSxD3m1o8pbK68MaxSc2a1vZGN/op9b48dZrS3zw1FrXEjCgDujHpm3zz4l04cS23bEb02iLwH0zjoQfCYT7LZ8E+FiUGZFDoK/pyvS8BS4wooqgZRpgMboPdG8fSn+MNY3uAr4FFlzfIJhajoVlKNzJpUInACvcplR2pC+w2M35UiMUukzKZX/8SUxwcGcrw8bS29uNvEXEIjr2ooIE17PYsEnJvpn23ijtmFEJlfU0BnZKx/PEOuNkkG/Kz0Do5ay0IcUVatWOl9sPdiksQBfOk4FrqEz8CJWPj68mEte22T/FIK82A/llACcJU2a4T7F1MqsG1000VTwcvwdDx45RNykz9kAcu8RQvj0N/ie7UV1GKgrL2OlE5G0QiwX+Emx9oSqVNugjNPHTJXOoP2D/IoRG+IU9EFf6+0JGBIeInNrOPWVya6S2BszQnJ/eypGqBMlGfrxMe3tebJV3rPNarPZM4CmRJ0g2abzNZQlWZTooXNksQlC9oRhbQKPYMpSVolJaJ77zYB/8BHxtd6GH5nrk5OFVj12SKtzNk7bxD8U95jk+5VeOdMlgMlrqB9RgA593pZ29Op0Xl60gigfi5vZtmkoU2Y9eOZbWN/w5UuvAJhJU1x37IEcc9rtDVV9GqebdFcJ3QdOzFffZ98ENUMfSEJ7ZvpOFWDYLJ6zp1JiJPUsIAIp/muZrkeXGCI/OS96tO1GZObx39Q65h7AuRvz5FRCj4gcWS6nxLwN5ynOeiOnFk1ZRkVzWJqswU3Up9dZvPhJ6kIv5QNa6jhIhev0UrFD8CG9ax7SNBCqLMQrwsu9rMEbXinZRRzP1ovayHB5nQK8+jRlzfZtRqY68LAMywdurn/0XHUFVyjQ5JAFfg0rBVgZ0Ymz7LYBBkM5Zu86YQ+YrHkILSbkiFlgY0A8dkuAZaen/kk19SOrJsq/oCuvF/ADM5yeTnZJjEDlAF6ZcAlTvBOqDcXafnMmvatJyB53sGJzH5hfcsmcKVQ+vgs2atnugQAd2vgLtAkoj0WQhXlyHcuJYXYVH65IbtCG+uIBODd3a53QnJT/CLjksVA4e7s4qSLcA0bvU6vqulVFN+Jub+/l0Qblk4AyR6dx0gQCUAAAP5+cB1qv4J4HG4x9JBk38Ggox/JiIJmcA5K3fLVGbA9ziJ5CyJ/CLEzGk/vYPZT3/jbRnFNl7UDEWjiYF6VEK95HI7G8s7xrD6AmgFTV//B/Ws0kmO/4aSWDPeYFgd6xUtYt9I0MAHMiSdCbTfU3qNjxSVn08VPyyPhhVQxaksua6eVqNXGojQzGtN1arzIFY/qOkBUT8xaVThPzgTiJyj5o5o+z3+YTGYGS17d4XrOzgI5IhS//0RoeEcc3b0Y18DF/Tr1Tu6Ho/ieDD70SilkVLP3qnm4cgB95gRoisWwQVP9wqFeDi5azegbqgrNi5Nf5pnd89Jb6QLX9my4C7RWkvLMTDGn80lrGdEE0loNlM1rlNN30CVtqAxhsjRDsK8oAvxQ+XRwgWq2S0+IZq81t13MRd76Pg1+MeWA5B6LI/Xg7f7NNcL/mqU7cAJR/wmN66odn37yuCsVJKTO0gRhc31Ikb2BOOJ14Ye4a9rQUZE/qo+rKakjaXwJhkgcrCxTE8CeEA8oudPAG7xWMDE+uDBZFtx45Ez+b0Zj0oomYBeKKtLdxK5Y/KJhebAW2qzL7Iy4kGWrVpkmAd+L6Qv9J5HyLHi05uHYcNKkaCb1L+irDDDg4GsnGWISOkKHBcpBSVYr65PteNNXBFS/wUzgve1Sqi07aaAKfqFqeZZrOuXw9vuxsCnmc+dqFkjfjYcxb+yMVdqRRFTXf9gLOumn/V5NmQAwY+eW05IdlT+s73p+x5Kh79Sy0cRGvpl9aN6GaBaDiotg2QXKPGRzxf12Xv33MUs00F3elyXWLhNtjETAJcggZo3YABP5SXtVVl/vp1Jbl+P6YrMMNpFKLLNMJHGwwUg2SngB+cjRlLqKg13Om0WYM7IoDxlBPachcU3iVSPdAm2OaV/g8HcsiX3A3b50WZLT0ApN/erKDHhFjcxB8kt0JNTvtt6ZG3DLO4u7giFhjAluBIYPkaxnrjpvDY6T3p2d2ethaAyjpefQrP+WY5Rs6QBpGvYkcTP4a3PavA8HxLm4f54J2tQ60LtZTJ4nkDZrc9MRZ7mu3UqWoubx9BDPg3HOwPWa5TEZLnmZQAQqfEOzWBubIEs4ezE6ghAeWgFoMedhIsjeLfLNrcdpSQZLelv8/Gse/RzPa0Ph1OHtB0IUwIEF7mxjbH2jJWyhSy+BoTCepm63NsnT/fPjtVfHg+6yl6NfZLyoNOxVNmzMaqPjHtGEIfI7XbGDo5dk/eeUMx20Rprxd9Gat7k/DhfBuo9ld3EZm5vAsX3sUVn1SUeVA/xpnnarsfABiVQS70dfMKUtHb57fxfTSzFtT7zTsZLWcBza6nmgzlz7YHJkcJ17Y+WK2hphCVuTlyCD33ypKWKEY05smCBvPBo7aWiZXKeE+TOPHzZ0PeOVzRaITZl64qcdDxkJd9scMATRwpVcweyw7R2rgmqRYmLozWADX66V0J7YycACI6xkjEr5U4ZJJAn3p/qXxgD0JoErSmux3LevveIrgywaMhr7odmqCckxMbou3K8IfxDPFUKooSytyV7XiO9Alo5q1NkqTKO2uquQnccTloiROgyukCrIxo65DqaSFAEkAuZPl5m6zLJ+q4eDgHNg+Sw+NSnhqO7Koi3TpjuAu2EH+IYEX7oxCudCilx0bId0DYPL0OvhVOm6P3Fk1/RK0NU3Yj0ieKLGphwDmLnIWLB0qNliInV/5Q3taNED7rNHT72KE9y7vnviLd49onL0Odj8wJ/eB1Y7gSQiHtlZw0FwhVfzzcajG8mG9CgxPDAOx/cvVO/1Rzr0CQQ7m8pYJ1sSHEBfZ6i7yUOUJTGZZ9HdazssbUxcafc9LcOCuhFuefs2kythRleVhMfr7F2q0nhZpcf86gDEwtq2kA6DZdgKF9E7F6GmMuJDxnedx6jdoDQnh9tx1axpc8W1i+X4jjxb3QrFD/dr4mbMiRxf+xGGCy8q+U2nNyEv7Rk4J/wDivj52UYYbMilddjI8LIuitCbgaVcvBXYP88IHK+8imMc1xdJHlNNw6i6E/bl9PmAYZPZSiV5wG7EtvfSUZ3tWvP66rPBDKe/whMsghh8TgqAbalepzqP4E30o9OcPVR2vpCAJMWSSS7TwNLeJ/25IF1rsOmJVSVaGuq22XCYjTf+58CeUklaYugitUtS4H8mRn98g5BNq02b/+RZRf9Xuxzny4gEr3KXXPAvgFP2I0jQB3Ycrtji4co1CqkxfwuMNjDlDLoSq0sCl7vpBi8VTT68JyzdXB1fNnP0zzwyRlCjSgGDt3i6TrHToMfx2ogU/vWJtt7n4JxMhejYMhs0GGf5C3NLKzzeqaeS0vv/B6J6/XmgaeSC5qiiSEbb0ZJFf0XUIS4dRVBlTR9eC//VRWEyQTFvt0twZsStdHhPc/ZOVH3YZxP0lwHYKtFbjJPS3fMQpWt87H6hnyF3CVhxRrDV34602+WnkQ1c7jkbPR4l+eBlqWY573vsRUkkjLu5TFqytiztN4Xz7MTBn2fes1U4vL9axRTKvCUb0h5CJ+OAklPjB1bvpsa6/c7OuH8NXsfKByXwh3EARSdx3KzFlaDYGIPcPCBbPUaQme7+1IX/w9Bvw3Aok/Lj9bM2PgLiREqcAU+YzDmt+XK75+1J4iwJdml+c0VVX1/p2Uudwwy7mTMWScoJXv/UWtFFeJn3qKiRldZG0UNkSBIQT/uHrDXUQ4HZIchrXRrhku0CSe1uTOAJ8JN0lIwdEd2QfxhfMZTAo0VFnY38QqdAmbK3vuStE799wqzqDhbQ8tf3dFqOh/9fs8rWUvEJZPLC1SMBJhEgdW7ePWkLEn1iU/T+MBGmkpPgSj2WqAE077LGIEH9uaZw3Zhgf9+HUMOaDcf4t0Stq+q0yBiZsfbcNUM7laCx09cN1eGLGGTRjXGIXsJBvNiz25K62riVVOdp0+qGVDXScNrFTV74HlJBUXgMtFg1qG10DcJzVCVnBUDRXam3saBe6tgq6Cc2DETPGXWcC/4ebvuOIOmD96KBAhXH3+i+AL/NzYxEj7Pkkx9oQzosrejQ4dE/ueZyCi6pM5MZzusUpp2RlH9X13vEyH3Z272eOkX0YwOS4LFQFPr57dtx9FVu/mh18u959dolBv3tNK7ui0LyZgf+lYTD+NgAtGQw9COg7ZbaxRaYGYTizccCCzfB7NYxokS2dr1H9IgoBw/PB+81kkoyu0aVdAm2irE7v6y4Wm+H49fGv1Fwhhav2ji9bl/pk4OBgS7iUX0eIdU0v7x1l6yptV6MwADsXKFYc/8ylzgw8kswdkpBq1ozsUhKrM6CI6GTO7WsaIoMRPszYn45Rx4fxCRQ+Cp/720jTjVwcyS9to3k2dX4b7y+Fy9s+c4elOkm9gpFbKPACoxcUuhTy2wfP/LFb13uFvxhCDp5IdEkOjpDloCNzL414ROJbwjJgFl552jgLuFZUtlw72rSJUCz3NOuzR8IUFGlHFRqAeOtw/G79UMnwn66Xt3NCo1zmqtptO2pUyq7Nce+iDuVkH29V9N4hGN+aX7OlMewlnz26SSq6ckdavaxUwvZn2hql6yfXHQqqgfOKh5aJsjE/yZN0eg17qQqn43OaWvhR3aJ6imkizDdlJfUPq9a266u/wiIwKD9xzT2AcS7swVcgCngMgyJhd96qPVrEiZQSIObxmTQuAJdE7ZmzxsygskO36bop8GUKmRxc63inNpR8+aWqLytJcwFDcDVAqX+MYrAIsDBNqQE/uXD1f2Gnly1NwdADME86CZpS5LnAKv1OT9K3jDcSo+UJYtuVFZEb69UE/YlYOA+DNBJx3vlY5c08VfuSbQ85coSJzdU7VZJ68VQzzqXHPj3mg6N/kgRXzZRLQpNqFaFf2ddQfauq91esKch1Lf5NyW3KRfR5cuYy8s56CeJ4RMAXns6GXk43KiS3AAClwP71gus3I8Pbaybnk8PMMFirVknWy2E2tdPT/+K1dWP7oHh98X+raq4PKhr/deNNTyReos6FEYA2X+GVb+j+v4nep9HzimIZ2EcsHbXIcGtG7rNU9wIvnbK6BcfFd428a7uN+KHupK3lsyIM6rw1/DFoOZthsdU66XZVllTtL2wq/V5RKNHY6UX3WOLjpRC6BOl1+y2Tfe5gWtSO1PZgMdN6/ZKQ8Ki9friXy+MRejXxKOV6yHcm1LmY7y+8JPRCmRovQkG08L7qA4GBw5//aoJmdCKRDVSvmC5/QaqnOBQGHdfXQGbCW+vNLg+w9dcnCJb2Hiy3+CqJKP5w1qjXicXUX0V6x130K+66UQNw5zzmeaThiz6iCr4pLZRP4F+xXt8Mjf32fcKGOIr/IPS1vJbiMU0tkVxwG7zUf65SJqyDP9+7ucLljnOwK2wE7z/cBjSegI++PXaWe6Z+sbZeVgsOhbY6DchJ56BqQsAj8ydp5G1dd1NChhdnIIWLrurb/CczIp4QeHc3OZV+dhQoyv/JzwcKhPzC/KsYORSGfKIcdGwCHWd8eqcEluLnhjT+JFIShSuf4R6NVWNtA0HlvsLqeEl4+R2vkGnUuWwLNP+W79hstr6iwGGH/eYqFbOlE7LboEffh8oI4K2uFqiQqQVmvn8YJXCLxq3MTyNg4ARNUcoFJCouGBilQzFB2YdOWK8tqyWrvws9NmwZUfaQKq/oerL2SmBqO2LvmnEtFtOKxVIrhyOeALOyJoANzBAAIOYrgGtVp+MmjaaqcGaqQI8dhy260hNhfSQy2YdiWnoo83PibORKE4UzoP6879B9VxCLVUTiUAECX4QCq7uWj4V+pRS5YaQjM0zB5R4dJC578TpJa9o17OTZTpYRiEG1WgDKVDWXzpBC2hVesPLxBTUbbL/0PNOo7w1xtQ9NFZKba63ALrVzUBTMS9fYpQ14Ztc1/m7Jap9dsO+eOwuXNoG8yR2Fd9f55YeOM+nJenLFTdLgn0JOTCAdko9VEWXwHuaAqoRo1vdvALP3SQV78Egehoijp45+rtVsxboztg1Hw7mW5oCmXadRyjk6j7LsOqvGTquojaT0QRz7RSQ3QgFlxiB2jeAI4taxs6FNzVnf7RWxoaLBwwGirPIJwuUfWpwNnhlbUrfvvu3HkwpK2b+Nwn06GVBhgedUD/PSQJ5f0QD/R7rJ4vL1s7rmzPPT/nA7+T1QFr45+W7ESTZvBM6rRexVWZuWzxAc01+rmjkcqIU6j3s0t/ASBPRFntQb7MQiTLL784wXi8EdCOCrsfuDGZdEgYETjx39OznFa3ylCkG/7+AEuX9jjMNVjIwZkI7zjF/+uRPpEj/c4oq6jKvmtnS1bjEDQ/YsJJ2qnETFFQULdqE9n/7Tj+az8OMdnup2DyZktA+rkH1D7X/2lP0CUx7EukTzctPDIt7Llji/W1OAXv+J/YSh9lfaC7LBiedph3XqJwf2qTFh7hZIKAvgyHV0Ecy9MLtdzpnBTUzSXaTL/INmgYGPr1I3IY2hOY4zT86Aa3ogOtDfBvnidGuCGjhDUfw/ZQ3lxZRX3RAHe2wRolQNJBcLYYc2nnxp2QxHvZqiSRyx2G5EGvUS/Fs5IyTKRZYn4sqXurW9qKSJxNdj/ZbGnv13nmhbsOVg51AUnfuHXI65psd+K/EnCkVn1sPdA+w/trVpR085u/pUok4/1p3T/YEJ+6+Ey7EDr+dh7Qufpit00VB7fBRFfUpYex+CXKBM+uI1L21ZFuXzk64lXCmwcE1J/GtMS/1MNxay7CFoqJ8dq/TK8XqO3qMTCHPUrYDFqeEKBzIDc1HqXipFo0GIMMyY9P11DkJLSPhT1GzIUvqv4XhW5HJ2CXt3kkKRtmDq7O4T5XDqLovhgJjp3rSdBnCh5ohjQALMZkpn4lhoijAy1xrdfrINPwSN0xWoC5jZ417EQX5WkBIMtMrPd2+biRD+gviWiFvcFjVex7hVoIgfgBO+VdVK4Airf1Q0L4H/96L+bCpBh6aUW0jfdv+imVyAjh43uLC53eRY6C14OKlmOE99+uZxPefo4WSbzU9+/ryGzcKMRLozFrmC/1ozV0vNKu62q3iQk6BxxDPi5kRBkf3UcLWnVXqO9Q5un6zu7XPyWJTzW/M2jNWZTcwz8J0POlVlzvelEgyCKGodnVWYNbp0VF5B3aAH+UK6C0BVxN88Ite9LKv7hDttLaLI0+bKP4drcRGZteTaFzPwY2AsnbNqNNyRA5/YLsmsikUZ7VBFUQuNx40yR0YNtIrEWOamgx/em1zkiPKetIs2V720TuvWNLAib0Io/pqxYWBZDR+yH1GQQDRSWTBc4af1sHGUPuw8cbNHWQcThLbUmmmUyiHscVELwNESR8571B4xEGghgk1Kh5zWQe6iHOoSh1iU0OtLTUkpXYJ2p/OI6nb4SlxnySErYoHHe5318pY0zq4IlQdwyr9xhN71VeBWE5L0BzFQZ4ykUyavSK6GOE8+wMsT4Iem9foWItC59YRyuJQsNAl/LfOHRW9n6Xtr/D5Z1yVAi4KXUd8ffpNi/VlJgSENMq0rRcGKuDgbVO3tpy0eMnGIxK9HSx9Gcl9KEY3oYxIfc6VsKFusklazcSWKev2jyfjkqpgMRPH0NAoWArTxO7s4Ht8xZMY5AYTg4lUCdZ356m9wmNBSOmhyxqgIIUoPJjThevd485IhcELj6QiN/nThe5SP+LS3ZWUy/2R730zU1Uebt2Zcgca+RuZ1L7PTM4WPAEq8VpnFxEjvrBE7ek1nkwoYwS+aMUGQ2QCgqaqwnRa/N/fjAcT5LicD6gC7Nsl4EungPx5VCtXQ7pdfiuUhPtORAffrSliPElmohBIrHIfymBsNBkywuotX/1agSp2a8F3vybqrF73gW6V4/HyVDeVzD35ATgcVNnzuzeCzp9NAdTqAdLDHywFgCVqRB/5s/6QjVHvwCUroEt5MT8tGybxhRwKwjjI+A6BDwUBlRs0/zWjNfpLnvt0hI6R3GDnEAfafZ9X0h0MhfhArH+zWs5GdE9LlLztjKAywI72j1P6FAK1I0es8Hmr4tA/H+2k1tGKjAmMkbAZOmTsL6bB9gYNbjtbDGtDsIGhW8gb/2kjFYzxiiFuVscOyvo1XIbZhN3/gWfOsupK9usQiafrgz7DezKk1PNPL+13njr/JjEz0XmiQUteftxan+waU3l+otiB0dklAfUzvUwl2FG3UJxZxoGYziOUVtPw+Pd7v1hBfg421MALArqyfzS8lZ7ca2a71OByXJpm1tztTH1/p0sJ2z/9IB6aofilBcUySie5Lq+jy6qAPvARpOv12rurbhI/D7IAVHoYZsAeqy6mVvHq6AFwSCNYD89KfZJ1Db22XutvoiG/dwR1hqbkgFaeuGfyn09AAAAAABMRjc/vrZ9EuxKG8yfyzTaxC9XnBvVTI/pZTnBKHKWXBjKXdkCNrKwWsV0iJ5qt+3YvtK6Nbi771vioTtzFOdO2FCYpw5YTswunogAAc4AAMkqnkHrxlnGnrEEmtpGaEP/IQUWde8CtlKBPHnRRGhjlhaXp0FkBjn6vEEuKLvWjbRuTBSZbT52OsOLPTHfsa4uU5msdqW5nBDgAAACF+BGn8PcYFfTje/DXTePmKdwopgAByvkX0a+bNf/yN42UYphaApr0QGSVrlRsUPe5zZlH/Czco9I30ZCf/vCbuTStlZJQHjy0WngAALpitICmyTlBjtSG9gAio40cBX4AAAAAABFWElGugAAAEV4aWYAAElJKgAIAAAABgASAQMAAQAAAAEAAAAaAQUAAQAAAFYAAAAbAQUAAQAAAF4AAAAoAQMAAQAAAAIAAAATAgMAAQAAAAEAAABphwQAAQAAAGYAAAAAAAAASAAAAAEAAABIAAAAAQAAAAYAAJAHAAQAAAAwMjEwAZEHAAQAAAABAgMAAKAHAAQAAAAwMTAwAaADAAEAAAD//wAAAqAEAAEAAACTAgAAA6AEAAEAAAB4AQAAAAAAAA==" + } + }, + "cell_type": "markdown", + "id": "4eff7563-3a93-493d-8788-bb055a23d07a", + "metadata": {}, + "source": [ + "Let's look at another backpropagation using a neuron\n", + "\n", + "![neuron_model.webp](attachment:34f0ebc5-cb10-4db9-b7a9-9f501a3732da.webp)\n", + "\n", + "activation function is usually sigmoid or tanh [[more here]](https://en.wikipedia.org/wiki/Activation_function)" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "id": "29f8a9d0-16a6-49e3-b1db-52084db2ce24", + "metadata": {}, + "outputs": [], + "source": [ + "# inputs x1,x2\n", + "x1 = Value(2.0, label='x1')\n", + "x2 = Value(0.0, label='x2')\n", + "# weights w1,w2\n", + "w1 = Value(-3.0, label='w1')\n", + "w2 = Value(1.0, label='w2')\n", + "# bias of the neuron\n", + "b = Value(6.8813735870195432, label='b')\n", + "# x1*w1 + x2*w2 + b\n", + "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n", + "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n", + "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n", + "n = x1w1x2w2 + b; n.label = 'n'\n", + "o = n.tanh(); o.label = 'output'" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "id": "1a1bdc0d-23e1-44bc-8b8e-1e88cdb64b2c", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410984016\n", + "\n", + "x2*w2\n", + "\n", + "data 0.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "2379410985680+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2379410984016->2379410985680+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410984016*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2379410984016*->2379410984016\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410987088\n", + "\n", + "x2\n", + "\n", + "data 0.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "2379410987088->2379410984016*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410981968\n", + "\n", + "x1*w1\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "2379410981968->2379410985680+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410981968*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2379410981968*->2379410981968\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410982992\n", + "\n", + "w1\n", + "\n", + "data -3.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2379410982992->2379410981968*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410988688\n", + "\n", + "x1\n", + "\n", + "data 2.0000\n", + "\n", + "grad -1.5000\n", + "\n", + "\n", + "\n", + "2379410988688->2379410981968*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410985680\n", + "\n", + "x1*w1 + x2*w2\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "2379410981264+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2379410985680->2379410981264+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410985680+->2379410985680\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377655405264\n", + "\n", + "output\n", + "\n", + "data 0.7071\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377655405264tanh\n", + "\n", + "tanh\n", + "\n", + "\n", + "\n", + "2377655405264tanh->2377655405264\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410981264\n", + "\n", + "n\n", + "\n", + "data 0.8814\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "2379410981264->2377655405264tanh\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410981264+->2379410981264\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410985424\n", + "\n", + "w2\n", + "\n", + "data 1.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "2379410985424->2379410984016*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410985936\n", + "\n", + "b\n", + "\n", + "data 6.8814\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "2379410985936->2379410981264+\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 126, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "draw_dot(o)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "2e570d89-e528-4d41-a691-da62fce06b2e", + "metadata": {}, + "outputs": [], + "source": [ + "o.grad = 1.0" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "35b5853e-489a-43d3-8c41-560f4a876131", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.4999999999999999\n" + ] + } + ], + "source": [ + "# o = tanh(n)\n", + "# do/dn = 1 - tanh(n)**2 = 1 - o**2\n", + "print(1 - o.data**2)\n", + "n.grad = 0.5" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "aa8737b2-6e53-4aa5-bb2c-2221e1dce011", + "metadata": {}, + "outputs": [], + "source": [ + "# a plus is just a distributor of gradient, so n.grad will flow to b and x1*w1 + x2*w2, and x1w1x2w2 will flow to x1w1 & x2w2\n", + "x1w1x2w2.grad = 0.5\n", + "b.grad = 0.5\n", + "x1w1.grad = 0.5\n", + "x2w2.grad = 0.5" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "ff460f77-a807-4fce-93b7-079662202a70", + "metadata": {}, + "outputs": [], + "source": [ + "# a multi is front grad * the other child\n", + "x1.grad = w1.data * x1w1.grad\n", + "w1.grad = x1.data * x1w1.grad\n", + "x2.grad = w2.data * x2w2.grad\n", + "w2.grad = x2.data * x2w2.grad" + ] + }, + { + "cell_type": "markdown", + "id": "d6965ba7-fefa-4c10-aae7-49ee2e9da76d", + "metadata": {}, + "source": [ + "Doing this manually is ridiculous. Let's implement this backprop in our Value object." + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "id": "32f79032-f121-4b86-b77d-dacdbdd01ee4", + "metadata": {}, + "outputs": [], + "source": [ + "o.grad = 1.0 # we need to set the base case for o" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "id": "66038692-d288-4052-9861-4aeeb8a84836", + "metadata": {}, + "outputs": [], + "source": [ + "o._backward()" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "ac1b36fa-1eae-473a-9182-2e18c0d18c1b", + "metadata": {}, + "outputs": [], + "source": [ + "n._backward()\n", + "x1w1x2w2._backward()\n", + "x1w1._backward()\n", + "x2w2._backward()" + ] + }, + { + "cell_type": "markdown", + "id": "aeb2bc8d-76cb-4a0a-8fec-848a5ea6600e", + "metadata": {}, + "source": [ + "Now, we get rid of us doing _backward manually. For this, we use topological sort [[more here](https://en.wikipedia.org/wiki/Topological_sorting)]" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "id": "4adbf2b0-5130-4908-8536-6da47081037a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Value(data=6.881373587019543),\n", + " Value(data=-3.0),\n", + " Value(data=2.0),\n", + " Value(data=-6.0),\n", + " Value(data=0.0),\n", + " Value(data=1.0),\n", + " Value(data=0.0),\n", + " Value(data=-6.0),\n", + " Value(data=0.8813735870195432),\n", + " Value(data=0.7071067811865476)]" + ] + }, + "execution_count": 110, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# imported from mircograd's codebase\n", + "\n", + "topo = []\n", + "visited = set()\n", + "def build_topo(v):\n", + " if v not in visited:\n", + " visited.add(v)\n", + " for child in v._prev:\n", + " build_topo(child)\n", + " topo.append(v)\n", + "build_topo(o)\n", + "topo" + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "id": "d9626671-9a47-41ce-a7d3-3517a3f61888", + "metadata": {}, + "outputs": [], + "source": [ + "o.grad = 1.0\n", + "\n", + "topo = []\n", + "visited = set()\n", + "def build_topo(v):\n", + " if v not in visited:\n", + " visited.add(v)\n", + " for child in v._prev:\n", + " build_topo(child)\n", + " topo.append(v)\n", + "build_topo(o)\n", + "\n", + "for node in reversed(topo):\n", + " node._backward()" + ] + }, + { + "cell_type": "markdown", + "id": "174e74ee-c432-41e6-acd4-a1e8c3ccbf7b", + "metadata": {}, + "source": [ + "Now let's put this inside our Value object" + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "id": "ebcd762e-0b83-4303-a6aa-c60f2b4e24c7", + "metadata": {}, + "outputs": [], + "source": [ + "o.backward()" + ] + }, + { + "cell_type": "markdown", + "id": "3507e421-b28b-486b-8b6b-4126bafe1964", + "metadata": {}, + "source": [ + "We have a bug" + ] + }, + { + "cell_type": "code", + "execution_count": 127, + "id": "82af7380-6803-4d7d-8088-8ee42edd0cdd", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377655700112\n", + "\n", + "b\n", + "\n", + "data 6.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377655700112+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2377655700112+->2377655700112\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654771536\n", + "\n", + "a\n", + "\n", + "data 3.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377654771536->2377655700112+\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 127, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(3.0, label='a')\n", + "b = a + a ; b.label = 'b'\n", + "b.backward()\n", + "draw_dot(b)" + ] + }, + { + "cell_type": "markdown", + "id": "88b4e727-0ab8-448e-8076-45fcb46cbe0a", + "metadata": {}, + "source": [ + "The grad of a should be 2, NOT 1!" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "id": "8d38e9af-126f-4574-9fd7-5ea1e9285d23", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654777360\n", + "\n", + "f\n", + "\n", + "data -6.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377654777360*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2377654777360*->2377654777360\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654827536\n", + "\n", + "b\n", + "\n", + "data 3.0000\n", + "\n", + "grad -2.0000\n", + "\n", + "\n", + "\n", + "2377654825616*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2377654827536->2377654825616*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654692304+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2377654827536->2377654692304+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653712016\n", + "\n", + "a\n", + "\n", + "data -2.0000\n", + "\n", + "grad 3.0000\n", + "\n", + "\n", + "\n", + "2377653712016->2377654825616*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377653712016->2377654692304+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654825616\n", + "\n", + "d\n", + "\n", + "data -6.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377654825616->2377654777360*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654825616*->2377654825616\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654692304\n", + "\n", + "e\n", + "\n", + "data 1.0000\n", + "\n", + "grad -6.0000\n", + "\n", + "\n", + "\n", + "2377654692304->2377654777360*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654692304+->2377654692304\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 129, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(-2.0, label='a')\n", + "b = Value(3.0, label='b')\n", + "d = a * b ; d.label = 'd'\n", + "e = a + b ; e.label = 'e'\n", + "f = d * e ; f.label = 'f'\n", + "\n", + "f.backward()\n", + "\n", + "draw_dot(f)" + ] + }, + { + "cell_type": "markdown", + "id": "6bd645d9-55c0-43fc-8d42-cc047042fa5e", + "metadata": {}, + "source": [ + "For a and b, there should be two leaves with grad -6.0 from e, and two leaves with grad 3.0 and -2.0. But the backprop only shows grad for one of the operations." + ] + }, + { + "cell_type": "markdown", + "id": "a9931101-1a6a-4ea3-8c89-51d2918b508e", + "metadata": {}, + "source": [ + "If you see the chain rule, for multivariate solutions, the gradients add up. Fixing this in our Value object." + ] + }, + { + "cell_type": "code", + "execution_count": 133, + "id": "042f67cb-6a18-484b-809d-43d14ac67b57", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410983504\n", + "\n", + "a\n", + "\n", + "data -2.0000\n", + "\n", + "grad -3.0000\n", + "\n", + "\n", + "\n", + "2379410986704*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2379410983504->2379410986704*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654486480+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "2379410983504->2377654486480+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410986704\n", + "\n", + "d\n", + "\n", + "data -6.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377654489040*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "2379410986704->2377654489040*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410986704*->2379410986704\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410909584\n", + "\n", + "b\n", + "\n", + "data 3.0000\n", + "\n", + "grad -8.0000\n", + "\n", + "\n", + "\n", + "2379410909584->2379410986704*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2379410909584->2377654486480+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654486480\n", + "\n", + "e\n", + "\n", + "data 1.0000\n", + "\n", + "grad -6.0000\n", + "\n", + "\n", + "\n", + "2377654486480->2377654489040*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654486480+->2377654486480\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "2377654489040\n", + "\n", + "f\n", + "\n", + "data -6.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "2377654489040*->2377654489040\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 133, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(-2.0, label='a')\n", + "b = Value(3.0, label='b')\n", + "d = a * b ; d.label = 'd'\n", + "e = a + b ; e.label = 'e'\n", + "f = d * e ; f.label = 'f'\n", + "\n", + "f.backward() # Fixed\n", + "\n", + "draw_dot(f)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/01b_mircograd.ipynb b/notebooks/01b_mircograd.ipynb new file mode 100644 index 0000000..6d2963a --- /dev/null +++ b/notebooks/01b_mircograd.ipynb @@ -0,0 +1,2359 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 142, + "id": "672d74cf-2fad-4493-9e3d-7ac2286dbfee", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "e7295cd8-1f33-40d2-aca5-e226e5b945ba", + "metadata": {}, + "source": [ + "Let's define a function f" + ] + }, + { + "cell_type": "code", + "execution_count": 143, + "id": "07a0fdd9-e9c4-4a28-9201-99429a638573", + "metadata": {}, + "outputs": [], + "source": [ + "def f(x):\n", + " return 3*x**2 - 4*x + 5" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "id": "67093e12-40d4-4abd-a81d-2f7f63881e19", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "20.0" + ] + }, + "execution_count": 144, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "f(3.0)" + ] + }, + { + "cell_type": "markdown", + "id": "ec1297be-8d68-4d59-af98-39e45eb9857b", + "metadata": {}, + "source": [ + "We can also plot it for a range of values" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "id": "0a3f756f-886b-455d-b1f2-53060088dce9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 145, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABC30lEQVR4nO3deXhU5eH28e+ZmezLhADZSELCGvZ9ExfUFFRcUESpuKEFrWBFXAptxfanNW5VXzewtipaEMWKaFUsokKRsAVB9j0QCFkgMNnINjPvH8G0UVSWSc4s9+e6zqWcmUzujFyZ2+c853kMt9vtRkRERMSLWMwOICIiIvJ9KigiIiLidVRQRERExOuooIiIiIjXUUERERERr6OCIiIiIl5HBUVERES8jgqKiIiIeB2b2QHOhMvlIj8/n6ioKAzDMDuOiIiInAK3201ZWRlJSUlYLD89RuKTBSU/P5+UlBSzY4iIiMgZyMvLIzk5+Sef45MFJSoqCqj/AaOjo01OIyIiIqeitLSUlJSUhs/xn+KTBeW7yzrR0dEqKCIiIj7mVKZnaJKsiIiIeB0VFBEREfE6KigiIiLidVRQRERExOuooIiIiIjXUUERERERr6OCIiIiIl5HBUVERES8jgqKiIiIeJ3TLijLli3jiiuuICkpCcMw+OCDDxo97na7mTFjBomJiYSFhZGZmcnOnTsbPaekpIRx48YRHR1NTEwMt99+O+Xl5Wf1g4iIiIj/OO2CUlFRQa9evXjppZdO+viTTz7J888/z6xZs1i1ahURERGMGDGCqqqqhueMGzeOzZs3s3jxYv71r3+xbNkyJk6ceOY/hYiIiPgVw+12u8/4iw2DBQsWMGrUKKB+9CQpKYn77ruP+++/HwCHw0F8fDxvvPEGY8eOZevWrXTt2pU1a9bQv39/ABYtWsRll13GgQMHSEpK+tnvW1pait1ux+FwaC8eERERH3E6n98enYOyd+9eCgoKyMzMbDhnt9sZNGgQ2dnZAGRnZxMTE9NQTgAyMzOxWCysWrXqpK9bXV1NaWlpo6MpbCso5fcLNvLRhvwmeX0RERE5NR4tKAUFBQDEx8c3Oh8fH9/wWEFBAXFxcY0et9lsxMbGNjzn+7KysrDb7Q1HSkqKJ2M3WLK1iDmr9vPGitwmeX0RERE5NT5xF8/06dNxOBwNR15eXpN8nzH9k7FZDHL2HWV7QVmTfA8RERH5eR4tKAkJCQAUFhY2Ol9YWNjwWEJCAkVFRY0er6uro6SkpOE53xcSEkJ0dHSjoynERYWS2aV+9Oft1fub5HuIiIjIz/NoQUlPTychIYElS5Y0nCstLWXVqlUMGTIEgCFDhnDs2DFycnIanvPFF1/gcrkYNGiQJ+OckV8OSgXg/XUHqKp1mpxGREQkMNlO9wvKy8vZtWtXw5/37t3L+vXriY2NJTU1lSlTpvDoo4/SsWNH0tPTeeihh0hKSmq406dLly5ccsklTJgwgVmzZlFbW8vkyZMZO3bsKd3B09TO69CKNjFhHDx2nE82HuKavslmRxIREQk4pz2CsnbtWvr06UOfPn0AmDp1Kn369GHGjBkAPPjgg9x9991MnDiRAQMGUF5ezqJFiwgNDW14jTlz5pCRkcHFF1/MZZddxrnnnstf//pXD/1IZ8diMfjlwPpJuLrMIyIiYo6zWgfFLE29DkpRaRVDHv8Cp8vN4nvPp2N8lMe/h4iISKAxbR0UfxEXHUpml/pbod9e3TR3DImIiMiPU0H5Eb8cWD9Z9p+aLCsiItLsVFB+xHkdW9MmJgzH8Vo+3XTI7DgiIiIBRQXlR1gtBmMHnJgsu0qXeURERJqTCspPGNM/BavFYHVuCbuKtLKsiIhIc1FB+QkJ9lAuytBkWRERkeamgvIzbtBkWRERkWangvIzzu/UmiR7KMcqa/ls88l3WxYRERHPUkH5GVaLwfUD6kdR5q7SyrIiIiLNQQXlFFw3IBmLAav2lrC7uNzsOCIiIn5PBeUUJNrDGibLztP+PCIiIk1OBeUUfbey7Hs5B6iu02RZERGRpqSCcoou6NSaRHsoRytr+WxzodlxRERE/JoKyimyWS1c1/+7lWV1mUdERKQpqaCchusGpGAxIHvPEfZosqyIiEiTUUE5DW1iwhjW+cRk2TVaWVZERKSpqKCcJk2WFRERaXoqKKfpws6tiY8OoaSihn9rsqyIiEiTUEE5TTarheu/myyrNVFERESahArKGbhuQAqGASt2H2Hv4Qqz44iIiPgdFZQzkNwinGGdWgMwb41GUURERDxNBeUMNUyWXXuAmjqXyWlERET8iwrKGbooI464qBCOVNSwaHOB2XFERET8igrKGbJZLYw9MYryj+x9JqcRERHxLyooZ+GGgalYLQarc0vYVlBqdhwRERG/oYJyFhLsoYzoFg/AmxpFERER8RgVlLN00+A0AD745iClVbXmhhEREfETKihnaXC7WDrFR1JZ4+SfOQfMjiMiIuIXVFDOkmEY3DS4LQBvrdyH2+02OZGIiIjvU0HxgKv7JhMZYmNPcQVf7zpidhwRERGfp4LiAZEhNq7p2waAN7NzzQ0jIiLiB1RQPOS7yzyfby3k4LHjJqcRERHxbSooHtIxPorB7WJxueHtVdqfR0RE5GyooHjQzUPSgPoNBKvrnOaGERER8WEqKB70i67xxEeHcLi8hkWbtD+PiIjImVJB8aAgq4UbBtbPRdHKsiIiImdOBcXDfjkwBZvFIGffUTbnO8yOIyIi4pNUUDwsLjqUS7onAPCWRlFERETOiApKE/husuwH6w/iqNT+PCIiIqdLBaUJDEhrQUZCFFW1Lubn5JkdR0RExOeooDQBwzC4aUj9ZNk5q/bjcml/HhERkdOhgtJERvVuQ1SIjb2HK1i+67DZcURERHyKCkoTiQixMbpfMqBbjkVERE6XCkoTuvHE/jxfbCvkwNFKk9OIiIj4DhWUJtQhLpKhHVrictfPRREREZFTo4LSxG4anAbAO2vyqKrV/jwiIiKnQgWliWV2iSPJHkpJRQ2fbDxkdhwRERGfoILSxGxWCzcMSgU0WVZERORUqaA0g+sHpBJkNVifd4yNB7Q/j4iIyM9RQWkGraNCuKxHIgBvZueaG0ZERMQHqKA0k5tPrCy7cEM+R8qrTU4jIiLi3VRQmknf1Bb0SrZTU+dirm45FhER+UkqKM3EMAxuOzcdgDdX7qOmzmVyIhEREe+lgtKMLu2eSHx0CMVl1Xy8Md/sOCIiIl5LBaUZBdss3DwkDYC/L9+L261djkVERE5GBaWZ3TAwlRCbhU0HS1mTe9TsOCIiIl5JBaWZtYgI5pq+9bscv7Z8r8lpREREvJMKigluG5oGwL+3FJBXol2ORUREvk8FxQQd46M4r2MrXG6YvSLX7DgiIiJex+MFxel08tBDD5Genk5YWBjt27fnkUceaTQh1O12M2PGDBITEwkLCyMzM5OdO3d6OopX++6W43fW5FFeXWdyGhEREe/i8YLyxBNPMHPmTF588UW2bt3KE088wZNPPskLL7zQ8Jwnn3yS559/nlmzZrFq1SoiIiIYMWIEVVVVno7jtS7o2Jr2rSMoq65j/to8s+OIiIh4FY8XlBUrVnDVVVcxcuRI0tLSuPbaaxk+fDirV68G6kdPnnvuOf7whz9w1VVX0bNnT958803y8/P54IMPPB3Ha1ksBuOH1o+ivLEiF6dLtxyLiIh8x+MF5ZxzzmHJkiXs2LEDgA0bNrB8+XIuvfRSAPbu3UtBQQGZmZkNX2O32xk0aBDZ2dknfc3q6mpKS0sbHf7gmr5tsIcFse9IJV9sKzI7joiIiNfweEGZNm0aY8eOJSMjg6CgIPr06cOUKVMYN24cAAUFBQDEx8c3+rr4+PiGx74vKysLu93ecKSkpHg6tinCg238cmAqoFuORURE/pfHC8q7777LnDlzmDt3LuvWrWP27Nk8/fTTzJ49+4xfc/r06TgcjoYjL89/5mzcPKQtVotB9p4jbMn3j5EhERGRs+XxgvLAAw80jKL06NGDm266iXvvvZesrCwAEhISACgsLGz0dYWFhQ2PfV9ISAjR0dGNDn+RFBPGZT0SAXjta42iiIiIQBMUlMrKSiyWxi9rtVpxuep3701PTychIYElS5Y0PF5aWsqqVasYMmSIp+P4hO8WbvtwfT7FZdXmhhEREfECHi8oV1xxBX/+85/5+OOPyc3NZcGCBTzzzDNcffXVABiGwZQpU3j00Uf58MMP2bhxIzfffDNJSUmMGjXK03F8Qp/UFvRJjaHG6WLOqn1mxxERETGdzdMv+MILL/DQQw9x1113UVRURFJSEnfccQczZsxoeM6DDz5IRUUFEydO5NixY5x77rksWrSI0NBQT8fxGbcNTefu/d/wj5X7+PWw9oTYrGZHEhERMY3h/t8lXn1EaWkpdrsdh8PhN/NR6pwuzn/yS/IdVTw9phfX9ks2O5KIiIhHnc7nt/bi8RI2q4Wbz0kD4O/L9+KDvVFERMRjVFC8yNgBKYQFWdl6qJSVe0rMjiMiImIaFRQvEhMezOh+bQDdciwiIoFNBcXLfLc/z+dbC9l3pMLkNCIiIuZQQfEy7VtHcmHn1rjd9ZsIioiIBCIVFC9027n1oyjz1x6grKrW5DQiIiLNTwXFC53boRWd4iMpr67j7dX7zY4jIiLS7FRQvJBhGPzq3HYAvLY8l5o6l8mJREREmpcKipe6qk8ScVEhFJRW8eGGfLPjiIiINCsVFC8VYrM2zEX567LduFxauE1ERAKHCooXu2FQKpEhNnYUlvPVjiKz44iIiDQbFRQvFh0axLhBqQDMWrrH5DQiIiLNRwXFy40fmk6Q1WD13hLW7T9qdhwREZFmoYLi5RLsoYzqXb/8/V81iiIiIgFCBcUH3HFB/S3Hn20pYE9xuclpREREmp4Kig/oEBdFZpd43G549T/aRFBERPyfCoqPuPPEKMo/1x2gqKzK5DQiIiJNSwXFR/RPi6Vf2xbU1LmYrU0ERUTEz6mg+JA7zq8fRXkrex/l1XUmpxEREWk6Kig+JLNLPO1aR1BaVcc8bSIoIiJ+TAXFh1gsRsMoyt+X79UmgiIi4rdUUHzMqD5taB0VwiFHFR9pE0EREfFTKig+JsRm5bah9ZsIvrJsN263NhEUERH/o4LigxptIri92Ow4IiIiHqeC4oPsYUHc0LCJ4G6T04iIiHieCoqPGj80jSCrwaq9JXyjTQRFRMTPqKD4qER7GFd9t4ngMm0iKCIi/kUFxYdNPHHL8aLNBew9XGFyGhEREc9RQfFhneKjuDgj7sQmghpFERER/6GC4uPuuKA9AO/lHKC4rNrkNCIiIp6hguLjBqS1oE9qjDYRFBERv6KC4uMMw+CO8+tHUd7MzqW0qtbkRCIiImdPBcUPDO8aT4e4SEqr6ngre5/ZcURERM6aCoofsFgMJl1YP4ry9+V7qaypMzmRiIjI2VFB8RNX9EyibctwSipqmLtqv9lxREREzooKip+wWS3cNax+FOWVZXuoqnWanEhEROTMqaD4kav7JNMmJozismreXZtndhwREZEzpoLiR4JtFu68oH512Vlf7aamzmVyIhERkTOjguJnxvRPIS4qhHxHFe+vO2B2HBERkTOiguJnQoOsDXv0vPzVbuqcGkURERHfo4Lih24YlErLiGD2l1Ty4YZ8s+OIiIicNhUUPxQebOP289IBePHLXThdbpMTiYiInB4VFD910+C22MOC2FNcwaebDpkdR0RE5LSooPipqNAgxg9NA+DFL3bh0iiKiIj4EBUUPzb+nHQiQ2xsKyjj862FZscRERE5ZSoofsweHsTNQ9oC8MIXu3C7NYoiIiK+QQXFz91+bjphQVY2HnTw1Y5is+OIiIicEhUUP9cyMoRxg1IBeGHJTo2iiIiIT1BBCQATz29HsM3Cuv3HyN59xOw4IiIiP0sFJQDERYcydkAKUD8XRURExNupoASIOy5oT5DVIHvPEdbmlpgdR0RE5CepoASINjFhjO6bDGgURUREvJ8KSgC5a1gHrBaDpTuK2ZB3zOw4IiIiP0oFJYCktgznql5JQP0ePSIiIt5KBSXA3HVhBwwDFm8pZEt+qdlxRERETkoFJcB0iIvksh6JADy/ZKfJaURERE5OBSUATbm4I4YBizYXsPGAw+w4IiIiP6CCEoA6xkcxqncbAJ5ZvN3kNCIiIj+kghKg7rm4I1aLwZfbi8nZd9TsOCIiIo00SUE5ePAgN954Iy1btiQsLIwePXqwdu3ahsfdbjczZswgMTGRsLAwMjMz2blT8yGaU1qrCK49sS6KRlFERMTbeLygHD16lKFDhxIUFMSnn37Kli1b+Mtf/kKLFi0anvPkk0/y/PPPM2vWLFatWkVERAQjRoygqqrK03HkJ9x9cQeCrAZf7zrCit2HzY4jIiLSwHB7eHvbadOm8fXXX/Of//znpI+73W6SkpK47777uP/++wFwOBzEx8fzxhtvMHbs2J/9HqWlpdjtdhwOB9HR0Z6MH3BmLNzEm9n76N+2BfPvHIJhGGZHEhERP3U6n98eH0H58MMP6d+/P2PGjCEuLo4+ffrw6quvNjy+d+9eCgoKyMzMbDhnt9sZNGgQ2dnZJ33N6upqSktLGx3iGZMu7ECIzcLafUdZuqPY7DgiIiJAExSUPXv2MHPmTDp27Mhnn33Gr3/9a37zm98we/ZsAAoKCgCIj49v9HXx8fENj31fVlYWdru94UhJSfF07IAVHx3KTYPbAvDM4h14eEBNRETkjHi8oLhcLvr27ctjjz1Gnz59mDhxIhMmTGDWrFln/JrTp0/H4XA0HHl5eR5MLHcOa094sJVvDzhYvKXQ7DgiIiKeLyiJiYl07dq10bkuXbqwf/9+ABISEgAoLGz8QVhYWNjw2PeFhIQQHR3d6BDPaRUZwvihaUD9KIrLpVEUERExl8cLytChQ9m+vfFtqzt27KBt2/rLCOnp6SQkJLBkyZKGx0tLS1m1ahVDhgzxdBw5RRPPa09UqI1tBWV8vPGQ2XFERCTAebyg3HvvvaxcuZLHHnuMXbt2MXfuXP76178yadIkAAzDYMqUKTz66KN8+OGHbNy4kZtvvpmkpCRGjRrl6ThyiuzhQUw4rx0Az36+gzqny+REIiISyDxeUAYMGMCCBQt4++236d69O4888gjPPfcc48aNa3jOgw8+yN13383EiRMZMGAA5eXlLFq0iNDQUE/HkdMwfmgaMeFB7Cmu4IP1+WbHERGRAObxdVCag9ZBaTqzlu7m8U+3kRIbxhf3DSPIqt0QRETEM0xdB0V8281D2tIqMoS8kuPMX3vA7DgiIhKgVFCkkfBgG5MubA/AC1/spKrWaXIiEREJRCoo8gO/HJhKoj2UQ44q3l693+w4IiISgFRQ5AdCg6xMvqgDAC99uZvjNRpFERGR5qWCIic1pl8KKbFhHC6v5s3sXLPjiIhIgFFBkZMKtlm45+JOQP2dPWVVtSYnEhGRQKKCIj9qVO8k2rWO4GhlLa9/nWt2HBERCSAqKPKjbFYL92bWj6K8umwPRytqTE4kIiKBQgVFftLIHol0SYymrLqOF7/cZXYcEREJECoo8pMsFoPpl2YA8GZ2LnkllSYnEhGRQKCCIj/r/E6tOa9jK2qdbp7+9/af/wIREZGzpIIip+S3l2RgGLBwfT4bDzjMjiMiIn5OBUVOSfc2dq7u3QaAxz7Zig/uMSkiIj5EBUVO2dThnQi2Wcjec4SvdhSbHUdERJqA2+1mV1GZ2TFUUOTUJbcIZ/w5aQA8/sk2nC6NooiI+Jt/fXuIXzy7jIcXbjI1hwqKnJa7hnXAHhbE9sIy/rnugNlxRETEg6pqnTyxaBtuN8RGhJiaRQVFTos9PIjJF9ZvJPjMv3doI0ERET8ye0UuB44eJz46hAnnp5uaRQVFTttNQ9rSJiaMgtIqXvt6r9lxRETEA0oqahoW5Lx/eGfCg22m5lFBkdMWGmTlgRGdAZj51W6OlFebnEhERM7W//t8B2VVdXRNjGZ032Sz46igyJm5slcS3ZKiKa+u44UvtAS+iIgv211czpxV+wH4w8guWCyGyYlUUOQMWSwGv7usCwD/WLmP3MMVJicSEZEzlfXJNupcbi7OiOOcDq3MjgOooMhZGNqhFRd0ak2dy81TWgJfRMQnZe8+wudbC7FaDKaf+B9Pb6CCImdl2qX1S+B//O0hvtl/1Ow4IiJyGlwuN3/+ZAsANwxMpUNcpMmJ/ksFRc5Kl/+ZTJX16TYtgS8i4kMWfHOQTQdLiQyxMSWzo9lxGlFBkbM29RedCLFZWL23hC+2FZkdR0RETsHxGmfDDvV3XdielpHmLsz2fSooctaSYsK47dz6BX0e/3QbdU6XyYlEROTn/H35Hg45qmgTE8ZtQ81dlO1kVFDEI349rD0twoPYWVTOezlaAl9ExJsVlVUx86vdADx4SWdCg6wmJ/ohFRTxiOjQIO6+qP765TOLd1BZU2dyIhER+THPLt5JRY2TXsl2ruiZZHack1JBEY+5cXBbUmPDKSqr5tVlWgJfRMQbbS8o4501JxZlu7yrVyzKdjIqKOIxwTYLD15yYgn8pbvIP3bc5EQiIvJ9j32yFZcbLumWwIC0WLPj/CgVFPGokT0SGZgWS1Wti6xPt5kdR0RE/seyHcUs3VFMkNVg2qUZZsf5SSoo4lGGYfDwlV2xGPDRhnxW7y0xO5KIiABOl5vHPtkKwE2D00hrFWFyop+mgiIe1y3JztiBqQD88cPNOF1avE1ExGzz1+axraAMe1gQv7m4g9lxfpYKijSJ+4d3JjrUxpZDpcw7MRlLRETMUVFdx18W7wDg7os6EBMebHKin6eCIk0iNiKYe3/RCYCnP9uOo7LW5EQiIoHrlWV7KC6rJjU2nJuGtDU7zilRQZEmc+PgtnSMi+RoZS3Pfr7D7DgiIgHp4LHj/HVZ/aJs0y7NIMTmfYuynYwKijSZIKuFh6/oBsBbK/exo7DM5EQiIoHnzx9voarWxcC0WC7tnmB2nFOmgiJN6tyOrRjRLR6ny83/fbRFux2LiDSj5TsP88nGAqwWgz9d1Q3D8M5F2U5GBUWa3B9GdiXYZmH5rsP8e0uh2XFERAJCTZ2Lhz/cBMBNg9vSJTHa5ESnRwVFmlxKbDgTz2sHwKMfb6Gq1mlyIhER//f613vZXVxBq8j/3rTgS1RQpFncdWF7EqJDySs5zt/+s8fsOCIifq3AUcXzS3YC8NtLMrCHBZmc6PSpoEizCA+2Mf2y+mWVX/pyN4cc2qdHRKSpPPbJVipqnPRJjWF032Sz45wRFRRpNlf2SmJAWguO1zp5XPv0iIg0iZV7jvDhhnwMAx65qrvX7lb8c1RQpNkYhsHDV3TDMGDh+nzW5mqfHhERT6p1unh44WYAbhiYSvc2dpMTnTkVFGlW3dvYGTsgBYA/fqR9ekREPOmt7H1sLyyjRXgQD4zobHacs6KCIs3u/uGdiQq1selgKe+uzTM7joiIXygqq+LZE/vtPDAiwyf22/kpKijS7FpGhnBvZv0tb099th3Hce3TIyJytp74dDtl1XX0TLZz/YmRal+mgiKmuGlI/T49JRU1/L/Pd5odR0TEp+XsK+Gf6w4A8Kcru2H10Ymx/0sFRUwRZLUw44quALyZncv2Au3TIyJyJpwuNzNOTIy9vn8KfVJbmJzIM1RQxDTndWzNiG7x1Lnc/G7BRlyaMCsictrmrt7P5vxSokNtPHiJb0+M/V8qKGKqP17ZjYhgKzn7jvKOJsyKiJyWkooanv5sOwD3j+hMy8gQkxN5jgqKmCrRHsZ9w+sbf9YnWykuqzY5kYiI73jqs204jtfSJTGaGwammh3Ho1RQxHS3nJNGjzZ2SqvqePTjLWbHERHxCRvyjjFvTf3I8yNXdcNm9a+PdP/6acQnWS0Gj13dA8uJFWaX7Sg2O5KIiFdzudzMWLgJtxuu6dOG/mmxZkfyOBUU8Qo9ku3cck4aAA8t3ERVrdPcQCIiXuydtXlsOOAgMsTGtBMbsfobFRTxGvcN70xCdCj7jlTy4he7zI4jIuKVisqqyPpkKwBTMjsSFxVqcqKmoYIiXiMyxMYfr+wGwCvLdrOjUGujiIh8358+3EJpVR092ti59cTIsz9SQRGvMqJbPJld4ql1uvm91kYREWlk8ZZCPt54CKvFIOuaHn43MfZ/+e9PJj7JMAz+dFU3woOtrMk9yvwcrY0iIgJQVlXLQx9sAuBX56XTvY3d5ERNq8kLyuOPP45hGEyZMqXhXFVVFZMmTaJly5ZERkYyevRoCgsLmzqK+Ig2MWFM/UX9ZoKPfbKNw+VaG0VE5KnPtlNQWkXbluFMubiT2XGaXJMWlDVr1vDKK6/Qs2fPRufvvfdePvroI+bPn8/SpUvJz8/nmmuuacoo4mNuPSeNronROI7X8uePt5odR0TEVDn7Snhr5T4Asq7uQViw1eRETa/JCkp5eTnjxo3j1VdfpUWL/25c5HA4+Pvf/84zzzzDRRddRL9+/Xj99ddZsWIFK1eubKo44mNsVgtZ1/TAMGDBNwdZvvOw2ZFERExRXefkt//ciNsNY/olc06HVmZHahZNVlAmTZrEyJEjyczMbHQ+JyeH2traRuczMjJITU0lOzv7pK9VXV1NaWlpo0P8X6+UGG4e3BaAP3ywUWujiEhAmvnVbnYVldMqMpjfj+xidpxm0yQFZd68eaxbt46srKwfPFZQUEBwcDAxMTGNzsfHx1NQUHDS18vKysJutzccKSkpTRFbvNB9IzoTHx1C7pFKXv5Sa6OISGDZWVjGSyd+9z18RTdiwoNNTtR8PF5Q8vLyuOeee5gzZw6hoZ5ZPGb69Ok4HI6GIy9Pd3YEiujQIP54Rf3aKDOX7mZXkdZGEZHA4HK5mfb+Rmqdbi7OiOPynolmR2pWHi8oOTk5FBUV0bdvX2w2GzabjaVLl/L8889js9mIj4+npqaGY8eONfq6wsJCEhISTvqaISEhREdHNzokcFzSPYGLM+Kodbr53YJNuN1aG0VE/N+c1fvJ2XeUiGArj4zqjmEYZkdqVh4vKBdffDEbN25k/fr1DUf//v0ZN25cw78HBQWxZMmShq/Zvn07+/fvZ8iQIZ6OI37gu7VRwoKsrN5bwvy1B8yOJCLSpA45jvPEp9sAePCSDJJiwkxO1Pxsnn7BqKgounfv3uhcREQELVu2bDh/++23M3XqVGJjY4mOjubuu+9myJAhDB482NNxxE8ktwjn3l905LFPtvHox1s4v1NrEuz+uf+EiAQ2t9vNQx9spry6jj6pMdx44maBQGPKSrLPPvssl19+OaNHj+b8888nISGB999/34wo4kNuG5pOr2Q7pVV1TH//W13qERG/9OmmAj7fWkiQ1eCJ0T2xWgLr0s53DLcP/pYvLS3FbrfjcDg0HyXA7CwsY+Tzy6lxunjy2p5c1193dImI/3BU1pL57FKKy6r5zUUdmDq8s9mRPOp0Pr+1F4/4lI7xUUwdXr/E8yMfbSH/2HGTE4mIeE7Wp1spLqumfesIJl3Uwew4plJBEZ8z4bx29EmNoay6jmnvb9SlHhHxC9m7jzBvTf0yGo+P7kmIzf+Xs/8pKijic6wWg6fH9CLEZmHZjmLeWaN1cUTEt1XVOvndgo0AjBuUyoC0WJMTmU8FRXxS+9aRPDCi/trsox9v5cDRSpMTiYicuScWbWPv4Qrio0P47aUZZsfxCioo4rPGD02nf9sWlFfX8dt/6q4eEfFNK3Yd5vWvcwF4YnRPokODzA3kJVRQxGdZLQZPXtuT0CALX+86wpxV+82OJCJyWhzHa7l//gag/tLOsM5xJifyHioo4tPatY7kwRH1w6GPfbKVvBJd6hER3/GnjzaT76iibctwfndZ4OxUfCpUUMTn3XpOGgPTYqmscfLge9/iculSj4h4v0WbDvH+uoNYDHjmul5EhHh8cXefpoIiPs9iMXhqTE/Cgqxk7znCP1btMzuSiMhPKiqr4ncLNgFw5wXt6ddWd+18nwqK+IW2LSOYfln9pZ6sT7ax70iFyYlERE7O7Xbzu/c3UlJRQ5fEaKZkdjI7kldSQRG/ceOgtgxp15LjtU4e0KUeEfFS89ce4POtRQRbLTx7fS+CbfooPhm9K+I3LCfu6gkPtrJ6bwmzs3PNjiQi0kheSSV/+mgzAPcN70RGgvaT+zEqKOJXUmL/OxP+u4WPRES8gdPl5r53N1BR42RAWgt+dV47syN5NRUU8TvjBqVybodWVNW6eGD+Bpy61CMiXuC15XtZnVtCeLCVv4zpjdVimB3Jq6mgiN8xDIPHR/cgMsTG2n1HmbV0t9mRRCTAbS8o46nPtgPw0OVdSW0ZbnIi76eCIn4puUU4D1/RFYBnFu9g3f6jJicSkUBVU+fi3nfWU+N0cVFGHGMHpJgdySeooIjfurZfMlf2SsLpcvObt7+htKrW7EgiEoCeX7KTLYdKaREexOOje2AYurRzKlRQxG8ZhsGjV3cnJTaMA0eP87v3N2pDQRFpVjn7jvLyV7sA+PPVPYiLCjU5ke9QQRG/Fh0axPNj+2CzGPzr20PMzzlgdiQRCRCVNXXc9+56XG64uk8bLuuRaHYkn6KCIn6vT2oLpg6vX6nx4YWb2V1cbnIiEQkEj368ldwjlSREh/LHK7uZHcfnqKBIQLjz/PYM7VC/yuzdc7+hus5pdiQR8WP/+jafuav2A/D0mF7Yw4JMTuR7VFAkIFgsBs9c15vYiGC2HCrliU+3mx1JRPxU7uEKpv1zIwB3DWvPuR1bmZzIN6mgSMCIjw7l6TE9AXjt6718sa3Q5EQi4m+q65xMfnsd5dV1DEhrwdRfaCPAM6WCIgHloox4bj0nDYD7539LUWmVuYFExK889vFWNh2sv6X4+V/2wWbVx+yZ0jsnAWfapRl0SYympKKGe99dr12PRcQjPt14iNnZ+wB45rreJNrDTE7k21RQJOCEBll54Zd9CAuy8vWuI7yybI/ZkUTEx+0/UsmD730LwB0XtOPCjDiTE/k+FRQJSB3iIvnjlfVL4f/l39v5Rkvhi8gZ+m7eSVl1Hf3atuD+4Z3NjuQXVFAkYF3XP4WRPROpc7n5zTwthS8iZ+bxT7fx7QEH9rD6eSdBmnfiEXoXJWAZhsFjV/egTUwYeSXH+cOCTVoKX0ROy2ebC3j961wA/jKmF21iNO/EU1RQJKDV/x9Pb6wWgw835POelsIXkVOUV1LJA/M3ADDhvHQyu8abnMi/qKBIwOvXNpZ7MzsC8NDCTWw9VGpyIhHxdjV1Lu5++xtKq+ronRLDg5dkmB3J76igiAC/HtaB8zq2oqrWxR1v5XCsssbsSCLixZ5ctI31eceIDrXx4g2ad9IU9I6KAFaLwfNj+5DcIoz9JZXcM289Tq2PIiIn8fmWQv62fC9Qv89OcotwkxP5JxUUkRNaRATzyk39CLFZWLqjmOc+32F2JBHxMgePHee+E/NObhuazvBuCSYn8l8qKCL/o1uSncdH9wDghS928dnmApMTiYi3qHW6uHvuOhzHa+mVbGfapZp30pRUUES+5+o+yQ379dz37gZ2F5ebG0hEvMIj/9rCuv3HiAq18eINfQm26SO0KendFTmJ34/swsC0WMqr67jjrRzKq+vMjiQiJnp79X7ezN6HYcCz1/UmJVbzTpqaCorISQRZLbw4rg/x0SHsKirn/nc3aBE3kQC1JreEGQs3AXDfLzppvZNmooIi8iPiokKZeWM/gqwGizYXMHPpbrMjiUgzO3jsOHe+lUOt083InolMurCD2ZEChgqKyE/om9qCP17ZDYCnP9vOsh3FJicSkeZyvMbJxDfXcqSihq6J0Tx1bU8MwzA7VsBQQRH5GTcMTOX6/im43PCbed+QV1JpdiQRaWJut5sH3tvA5vxSWkYE8+ot/QkPtpkdK6CooIj8DMMw+NNV3eiVbOdYZS13vJXD8Rqn2bFEpAm9/NVu/vXtIWwWg5k39tMmgCZQQRE5BaFBVmbe2I+WEcFsOVTK7xds1KRZET/1+ZZCnv73dgD+dFU3BqbHmpwoMKmgiJyipJgwXrihD1aLwfvfHGT2ilyzI4mIh+0sLGPKO+txu+HGwamMG9TW7EgBSwVF5DSc074V00+sHvnox1vJ3n3E5EQi4imOylomvLmW8uo6BqXH8vAV3cyOFNBUUERO0+3npnNlryTqXG7ueGstu4rKzI4kImepzuli8tvryD1SSZuYMF4e11c7FJtM777IaTIMgyev7Unf1BhKq+q49fU1FJdVmx1LRM7C459u4z87DxMWZOXVm/vTMjLE7EgBTwVF5AyEnvgl1rZlOAeOHudXs9dQWaPl8EV80T9zDvC35XsB+Mt1veiaFG1yIgEVFJEz1jIyhDfGD6RFeBAbDjj4zdvrcbp0Z4+IL/lm/1GmL9gIwG8u6sBlPRJNTiTfUUEROQvprSJ49eb+BNssfL61kEf+tcXsSCJyivYfqWTCmznU1LkY3jWeKZmdzI4k/0MFReQs9U+L5ZnregHwxopc/n5iqFhEvNfh8mpufm0Vh8ur6ZIYzTPX98Zi0TL23kQFRcQDLu+ZxLSG24+3sGhTgcmJROTHVFTXcfsba8g9UklyizBmjx9AZIiWsfc2KigiHnLH+e0YNygVtxvumfcN3+w/anYkEfmeWqeLX89Zx4YDDmIjgnnztoHERYeaHUtOQgVFxEMMw+BPV3bjws6tqa5z8avZa9l/RBsLingLt9vNb9/7lmU7igkLsvL3W/rTrnWk2bHkR6igiHiQzWrhxRv60i0pmiMVNdz6xmqOVdaYHUtEgCcWbef9bw5itRi8PK4vfVJbmB1JfoIKioiHRYTYeO3WASTZQ9lTXMHEN3OortPuxyJmem35XmYt3Q3A49f04MKMOJMTyc9RQRFpAvHRobw+fiBRITZW55bwwPxvcWmNFBFTfLQhn0c+rl8C4IERnRnTP8XkRHIqVFBEmkjnhChm3tgPm8Xgww35PHVi+3YRaT4rdh3mvnc34HbDLUPactew9mZHklPk8YKSlZXFgAEDiIqKIi4ujlGjRrF9e+NfzFVVVUyaNImWLVsSGRnJ6NGjKSws9HQUEdOd27EVWdf0AGDmV7uZ+dVukxOJBI7N+Q4mvpVDjdPFZT0SmHFFNwxDa534Co8XlKVLlzJp0iRWrlzJ4sWLqa2tZfjw4VRUVDQ859577+Wjjz5i/vz5LF26lPz8fK655hpPRxHxCmP6p/DgJZ0BeGLRNl7TQm4iTS6vpJJbX19DeXUdg9Jjeea63li1EJtPMdxud5NeGC8uLiYuLo6lS5dy/vnn43A4aN26NXPnzuXaa68FYNu2bXTp0oXs7GwGDx78s69ZWlqK3W7H4XAQHa1NncQ3PPPv7Tz/xS4AHru6BzcMSjU5kYh/OlJezZhZ2ew5XEFGQhTv3jmE6NAgs2MJp/f53eRzUBwOBwCxsbEA5OTkUFtbS2ZmZsNzMjIySE1NJTs7u6njiJjm3l90YuL57QD4/QcbeX/dAZMTififypo6bpu9lj2HK2gTE8bs2waqnPioJl3b1+VyMWXKFIYOHUr37t0BKCgoIDg4mJiYmEbPjY+Pp6Dg5MuDV1dXU11d3fDn0tLSJsss0lQMw2D6pRlU1zqZnb2P++dvINhm4fKeSWZHE/ELlTV1jH99DRvyjhETHsTs2wYSr1VifVaTjqBMmjSJTZs2MW/evLN6naysLOx2e8ORkqJbxMQ3GYbBw1d0Y+yAFFxumDJvPf/erH17RM7Wd+Vk1d4SokJsvH7rADrEaZVYX9ZkBWXy5Mn861//4ssvvyQ5ObnhfEJCAjU1NRw7dqzR8wsLC0lISDjpa02fPh2Hw9Fw5OXlNVVskSZnsRj8+eoejOqdRJ3LzeS537B0R7HZsUR81vfLyezbB2qVWD/g8YLidruZPHkyCxYs4IsvviA9Pb3R4/369SMoKIglS5Y0nNu+fTv79+9nyJAhJ33NkJAQoqOjGx0ivsxqMXh6TC8u7Z5AjdPFxDfXkr37iNmxRHzOycpJX5UTv+DxgjJp0iT+8Y9/MHfuXKKioigoKKCgoIDjx48DYLfbuf3225k6dSpffvklOTk5jB8/niFDhpzSHTwi/sJmtfD/xvbh4ow4qutc3D57DTn7SsyOJeIzVE78m8dvM/6xRXBef/11br31VqB+obb77ruPt99+m+rqakaMGMHLL7/8o5d4vk+3GYs/qap1MuHNtfxn52GiQmzMmTCInskxZscS8WoqJ77pdD6/m3wdlKaggiL+5niNk1teX83qvSXYw4KYN3EwXRL1d1vkZFROfJdXrYMiIj8vLNjKa7cOoHdKDI7jtdz4t1XsLCwzO5aI11E5CRwqKCJeIjLExuzbBtItKZojFTVc90o26/OOmR1LxGuonAQWFRQRL2IPC+Iftw+iV0oMRytrueHVlXy967DZsURMp3ISeFRQRLxMi4hg5vxqEEM7tKSyxsn419ewaNMhs2OJmEblJDCpoIh4ocgQG6/dOqBhnZS75qxj3ur9ZscSaXaO47Xc+prKSSBSQRHxUiE2Ky/e0LdhWfxp729k1tLdZscSaTb5x44zZtYKVueqnAQiFRQRL2a1GGRd04M7L2gPwOOfbiPrk6344OoAIqdlW0Ep17y8gh2F5cRHh/DOHUNUTgKMCoqIlzMMg2mXZjD90gwAXlm2h2n/3Eid02VyMpGmsWLXYcbMzKagtIqOcZG8f9dQuiZpXaBAo4Ii4iPuuKA9T47uicWAd9bmMXnuN1TVOs2OJeJRC9cf5JbXV1NWXcfA9Fjeu/Mc2sSEmR1LTKCCIuJDrhuQwsvj+hFstbBocwG3vbGG8uo6s2OJnDW3280rS3dzz7z11DrdjOyZyJu3DcQeHmR2NDGJCoqIj7mkewJv3DaAiGArK3Yf4YZXV1JSUWN2LJEz5nS5+dNHW8j6dBsAt5+bzgtj+xAaZDU5mZhJBUXEB53TvhVvTxxMbEQw3x5wcO2sFew9XGF2LJHTVlXrZNKcdbyxIheAP4zswkOXd8ViOfnGsxI4VFBEfFTP5BjevWMISfZQ9hRXcNWLy1m6o9jsWCKn7GhFDTf+bRWLNhcQbLXw4g19+NV57cyOJV5CBUXEh3WIi+SDyUPp17YFpVV1jH99Na8s3a3bkMXr5ZVUMnrWCtbuO0p0qI03bx/I5T2TzI4lXkQFRcTHxUWFMnfCIK7vX7+gW9an25jyznrd4SNea0PeMa6ZuYI9xRUk2UN579fnMLhdS7NjiZdRQRHxAyE2K4+P7sH/XdUNm8Vg4fp8rp21goPHjpsdTaSB2+1mzqp9jJmVTXFZNRkJUbx/11A6xUeZHU28kAqKiJ8wDIObh6Tx1u2DiI0IZtPBUq56cTlrckvMjibC8Ron983fwO8XbKLG6WJ413jevXMICfZQs6OJl1JBEfEzQ9q35MPJQ+mSGM3h8hpueHUlc1btMzuWBLC9hyu4+uWveX/dQawWg+mXZvDKTf2IDtUaJ/LjVFBE/FByi3D++eshjOyZSK3Tze8XbOL3CzZSU6fl8aV5fba5gCtfWM62gjJaRYYw51eDuOOC9hiGbiOWn6aCIuKnwoNtvPjLPjwwojOGAXNW7Wfc31ZSXFZtdjQJAHVOF1mfbuWOt3Ioq65jQFoLPvnNuZoMK6dMBUXEjxmGwaQLO/D3W/oTFWJjTe5RrnxxOevzjpkdTfxYUVkV4/62ileW7gFgwnnpzJ0wmLhozTeRU6eCIhIALsqIZ8GkobRrFcEhRxWjZ67g/32+Uzsii8etyS3h8ueXs2pvCZEhNl4e15ffj+xKkFUfN3J69DdGJEB0iItkwaShjOyZiNPl5tnPdzDmlWxytUS+eIDb7eZv/9nD2L+upKismk7xkSycPJTLeiSaHU18lAqKSACxhwXx4i/78Nz1vYkKtfHN/mNc9vx/eHv1fq0+K2espKKGX/9jHY9+vBWny81VvZP4YNJQ2reONDua+DDD7YO/lUpLS7Hb7TgcDqKjo82OI+KTDh47zn3vrmflnvp1UjK7xJF1TU9aR4WYnEx8ycffHmLGwk0cqaghyGow4/Ku3Di4re7SkZM6nc9vFRSRAOZyufn78r089dl2apwuWkYE8/jonvyia7zZ0cTLFZdVM2PhJj7dVABA5/gonh7Tix7JdpOTiTdTQRGR07L1UCn3vrOebQVlAIwdkMJDl3clIsRmcjLxNm63m4Xr8/njR5s5VlmLzWJw14UdmHxhB4JtmjUgP00FRUROW1Wtk2cW7+DV/+zB7Ya2LcN55rre9Gvbwuxo4iWKSqv43YJNfL61EICuidE8NaYn3ZI0aiKnRgVFRM5Y9u4j3PfuevIdVVgMuGtYByZf1IHQIKvZ0cQkbrebf647yP99tJnSqjqCrAa/uagjdw5rr9uH5bSooIjIWXEcr+WPH25mwTcHAUiJDeMPI7syvGu8Jj8GmEOO40x/fyNfbS8GoGeynaeu7UXnBO1ALKdPBUVEPOKTjYf4v4+2UFBaBcC5HVrx8BVd6RivDyd/53a7mbcmj8c+3kpZdR3BNgv3ZnZiwnnp2DRqImdIBUVEPKaiuo6ZX+3mr8v2UON0YbUY3DIkjXsyO2IP0260/ihnXwmPfbKNnH1HAeiTGsNT1/akQ5yKqZwdFRQR8bh9Ryp49OOtLN5SP0GyZUQwD4zozJj+KVgtuuzjD/YUl/Pkou0s2lx/63BokIX7h3dm/NB0/TcWj1BBEZEms2xHMX/6aDO7i+uXyO/Rxs4fr+xKv7axJieTM1VcVs3zS3Yyd/V+nC43FgOu65/Cvb/oRLw2+BMPUkERkSZV63TxZvY+nlu8g7LqOgCu7tOGaZdm6APNh1TW1PG3/+zllaW7qahxAnBxRhy/vTSDTppnJE1ABUVEmsXh8mqeWrSdd3PycLshPNjKhPPaces5abSICDY7nvyIOqeL+TkHeHbxDorKqgHolWxn+mVdGNyupcnpxJ+poIhIs/r2wDH++OFm1u0/BtQXlV8OTOVX56WTaA8zN5w0cLvdLNlaxOOLtrGrqByA1NhwHrykMyN7JOoWcmlyKigi0uzcbjefbCzgpS93seVQKQBBVoNr+iRzxwXtaKedbU3jdLn5YlsRry7bw+rc+s0hW4QHcfdFHRk3OJUQmxbhk+ahgiIipnG73SzdUczMr3azam/9h6FhwKXdE/j1BR20mVwzKquq5d21B5i9Ipf9JZUAhNgs3HZuOnde0F63iUuzU0EREa+Qs+8oM7/axedbixrOndexFb8e1p4h7VrqkkIT2Xu4gtkrcpm/Nq9h8mt0qI1fDkzl1qFpuuwmplFBERGvsq2glFeW7uHDDfk4XfW/cnqnxHDnBe3J7BKnlUk9wO12s3zXYV7/Opcvtxfx3W/2DnGR3HpOGtf0bUN4sHanFnOpoIiIV8orqeSvy/bw7to8qutcQP2Cb1f0SmJUnzb0SrZrVOU0Ha9x8v43B3jj61x2npj4CnBh59aMH5rOeR1b6T0Vr6GCIiJerbismte/3ss7a/I4UlHTcD69VQRX9U5iVO82pLWKMDGhd6tzulidW8KiTQUsXJ+P43gtABHBVq7tl8wt56RpUrJ4JRUUEfEJtU4Xy3cd5oNvDvLZ5gKqal0Nj/VJjWFU7zZc3jORlpEhJqb0DtV1Tr7edZhFmwpYvKWQo5W1DY+lxIZxy5A0rhuQQnSoJr6K91JBERGfU15dx783F/DB+nyW7yzmxFQVrBaD8zu2YlSfNmR2iSciJHDmUVRU1/HV9mIWbS7gy21FlJ9YtRfqbxP+Rdd4Lu2eyPmdWmuvHPEJKigi4tOKyqr4aMMhFq4/yLcHHA3nbRaD7m3sDGoXy+D0lvRLa+F3IwbHKmtYsrWIRZsLWLajuGGuDkB8dAiXdEtgRPcEBqbFanKx+BwVFBHxG7uKylm4/iAfbshn35HKRo9ZDOiaFM2g9JYMSo9lYHosMeG+s8R+dZ2TrYfK+PbAMTbkOfj2wDF2FZfzv7+V27YM55LuCVzSLYFeyTFYNFIiPkwFRUT8Ul5JJav2lrB67xFW7S35QWEByEiIYlB6LAPSY2nfOpKU2HAiveCykNPlZldRORvyjrHhwDG+PeBgW0Eptc4f/gruHB/FJd0TuLRHAp3jo3QXjvgNFRQRCQgFjipWnSgrq/YcYXdxxUmfFxsRTEpsOKmx4aTGhpEaG05Ki3BSYsNJtId65FLJ8RonxWXVFJdX1f/zu6O8mt1FFWzKd1B5YtG072frmWynZ3IMvU78s3WUJgWLf1JBEZGAVFxWzeoTIyzr846xv6Sy0d0uJ2OzGLRpEUZsRDBBFgs2q4HNasFmMbBZDIKsJ85ZTpyzGlgMg5LKGorLqjl8ooiU/c8E1h8TEWylexs7vVJi6Jlsp1dyDMktwjRCIgFDBUVE5ITSqlrySirJKzlOXkkl+08ceSWVHDh6nBqn6+df5BSF2CzERYfQOjKE1lEnjshQkluE0TPZTrvWkbrbRgLa6Xx+m39hVkSkCUWHBtEtyU63pB9uUuhyuSksq2LfkUpKj9dS53JT63RR53RT53JR63Tj/O6cy02ds/6cy+2mRXjwf0vIiSMqxKbREBEPUUERkYBlsRgk2sO0eZ6IF9JN9CIiIuJ1VFBERETE66igiIiIiNdRQRERERGvo4IiIiIiXkcFRURERLyOqQXlpZdeIi0tjdDQUAYNGsTq1avNjCMiIiJewrSC8s477zB16lQefvhh1q1bR69evRgxYgRFRUVmRRIREREvYVpBeeaZZ5gwYQLjx4+na9euzJo1i/DwcF577TWzIomIiIiXMKWg1NTUkJOTQ2Zm5n+DWCxkZmaSnZ39g+dXV1dTWlra6BARERH/ZUpBOXz4ME6nk/j4+Ebn4+PjKSgo+MHzs7KysNvtDUdKSkpzRRURERET+MRdPNOnT8fhcDQceXl5ZkcSERGRJmTKZoGtWrXCarVSWFjY6HxhYSEJCQk/eH5ISAghISHNFU9ERERMZkpBCQ4Opl+/fixZsoRRo0YB4HK5WLJkCZMnT/7Zr3e73QCaiyIiIuJDvvvc/u5z/KeYUlAApk6dyi233EL//v0ZOHAgzz33HBUVFYwfP/5nv7asrAxAc1FERER8UFlZGXa7/SefY1pBuf766ykuLmbGjBkUFBTQu3dvFi1a9IOJsyeTlJREXl4eUVFRGIbRDGm9X2lpKSkpKeTl5REdHW12HL+n97v56T1vXnq/m18gvOdut5uysjKSkpJ+9rmG+1TGWcTrlZaWYrfbcTgcfvsX25vo/W5+es+bl97v5qf3vDGfuItHREREAosKioiIiHgdFRQ/ERISwsMPP6zbsZuJ3u/mp/e8een9bn56zxvTHBQRERHxOhpBEREREa+jgiIiIiJeRwVFREREvI4KioiIiHgdFRQ/Vl1dTe/evTEMg/Xr15sdx2/l5uZy++23k56eTlhYGO3bt+fhhx+mpqbG7Gh+46WXXiItLY3Q0FAGDRrE6tWrzY7kt7KyshgwYABRUVHExcUxatQotm/fbnasgPH4449jGAZTpkwxO4rpVFD82IMPPnhKywnL2dm2bRsul4tXXnmFzZs38+yzzzJr1ix+97vfmR3NL7zzzjtMnTqVhx9+mHXr1tGrVy9GjBhBUVGR2dH80tKlS5k0aRIrV65k8eLF1NbWMnz4cCoqKsyO5vfWrFnDK6+8Qs+ePc2O4h3c4pc++eQTd0ZGhnvz5s1uwP3NN9+YHSmgPPnkk+709HSzY/iFgQMHuidNmtTwZ6fT6U5KSnJnZWWZmCpwFBUVuQH30qVLzY7i18rKytwdO3Z0L1682H3BBRe477nnHrMjmU4jKH6osLCQCRMm8NZbbxEeHm52nIDkcDiIjY01O4bPq6mpIScnh8zMzIZzFouFzMxMsrOzTUwWOBwOB4D+PjexSZMmMXLkyEZ/1wOdabsZS9Nwu93ceuut3HnnnfTv35/c3FyzIwWcXbt28cILL/D000+bHcXnHT58GKfT+YNdzuPj49m2bZtJqQKHy+ViypQpDB06lO7du5sdx2/NmzePdevWsWbNGrOjeBWNoPiIadOmYRjGTx7btm3jhRdeoKysjOnTp5sd2eed6nv+vw4ePMgll1zCmDFjmDBhgknJRTxj0qRJbNq0iXnz5pkdxW/l5eVxzz33MGfOHEJDQ82O41W01L2PKC4u5siRIz/5nHbt2nHdddfx0UcfYRhGw3mn04nVamXcuHHMnj27qaP6jVN9z4ODgwHIz89n2LBhDB48mDfeeAOLRf3/bNXU1BAeHs57773HqFGjGs7fcsstHDt2jIULF5oXzs9NnjyZhQsXsmzZMtLT082O47c++OADrr76aqxWa8M5p9OJYRhYLBaqq6sbPRZIVFD8zP79+yktLW34c35+PiNGjOC9995j0KBBJCcnm5jOfx08eJALL7yQfv368Y9//CNgf6E0hUGDBjFw4EBeeOEFoP6yQ2pqKpMnT2batGkmp/M/brebu+++mwULFvDVV1/RsWNHsyP5tbKyMvbt29fo3Pjx48nIyOC3v/1tQF9a0xwUP5Oamtroz5GRkQC0b99e5aSJHDx4kGHDhtG2bVuefvppiouLGx5LSEgwMZl/mDp1Krfccgv9+/dn4MCBPPfcc1RUVDB+/Hizo/mlSZMmMXfuXBYuXEhUVBQFBQUA2O12wsLCTE7nf6Kion5QQiIiImjZsmVAlxNQQRE5a4sXL2bXrl3s2rXrByVQA5Rn7/rrr6e4uJgZM2ZQUFBA7969WbRo0Q8mzopnzJw5E4Bhw4Y1Ov/6669z6623Nn8gCVi6xCMiIiJeR7P4RERExOuooIiIiIjXUUERERERr6OCIiIiIl5HBUVERES8jgqKiIiIeB0VFBEREfE6KigiIiLidVRQRERExOuooIiIiIjXUUERERERr6OCIiIiIl7n/wOmIpCi+M1VdAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "xs = np.arange(-5, 5, 0.25)\n", + "ys = f(xs)\n", + "plt.plot(xs, ys)" + ] + }, + { + "cell_type": "markdown", + "id": "5c9cb820-159e-420d-b0d0-a8706e19f9d5", + "metadata": {}, + "source": [ + "Now, what's a derivate?\n", + "> It is sensitivity of the function to the change of the output with respect to the input.\n", + "\n", + "In simpler terms, (f(x+h) - f(x))/h, where h tends to zero. This gives us the slope." + ] + }, + { + "cell_type": "code", + "execution_count": 146, + "id": "7866b622-e10c-4003-844f-9044e7bdc080", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "3.0000002482211127e-05" + ] + }, + "execution_count": 146, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "h = 0.00001\n", + "x = 2/3\n", + "(f(x+h) - f(x))/h" + ] + }, + { + "cell_type": "markdown", + "id": "4f027e6d-d889-4827-867e-335a96c20045", + "metadata": {}, + "source": [ + "Let's define a function with mupltiple inputs" + ] + }, + { + "cell_type": "code", + "execution_count": 147, + "id": "a3bcd24a-4a48-4d85-8d49-3fb751589b40", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4.0\n" + ] + } + ], + "source": [ + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d = a*b + c\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 148, + "id": "78dcb378-190f-4421-8d95-af79cdf05b18", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "d1 4.0\n", + "d2 3.999699999999999\n", + "slope -3.000000000010772\n" + ] + } + ], + "source": [ + "h = 0.0001\n", + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d1 = a*b + c\n", + "a += h\n", + "d2 = a*b + c\n", + "\n", + "print('d1', d1)\n", + "print('d2', d2)\n", + "print('slope', (d2 - d1)/h) # By the good old derivation (wrt to a), we know that this will be b" + ] + }, + { + "cell_type": "code", + "execution_count": 149, + "id": "9fb1208f-1a1e-4fdb-a290-a7d0e8a5050a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "d1 4.0\n", + "d2 4.0002\n", + "slope 2.0000000000042206\n" + ] + } + ], + "source": [ + "h = 0.0001\n", + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d1 = a*b + c\n", + "b += h\n", + "d2 = a*b + c\n", + "\n", + "print('d1', d1)\n", + "print('d2', d2)\n", + "print('slope', (d2 - d1)/h) # By the good old derivation (wrt to b), we know that this will be a" + ] + }, + { + "cell_type": "code", + "execution_count": 150, + "id": "e7bfb7ef-4c16-44c6-afce-ac20ab858b4f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "d1 4.0\n", + "d2 4.0001\n", + "slope 0.9999999999976694\n" + ] + } + ], + "source": [ + "h = 0.0001\n", + "a = 2.0\n", + "b = -3.0\n", + "c = 10.0\n", + "\n", + "d1 = a*b + c\n", + "c += h\n", + "d2 = a*b + c\n", + "\n", + "print('d1', d1)\n", + "print('d2', d2)\n", + "print('slope', (d2 - d1)/h) # By the good old derivation (wrt to c), we know that this will be 1" + ] + }, + { + "cell_type": "markdown", + "id": "ad332637-3dbe-4630-bb5a-d5a76e935a9b", + "metadata": {}, + "source": [ + "The NN will be mathematically very large expressions. We now start by building the data structures for this. Let's start by making the `Value` object from mircograd" + ] + }, + { + "cell_type": "code", + "execution_count": 167, + "id": "e11f7575-00be-44a4-9bde-ae9288b1922a", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "class Value:\n", + "\n", + " def __init__(self, data, _children=(), _op='', label=''): # Define a empty tuple `children` to keep the pointers of other Value objects\n", + " # Define a empty set `op` to keep track of what created that Value object\n", + " self.data = data\n", + " self.grad = 0.0 # this is the derivative of Value wrt to its nodes; initalized as 0 (we assume at the beginning that every Value doesn't impact the output)\n", + " self._backward = lambda: None # for a leaf node, this should be nothing.\n", + " self._prev = set(_children) # This will be a empty set when we define a new Value object (a, b, c)\n", + " self._op = _op\n", + " self.label = label\n", + "\n", + " def __repr__(self):\n", + " return f\"Value(data={self.data})\"\n", + "\n", + " def __add__(self, other): # a.__add__(b)\n", + " other = other if isinstance(other, Value) else Value(other) # if other is NOT a Value obj, just make one\n", + " out = Value(self.data + other.data, (self, other), '+') # Since self.data and other.data is python floating point number,\n", + " # the addition is according to whatever is defined in the python kernel\n", + " def _backward():\n", + " self.grad += 1.0 * out.grad\n", + " other.grad += 1.0 * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + "\n", + " def __radd__(self, other): # other + self\n", + " return self + other\n", + " \n", + " def __mul__(self, other): # a.__mul__(b) # You can't name it mult or multi, because we are defining magic methods for the Value object\n", + " other = other if isinstance(other, Value) else Value(other) # if other is NOT a Value obj, just make one\n", + " out = Value(self.data * other.data, (self, other), '*')\n", + "\n", + " def _backward():\n", + " self.grad += other.data * out.grad\n", + " other.grad += self.data * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + "\n", + " def __pow__(self, other):\n", + " assert isinstance(other, (int, float)) # other is only int or float\n", + " out = Value(self.data**other, (self, ), f'**{other}')\n", + "\n", + " def _backward():\n", + " self.grad += other * self.data**(other-1) * out.grad\n", + " out._backward = _backward\n", + "\n", + " return out\n", + "\n", + " def __rmul__(self, other): # other * self\n", + " return self*other\n", + "\n", + " def __truediv__(self, other): # self / other\n", + " return self * other**-1\n", + "\n", + " def __neg__(self): # -self\n", + " return self * -1\n", + "\n", + " def __sub__(self, other): # self - other\n", + " return self + (-other)\n", + " \n", + " def tanh(self):\n", + " x = self.data\n", + " t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n", + " out = Value(t, (self, ), 'tanh')\n", + "\n", + " def _backward():\n", + " self.grad += (1 - t**2) * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + "\n", + " def exp(self):\n", + " x = self.data\n", + " out = Value(math.exp(x), (self, ), 'exp')\n", + "\n", + " def _backward():\n", + " self.grad += out.data * out.grad\n", + " out._backward = _backward\n", + " \n", + " return out\n", + " \n", + " def backward(self):\n", + " \n", + " topo = []\n", + " visited = set()\n", + " def build_topo(v):\n", + " if v not in visited:\n", + " visited.add(v)\n", + " for child in v._prev:\n", + " build_topo(child)\n", + " topo.append(v)\n", + " build_topo(self)\n", + " \n", + " self.grad = 1.0\n", + " \n", + " for node in reversed(topo):\n", + " node._backward()" + ] + }, + { + "cell_type": "code", + "execution_count": 152, + "id": "cdf24235-ee48-4675-bb65-bd78d1ae201e", + "metadata": {}, + "outputs": [], + "source": [ + "# imported from mircograd's codebase\n", + "\n", + "from graphviz import Digraph\n", + "\n", + "def trace(root):\n", + " # builds a set of all nodes and edges in a graph\n", + " nodes, edges = set(), set()\n", + " def build(v):\n", + " if v not in nodes:\n", + " nodes.add(v)\n", + " for child in v._prev:\n", + " edges.add((child, v))\n", + " build(child)\n", + " build(root)\n", + " return nodes, edges\n", + "\n", + "def draw_dot(root):\n", + " dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n", + " \n", + " nodes, edges = trace(root)\n", + " for n in nodes:\n", + " uid = str(id(n))\n", + " # for any value in the graph, create a rectangular ('record') node for it\n", + " dot.node(name = uid, label = \"{%s | data %.4f | grad %.4f}\" % (n.label, n.data, n.grad), shape='record')\n", + " if n._op:\n", + " # if this value is a result of some operation, create an op node for it\n", + " dot.node(name = uid + n._op, label = n._op)\n", + " # and connect this node to it\n", + " dot.edge(uid + n._op, uid)\n", + "\n", + " for n1, n2 in edges:\n", + " # connect n1 to the op node of n2\n", + " dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n", + "\n", + " return dot" + ] + }, + { + "attachments": { + "34f0ebc5-cb10-4db9-b7a9-9f501a3732da.webp": { + "image/webp": "UklGRqA9AABXRUJQVlA4WAoAAAAIAAAAkgIAdwEAVlA4IMA8AACQ6wCdASqTAngBPnU4mEkkoyKhI9KZuJAOiWVu/HyZcejLHpANyu8/6D+1/jt75nKfcj6UyidrHYflo8+/937tPnH/lP+f/mfcn+m/+57gH6r+ev6kf26/WX/S/AP+lf5r9uvdf/2v/f/2Hua/q3+s/KP5Bv69/jf/H7YP+59iD/A/8b/2e4h/OP8R/9PXW/bz4S/7J/wP3H+BL9h////tPcA/+vqAdS/1//wv4++B/9V/uH7P/2X0//Efln7X+TP9s/bL47s3fYZ/c+hX8a+xn3v+5ft1/g/3p+8/8J/qv6/4p/Fj+e/vv7qf1v5BfyX+Xf4b+6/uX/gfVH/vu2Zs//tvUF9m/qX+y/u/7w/5v0fP6v0J+v//G/vHwAfy3+l/5/83P8r////b9996190/4P7U/AF/LP7T/1/8r/k/3d+mb+f/6f+U/1n7Se1/9C/xf/d/z/+w+Qn+Yf1n/hf4P/P//H/Tf///3/ez///cV+43//92H9iP/qOsuHRJyFkY/e0Z+Pab6crGqCJe2/fhZEbyFf32s7BkXoX06BbJ9XFGiuDIvKdUElTI1MS2A+iwRZBLdG5+DD5/4Z94nImfkroNl5TJw4ws306BbJ9XFGiuDIvL9Qu/QmsRLZR0oKlvM6cgop3bxZTc7CTIXxiCIPZe2N4NNWRW3tFTWaar3BVET8cqGS1BSEC3OuzbautSQWRehfToFsn1ba5Dx8ZKCrDh1lkOSjcNFOiPqpZ9QoDLuKsRPRmVOus8dJ01jfN4GXnpws3P7595+noqZLhSviE9z9ybnDuPGTejv3n76t9zYP9qpYkm7kckzjdA8X8EGAI0VwZF6F9OgWycK/Lc1+DyDvrqhC9Xa6/Q2Vhc05qrWqP570fvYgjHbpwmyUkAXyd+6euJ04n+LM2x0SPBzgu2TrZALeFCaeQ8GmfYlzeRTR90Mhn0YUj8W7yYYrijRXBkXoX06BtbgyL0L6eyNre/HV/JtR7lqgX2FlXXAKFG37IfhPeygAw2oJf8IijRXBkXoX06BbJ9XFGipYxyKuQr8TbvlJPCrO6Dcai2BrikHb+JFm+nQLZPq4o0VwZF6C1Bq8CQHJu+AHUsBbotBFFRqoU/lt9noEUeNYkBA3gjAcF6uKd2Z1UsYe7vlX/3Cw2RJMs5ZbKx/AW8h2/7a5Asi9C+nQLZPq4o0VwnpKJc3bzW7rqiL8RAF6G8ef0NYXCJ0jjfLo9MWYFOVa3oLHZ+BLlDLXWReX1IuxIiAo0VwZF6F9OgTBrKYzOZbkD4Tf9/AXwssDWmyIAz/QEIS5EI2xneZkg7sNUWhp7peGFh6MxWzWtdsh8TnjLyCHbWPwt8b1YY8dGC+hql6Z+6jwHPMwEPLla8Hy3pK2Go5lkmrk3gg3XXcUaK4MhoPHAxMs7OdCY4IxBlD9rbp1DGTzT9AEgwkISyCwYBJnYaUjmN3km0FXlcBb6YZWprLEkP12HUgTZ32Nf2fcOb4hKcQx33kEaqrRUSLN9OfpN/Z9ghhiQDCXaodZlPENDkqu8w1VCWRPZi3DGvQFwBqRVOI9U3ZaYsJ8bjVeJVIlK7iBCLfQ2KIdsCF5iRYyN2ZlebiMzws/1A8/G2R32vPJILFJHIGmfVaeBwlIp1rJc4wphdNfbhKdTFemjmBO0UWEvKNQDGu6byuGfoJSSp6HmrEv0Rp82abnsq9VPACAzDLJMA/xiCFzoElmC//puPvVv/vlgyUFJFji04l8pQhzcqbntKi6qYZF6GufKZeTmdc+B1/EeDGqs3h7h4SUj06VEQkZwscI1KEyq3+7kUGT6OMkx7/KgMZvySCi5/OuiYuZOvEbE/CnZP65MTwyoyOSA4JkTTyEy6URKljU3hbLHtuSOXX0lti2T6uKNFcEC6/fB5R4VU/tNWgJQnP0cfGOxYB9j5iOxuFLHBzID/5nKz1VnoHPSjYk2m0Uqc57/yztMgjxl283Vd1tT+k6tXjdtYvQvp0C2T5AYSQazj1XFGh24d3Jcv29f+yjZnf5kYPvary1GHx5y1M94cs7f6QhyEoQw2ItOgWyfVxRorgyLcep/0G2CYVRC+nIy4x60A2AOCVmyoIFnG6zjrwNulgUJy6rPmgmFIb3COFI14SQWRehfToE2pV5sBi18gwhgadjkeGybpmrspKGKLUIw2uIJXH+DId12Gu0aoTDMykj7uLXs3yI6cmdzC7/hRJ3/KydAtk+rijRXBhwAv2GVhXvANsjUdLXOmjlPpKkzca2Jc05tusR5JEKGafO6uPzgsIqrkybR6W1Kmos/clGntbor/f+wH5/qsC/85VdQqy64GWLxyiIvu/6kMoueSQWRehfToFHQIcrqo4JAOuWsvJoal/Em1ZPRotFp0C2T6uKNFcGRehfH2wCKKo6MIkgsi9C+nQLZPq4o0VwZF6F9OfkCpVhnjZkoFGiuDIvQvp0C2T6uKNFcGRehcbbS04xLU6K4Mi9C+nQLZPq4o0VwZF6F9Ofp9KMqsWyfVxRorgyL0L6dAtEAA/v+dzjZP8lj9Apmy6V7P6Y2n89UCzstoO1HuzUPoKtZ0a73r8corm+Px+gfOLr8yQDrsJ9lo+z2ZxO3xFWzj+UyY9AW3MKv2SAmiq1rmkLMNMOozdbKNaarQm6KuWguV5E9C2mBUByK4iCzUJkHnu5aoHh//gP18C3dotv3BbDvI/2qDUOGusfY8VDSCtB5utqRZxJ+A+XJ/6KrSs9Ln2zTahYGbSvIvX7eAZzE1fD8If6mIMa9nZtrAncTBR0OPbH6F+mCc4YN0N+wmdONhUCJ+OIltXfCTNTN8mGJrjHSU/JEeL+WWUbpcTiXi6RG6lduUsA0+0pQ2HsgBCR35m7UslJ1QQSmjvEw2IkjmZKcOFOP1ddsZcg6yGQYSOYSDNtq3fNm6ElqdpbrCP4RY75Kw6xMnmkZ6h1qa1DUhK8/y+LgDm1VnHMZsJRzDE+18XHYws47eB0rmxV9tPKKLRgtXRmYhZx6KJcj95/xHCiyu2Qux14u2nsX6RJqYF6OtJKtslpQHY7vWI+ii/MAHYG/baoEKM6Cp+kr0ciS/ezSdI/XWsyoMH5FgG22ttl6VfXyrSc2u3DUX6p7DMPGct600W9ElseW+r7xKo6sithV4VfUXEo3SQVZa3tlYdnALyz7sx4ngvZxbcpY1J18YvfgWrS2X6R072nMOCmw4n1pnHm4zZlJnFQFXN4KDTIzCGgf9DDdwAP3yC2Ihm0C48DYoMh2BROzvrSsg7fmyVnTRCA4x8vGOH1TyQEVSElwhnxFEmbYt8avd5pYizJj+9X28STe2Fj/zfbjL4v/hZWseqjuitxAj5eyqZbIAUQRLSIzu6e8YUSRHPy4zA30o/FEDfm+GZuENWLOddnr82Sfwnq0XVR38VFUrcYdpvjCY1j5U2SYGswbwAsaYHI6PET4dONRB/9acV/4z3929EfimyCtRx/F4CFNkkBurRhjA2ULD+grZQ8YYeS4WkrNe681zm5fJTi4Lh7ho+7ytv65Hx54bhe35suHMqPEjUZferE36VyUvSHjYw/kTFjzV4Wu7aQii2d8m0LYU89GkBlsmw5PcDt6Wzp7fAyiIP9erD//lW/J9BmYV9vQsR6RBklhvjbZeZsQu4z1kxka68q/Zum+AMg6GMdPhd+OTS42tvjRXGgfrdsP56XyMKIS+ns6+2GIAtQg++3Jgz5BYf3OqcdvsH146rMzEO4xWBGwjz1+NgGl+SeiAgNrWutJvvAwXkn/42zaTLWltpyln4ZYjXZnSS4BDQOoE4GvCpolcDdW968PufZ781OXo1oZAX89Vp6pr7VIrYP66VV4Z3PlLJnup64bmEFaI6ns/ExxIdsULHTGP/DE+LcIx7fvMO00IqasdK8YptROQu2/M4i/i85QWFF8Xt/F5e7joSoZQAg68pHlmxcsRUMxGh/RkoG4Bzrg6CFQMk0y1SYEcvj4WWBV6dFiGPN1SOdSOFBF6KUscvhO45Fr944aw/WFj+tEqPI36v+V118dK29PcokGl/mLpBWnwzvMtRb3G08Mkuo0S6+DhKeDFE2giiWIiZ5uQQqUQwAbZAiB3dLr9c3MmvxCw2+HnfAf41Y3KyFxs16dXWzNAUQijE7xeJ1gLtRcv5IEBrV/Vm4IgxsYndqTR9JGQVXb4wQK21ogBLuJ1/bSv6bTd/poSQL8VPlrqexzPkFP1/NvZZfMgew8CF3t4hLIqH39dx6VK9TTS7XspQITaFpUZdF4G5rbm6dUpXogeHdccnb7rvLX2b1ahyswstPgLgmxtGQilScyFns0VH/lwgCQIg2o8yJgFo2DwEWgLanSobK6BE0HBRN947EVJs4aFNZXW+bJAr3LX0k9iPPqcVeXB8kMzO4v4IO/bqnSF6wKd7SdaLW3SuIKgLODJFL1ZggrJSg63hkDyzDe/+C+05gQIPPYIOR3VtntibDuWSjeu19kM0uy72VGScNWudYnI/wtCRCMJsarchgUlnaNhq5dpvqYiRQZG1HFO6E8kCW1gMJD5XI7en0QbuneLmu+pZ6WSNq1XmciVsHhljn1bSgwiKjd+nkSd5Jj3XTAxCZfzDq6wpMGLSdQRukVLsfjl+Z0DVYc2fZZTmAlGbGIyIQE3WdcO3QOod6K97y/iXKuK4pM/lqkmE6730P7KIm0vWy/KwziT6g3dCByNl9qq6No9F1lAJp/Hg5e4TeKFdowN61lf8YgfZQvFSFpaH7d5Q73ewPFuWU4a+q8C4KtcZ7E3vu5CpKIKwgP0uH50io+7zxuAc7zcWxHvNuBTRQUDwHmV+kRXAg3OQ/2l9gn0kKC8glcuhNc5TObYuvz/fTO6uvGZXWVV9zX9YrhGHWX1XUedebz3I6TvpL/XYrh/+z7b8g2RVNKk4QOmj5+JmAelxMFpHTbZP5+V+xzR1YB83FDc+y1Zq3KgGFtIOjOyA08qBBSUKWbuo/sfNaZgjAMKLHslK24aDaRgMP4O6m1MN1OWFcmrIOEnLmRznbzgnjzEuCn/qjQz/QOPzzkBfbjv43L8y7EMDZkNWj+4nh4+rLUh4X6D3wOqwmpnp9ET9071J5Rdh+rou/aEAN3gA6BWofwQED4vTLFNRLmuLzjN0jicOfAXyfrPGr6dhjHTHjxbxnVCuEPQ75kSubMAfRbwU5kGP1HLdB43yUx7QVBo4YVg0648QnNmWNi8qkoZ5NIRy9kR9f5Gb5vSRrddRhbZf0PsjtrV6XctReQuKltNx63gkTRG9x5VlyVPMa/XnKc1QMsyPrkP28mJZjbOR8dzTXgGGob3UONYdReqkQHkG1WyhGcN5t3XE0wMXLooYHB17LC+rEzhkFzSvSZVhTYiR4L2lO6P2TX8ZeqnYjvg0DA5JoaB7OroqPhXW+2OTlNW9ZE2/MG9yl2l5t7hNLi8MZFlwE9dGJHWMnDb0sxGaLp6DY3+lhSzHcLaXWCNSiy83iiHNa5vCqa0dd8tvOz97s6CvTptCoiGRuwDgLFkeqFKO//4BIQgWt87ewiE84HDw6+EPXFqgapXylcBn1hmXsbYUI3VZj91hadUrlCGNIV8fbTVgbKvABb+bM9Tg6rc97QOqJG/NxXOZMjyXCYeWA93jEhYxtR92KK8VJHemVhk8KgD17sohRP4msOgmiU8Kq3LKyxkalLolxr92M196cjmJCM4QDPR6EJNKuL/aXY/ToMTqLlrbAoUL2uoqjbaFgSI9nqDy5pz2/68z3iy7sQacXACYTbpBF/km6lOs/pVisQzrFRbCcSG0VcJnzBrBYGN+vkVifYRrylGg4Ui5wryHdswAYsSAlsWFcDZ+je3FjKDJaaUoN6SIHgFN4eHOQriDWFLP8LXVhchXv8Q1Wx79idlsHKjiAW4h2Ph2jErjiVVy7DH3KDYk2Jmm/WImVTS/lQvnq5EyNP929sNb43aXZjky/bVguLFRTnqCR4L2vLDN7p3xmqJ08ZrAyyI8sw0+HppAkkDhc6dAQBs/oG+fSBlW0hDIeSLkUL9B+F8ZL28HpNeLWfJz4TQ2GnW9vdVDFKrMZPbJxON0NQdtFlVK0einRZtYJs5EdyIjNg7Y/aac0NsvdGaSZgo8NleH6QV/vUe2To0hJr6KrjpEcl/pYEmJ3qWJwpemNUk/J6qdBbZbPQwIEZATaSiy+yynLX7ulKRQvCwHu2FcEpj+1T+hFY00JxKvkUB3XtsLHtveeOGJqxiDW4zkDQVN2UcVZCxNUjXbjLobUa/ocqH2BfSXI08HOWIkEAGtbUIUDco7tHlADrwGJRqotzFGDZMS0ydMpDP2Fzo1PLytTA1PZa1ox0FwzvbEIjtm2R/POzkEYiVHv/TLbEnAv57teXsv4adWZX3RF36x96zjK6hJwqK3Z/7agHiNaIY5Bjo4tbzWmc2DpXp/pl2t/Npf9xzf49Ijmu3h7wKalgCaYBQO+U+EC0xdLTTZ4HW1RIcWh72Am25RWIo/7+8bSsAQlrnSPwAAw5Hia1dOs1D8g4EscNlMx3G9YAAyJTpLZcPWmzweCeEKeqAAAAAAAEci5vXK0cTje0Zm56kTX8WPMLrKdHHtMl/Z/CEKf/mqRJJ3fhAsxgMwxcMo8v/STKvKGM/fQDYkq8VcFDy6F0PQBRxU3Yh8roPzMj5LJnz63942WfnP33GvA9OS9xWp/WG85KGs9TbKPnByKNrVOulK4wxr95oe7zpuau6boUhaHQ2yKSRSlu0N0RLfcNRyLOD8Yx7QJP+bHYICS0K93JT+E6F/9lOCijffil/APHlRrPXz71a7LVmYjhn9bOm7Qdmfis4BB8riaZDQidYviD/s5u9l7WHPoznvEiOl9AUGa2do5XUCH25O5CWLfDjd4AASrUnE2TZlr3PWL9xF/dYEJXr2FjRH8eKkBQ1O8FcypUju1eYa0wJnxLl2amfCXXwR6HQu4AK4Vxvksfd+kRtNkjqRtc3j6r0SfbaSkunpgeUIwYcEuwxRMZFDcvABfnauXNe6wb6NZkeVSRUZuEGNfJNovvyjdYbx+TT3Nb8BLjLnY9hB21pJa/4JdH2+vRZOHuHABGLBpVK7xxy/IiISdZfmYYkuxcvrLbDu5oVHkeoRowJUhAb1Lt0Jt7gwPVg6UsK3G8z8d37h0pUNk3GieelTCz1VsqF3rtQOqs+9JR5VuIUaMSY5/kMH2Bb8hcR5D2LsO0TErpOhVkIgP5okiYYujd45QB5WyJoI/wz7/cOeyTJYQwmZeswEzI5Hlp+BY+n7E/tG/RXlO6VR7LInaBqjHP9Nyoyz9VsqWsKoztC0MGBxpXUUil/+7CZUYAQy0rvug+GTuG2XbyrhsXCd6eF4YC69NVLgTTmawBjhQAMYQuq/xAwYlnD6+eWjGGaob1s22dWtDeQNKNvmoqBoWhFoCsq+0WRfh2/jp8tUp7q/QfP4JcYNW7+zC+z1azWjibCty2wB3Antc6Rrm7ggAcgIOPvTILmR7Eu5og8zzyFLF3MSjT/ev8/YL3Kis03cEnmeZXIsuapbAB4UA4sD0DdArInHZEaEoccLU8GaC8E6oDkQRIuz4Wa80bXryqORco6hIGF/iFTwQRHryU8ja1ufEM36X9UnI1PmV9PsIvh07Qk+p4/Fp+CdhDAOZjm8hBGE0SvXJ6X91T5ZFVwlf+1rptcq0aove8EFFB9kxWOa8itZ+otplGrn1+ecSZuXY/uKiQ8L9OiNcRww7JFVehS28kYg5J6QNTOoslz3P8mwfewT5XEZOPVnJoLeKgC5M8iCRJPoNnUISd6SwmXzOsMo2O4HSRQDKs/Tn4Z/lSN3nb1wzrx0rQA/57AFKeEDlfH7kFdaI4bYJwcdllhvpxAbYxm1Z3RxLO/cRdgnFPWj4IkCxkZIuhXKrjG7NG5xWcDWBz2s5/J0osGT+XU4/4oFEfSCM9xzy5g6bfR/W1FjyqOaP5aI/Z/YEfMgU64gr+2QpV9iBw/rSpmdfsbAdf9RdRRTnmDv2mkOaQ8UULDbXtauXDAUoa+XZlrb9Dejt8kbIIbsUtA0gjs9t4Crr+1Md6d5O8H3KTT1cwBP+Tj+OEaiJQsDkkEUKxZaFVAl/s/s3fAnoXd8JD5voMg0G9Fjwb1izop1NncJqr1wO6QPQw7WLVB1vhgavm7BTSmRF6CIp6tYkPMKHI1voWsSXWxi95FxvErjnWxnKWFBwpbKG6YfNyaSBjmtc6+DFg3WybfGp5xtPsVIdbrTOt5l/auszqjXrn0Sg0e1UHb17OLQRyCoEgWh8XxVg//ULQNq43+WzJ97MFjW7nVMEkypEH9gU9jq+qQ0GGszLOi8Z6Jlb3g4kHfXcbbxGPyIvZ+8ZD7+Gt4IGgbR3S0eO8YMdBiokbOI1kq9evmaulcLTM1CBQGUOdbdlSDKnj7T4LKJAffM5yrBR1ZB3hLToClrXA7cixLLaKuuI6+Ok2M0iJph7jkgOkAkmlpHSpCEB8DH6p1JbwAAAAAFqI3ufVq6V0ix9SQgSiACkLUPi7uKh2KEfk2G8Jsu0+O6yys5KZeMX501ZBWWeNxvCH7a/GyPgVQUAQhJj/+MwIjQj3P9eNgdWbLMR3a8QlMZ75C0A8KqblejIp7WyYQhdN1KJgiK/5u3DoqY7biylZ80EMZ3A+d0mHcx3k29q77+jYcz4sId1AitjB+Tv0QPTwdy08yVJRaBCw/2ueEYPFAex7141tK4dYIK/DaDKIrFsZdm1X7Me2R3uW/ZWaG6qLp7XABKTKg3zTgGi9FTd9wqrAf4uWiXMBg2rvU8y0T04S3l1wUDFBm+2ufI1lWjFWV1rFd6DTuCMGXk/JpSqUGpFaK5abiVg4620OzZSJBBa5kDq7jRIxxSd5MyZ6CClvC8g7GrBnOrxHoEPbb/vl8Vw22Y00Y4aGAOYku4bjS5LXkHSTL+XribqoVHyKLP45RZ5MKycLw5Uy7+wTaleEDXIaItyUIeN4rlKU9ZqvoYkF4ZlzFB2bwwNG6mMi3JnRXnHukzQah2GgdMBogeAICVKOAfavGAJpC5KwucKjkVTvAZLMwP8xekWBbtCQ6v443pzIX+47TRNqNjEUwwyAWtC5k2UCpDsTOYILfP4suHWHn4MdiJQmjK21UFgW/euyQOIofGLvhIUv6vxPiFaiuo9xrZbi2rd+bdSTpdpZVET8XCR8VxTxCI6qk+IzIIzGNri1Hj7q6ubcLp0ffepAqLPrWRIu33c3YmKMTBw9TSHXhZvuSI5QMBqogmKQ1gA8KnO8cjdMMjsu7NHY/zSjmW3qn/amRP859VdFlHVxX8Rfo1Th8B0MJDdi2K5pb00gAm8LFV4/V1pd8qgmQiLGVthm3pAs4/cNhErHYaMjcW9x74SbdKVemTmLJJ4dMGifRbNN9jJyP13joRhIXB1p63z5ypfzY4CJZ2Z8I3e/9tp+JnWuRnPs/5JXCXjiD6TUrftIIjgY2R/PISamhKQ8hJdu7vo3AMlDCEmR3zPWoovULeTEaqTkshJ3ZoufrpVA7NB7gtrYjfYPX77b+k4dHuIKvjPPqnUxYMfT5f7EI+qFxTV1KTXTuTOZ5yB2p3GmZP+B2NeXRyTOX5hs/jE5ks3ZM+7XTZ3giltv+lx2gmMGXIME2NPPUgmFRK4eyZlOJ6QeiTl1HIXGQIfPZW1AdUQygUGZ8/7/tDA7IMpEA0N43KN5H0Cpzq6ewKf1zlq7vaLujEzM9mTLJ2ZFQ03h3C/BxYLc82tcktWNHDoftkracBfhXnP1pd/wETUwQx40/2JIo27o6BpCOS74Po+OfsYgXjMljfz0l4zWhkW+0iapL6/7Qa0IFHjEMjv+T7KCczY6W2XC4Xprwtsgq/15Bw6wXopn0c3+m7DgHXsxY0bPrig3W1wq58Lr0Dd44LV9xMzr3jea/32f4A4SoiW2PCODcmuh8Fd+glgNLD15UZI2i+4XOCGyP2rh+d+m+y/JCAB1iQz3i14XE0K0uJczs1CwgbJGEPr/No4Thxcow2vg1HqHGfBvGH8Ie+7Wivu7QWGs8W5tWEl5b+YAqBT4jUcgz/Vf/AbDkLO10GuwX6XbdrJ5NbM9g07LQIjjeeshL1vO0xmL7LhhPVQwP2/m6cdO13HVogXLmQ7o++dNpUZB7+MGVsQObCrk5jSl+peSfkh61tQuHC1/H1v8/4HwpB4KlxF0+nm2rA6swDe1p0/fp0RzZr9aklyfwSB4AGwuBlMqjLjn+mbnh1Zb18t6cdetw2mROtuvAdiF0bIasTCOq/9QLKtGguUrinqUiQMV4QuF+I3ivP7LQSX9ip4xJQ0V2unFwLxzSDMrUj2JKUktQUQktJ2QmXkSvAEA+JNcyJE932GKvH+6eXaicXIULAbdHcZZ+pHKX804zN1I4zOG+paGBIMglpHDWo2c7yhN2A3jX4GS+9Sf+v+YCk3Ohd8KGiTRMVMA+CtOt1uHV66RSa40nqhT8hRNOU4e5Gdo08fqaJEDurFIJWsyRXp1QnrYK3Au3xj4BGYcr+K3XR3gHsJEvwzXvrD1jNAowhu7EUPf5LTyS5MtfC4N8zDND2h2AAk2ASvnSCH6DsiC/+9S2ryw3xt0O/WCuMif79i8B1Sexy+Fvh1nXv4RXwqsw5UD/4WxdfH2ygXNRqN+Z45C+06cNC0afVPf98IPULF9HCpKKxyjsX8M+hEJYsehIHX+M6WSp1H3Bp4cI4boTeRtjkAuO6MB4IHPMQv14KuEBM+ckKE4YLd8Gt/7B2ZnJhR80klaetgPmMVUQ/SqsDNs7+6mm7ItVx/fq/H+9OtHGUJnBKvL3q/4Xh95poXMO4unu8tXEd+Gzcu80nQOiqljaTLmKKNOOwVNnOerpcdR2Q2vpindY2zLPq19H2b0Zr95tKQy6YwFduxLao04eYMBO1nzFZq62NEdZBephLseGaw9sHboc4STQF2hn4bhL+mf2REEjRZRir+OVfV7Cdf93Djhtyhj0S6JsKX1MlpK+8rjWjjzYbjHuMzSUAfMJIITuJ7XWssC8VwJv0MSM44NvuzLSQCaMbxajrou2lbf6UQ4aF3CSvSrg7sSiWTuxkXUpUSWY2ZePJ/Mqk7fh+xF7lCB8q8culLZDkJ00D/wuKKneQKOI4HJBcaQelhMk7nVuuFJ3mVduMhZDZxopImc7rPSMXqOLP5yR6MimiweKLKEO9+/swGwnlkbEB24iOcxfnubp0OBmZUky6ggjtSPsXTJP+FpHn9uk7qrXqybRgCX74CHvE2jIAz5B3bh6+A/JObbIZZfXulAgWv1hRTAADvStpELodVgugME4bywQtSQmlHQWmFLUgr/bPbAx+uxtT8v5/1nbQTVfmm2A5K6O6H379pOo51OwNx278QdSthNuEkd1OJL1STFX/jAQPci3+9N9ski1S9OUv/fNnBJreYQ5yFM10uHcKiB2Fxaaalul8rFBChLsp5/Irr4ChM1w2BbPHdIQDBUnr+KgjjUyKFVu7eCdmwJ76tKoaZJUS/z36chbChWBtJtzOvjqTWfy/zerV6UtUIaOp+T1Omc9pMdK8e/2p7aozvm2ZUp5McySq2EORSYeLt5bFkr6Eyh2gQsAK3VYc17pZotIhuPXLybeBd9a+l9QoeG/7nulODwSRYZVyPSJkd1xcO+UxHRM7JMoAKKi7gP4g3X3/rJ+iFyHSqJXTrztQ1znb3iFk1SVAMTaeKhY3O8vO18u1ZEBNiGkujcycEi5+bExEcoG6jJ99yHyGITnYxNFv0jCaKOzBNcLsvbaQd7uLfPNxytXnADt0oX0PeyDtMgNXh538mq76FFnQwRZyPkYsqfsbD/U9RMBX0UEswP1Wxre1966jXbGiqU69UQJK3MDhP15MCNH3J/Yojvk+gcV4SAZXgcfPRD1CYBJsAtoSxD3m1o8pbK68MaxSc2a1vZGN/op9b48dZrS3zw1FrXEjCgDujHpm3zz4l04cS23bEb02iLwH0zjoQfCYT7LZ8E+FiUGZFDoK/pyvS8BS4wooqgZRpgMboPdG8fSn+MNY3uAr4FFlzfIJhajoVlKNzJpUInACvcplR2pC+w2M35UiMUukzKZX/8SUxwcGcrw8bS29uNvEXEIjr2ooIE17PYsEnJvpn23ijtmFEJlfU0BnZKx/PEOuNkkG/Kz0Do5ay0IcUVatWOl9sPdiksQBfOk4FrqEz8CJWPj68mEte22T/FIK82A/llACcJU2a4T7F1MqsG1000VTwcvwdDx45RNykz9kAcu8RQvj0N/ie7UV1GKgrL2OlE5G0QiwX+Emx9oSqVNugjNPHTJXOoP2D/IoRG+IU9EFf6+0JGBIeInNrOPWVya6S2BszQnJ/eypGqBMlGfrxMe3tebJV3rPNarPZM4CmRJ0g2abzNZQlWZTooXNksQlC9oRhbQKPYMpSVolJaJ77zYB/8BHxtd6GH5nrk5OFVj12SKtzNk7bxD8U95jk+5VeOdMlgMlrqB9RgA593pZ29Op0Xl60gigfi5vZtmkoU2Y9eOZbWN/w5UuvAJhJU1x37IEcc9rtDVV9GqebdFcJ3QdOzFffZ98ENUMfSEJ7ZvpOFWDYLJ6zp1JiJPUsIAIp/muZrkeXGCI/OS96tO1GZObx39Q65h7AuRvz5FRCj4gcWS6nxLwN5ynOeiOnFk1ZRkVzWJqswU3Up9dZvPhJ6kIv5QNa6jhIhev0UrFD8CG9ax7SNBCqLMQrwsu9rMEbXinZRRzP1ovayHB5nQK8+jRlzfZtRqY68LAMywdurn/0XHUFVyjQ5JAFfg0rBVgZ0Ymz7LYBBkM5Zu86YQ+YrHkILSbkiFlgY0A8dkuAZaen/kk19SOrJsq/oCuvF/ADM5yeTnZJjEDlAF6ZcAlTvBOqDcXafnMmvatJyB53sGJzH5hfcsmcKVQ+vgs2atnugQAd2vgLtAkoj0WQhXlyHcuJYXYVH65IbtCG+uIBODd3a53QnJT/CLjksVA4e7s4qSLcA0bvU6vqulVFN+Jub+/l0Qblk4AyR6dx0gQCUAAAP5+cB1qv4J4HG4x9JBk38Ggox/JiIJmcA5K3fLVGbA9ziJ5CyJ/CLEzGk/vYPZT3/jbRnFNl7UDEWjiYF6VEK95HI7G8s7xrD6AmgFTV//B/Ws0kmO/4aSWDPeYFgd6xUtYt9I0MAHMiSdCbTfU3qNjxSVn08VPyyPhhVQxaksua6eVqNXGojQzGtN1arzIFY/qOkBUT8xaVThPzgTiJyj5o5o+z3+YTGYGS17d4XrOzgI5IhS//0RoeEcc3b0Y18DF/Tr1Tu6Ho/ieDD70SilkVLP3qnm4cgB95gRoisWwQVP9wqFeDi5azegbqgrNi5Nf5pnd89Jb6QLX9my4C7RWkvLMTDGn80lrGdEE0loNlM1rlNN30CVtqAxhsjRDsK8oAvxQ+XRwgWq2S0+IZq81t13MRd76Pg1+MeWA5B6LI/Xg7f7NNcL/mqU7cAJR/wmN66odn37yuCsVJKTO0gRhc31Ikb2BOOJ14Ye4a9rQUZE/qo+rKakjaXwJhkgcrCxTE8CeEA8oudPAG7xWMDE+uDBZFtx45Ez+b0Zj0oomYBeKKtLdxK5Y/KJhebAW2qzL7Iy4kGWrVpkmAd+L6Qv9J5HyLHi05uHYcNKkaCb1L+irDDDg4GsnGWISOkKHBcpBSVYr65PteNNXBFS/wUzgve1Sqi07aaAKfqFqeZZrOuXw9vuxsCnmc+dqFkjfjYcxb+yMVdqRRFTXf9gLOumn/V5NmQAwY+eW05IdlT+s73p+x5Kh79Sy0cRGvpl9aN6GaBaDiotg2QXKPGRzxf12Xv33MUs00F3elyXWLhNtjETAJcggZo3YABP5SXtVVl/vp1Jbl+P6YrMMNpFKLLNMJHGwwUg2SngB+cjRlLqKg13Om0WYM7IoDxlBPachcU3iVSPdAm2OaV/g8HcsiX3A3b50WZLT0ApN/erKDHhFjcxB8kt0JNTvtt6ZG3DLO4u7giFhjAluBIYPkaxnrjpvDY6T3p2d2ethaAyjpefQrP+WY5Rs6QBpGvYkcTP4a3PavA8HxLm4f54J2tQ60LtZTJ4nkDZrc9MRZ7mu3UqWoubx9BDPg3HOwPWa5TEZLnmZQAQqfEOzWBubIEs4ezE6ghAeWgFoMedhIsjeLfLNrcdpSQZLelv8/Gse/RzPa0Ph1OHtB0IUwIEF7mxjbH2jJWyhSy+BoTCepm63NsnT/fPjtVfHg+6yl6NfZLyoNOxVNmzMaqPjHtGEIfI7XbGDo5dk/eeUMx20Rprxd9Gat7k/DhfBuo9ld3EZm5vAsX3sUVn1SUeVA/xpnnarsfABiVQS70dfMKUtHb57fxfTSzFtT7zTsZLWcBza6nmgzlz7YHJkcJ17Y+WK2hphCVuTlyCD33ypKWKEY05smCBvPBo7aWiZXKeE+TOPHzZ0PeOVzRaITZl64qcdDxkJd9scMATRwpVcweyw7R2rgmqRYmLozWADX66V0J7YycACI6xkjEr5U4ZJJAn3p/qXxgD0JoErSmux3LevveIrgywaMhr7odmqCckxMbou3K8IfxDPFUKooSytyV7XiO9Alo5q1NkqTKO2uquQnccTloiROgyukCrIxo65DqaSFAEkAuZPl5m6zLJ+q4eDgHNg+Sw+NSnhqO7Koi3TpjuAu2EH+IYEX7oxCudCilx0bId0DYPL0OvhVOm6P3Fk1/RK0NU3Yj0ieKLGphwDmLnIWLB0qNliInV/5Q3taNED7rNHT72KE9y7vnviLd49onL0Odj8wJ/eB1Y7gSQiHtlZw0FwhVfzzcajG8mG9CgxPDAOx/cvVO/1Rzr0CQQ7m8pYJ1sSHEBfZ6i7yUOUJTGZZ9HdazssbUxcafc9LcOCuhFuefs2kythRleVhMfr7F2q0nhZpcf86gDEwtq2kA6DZdgKF9E7F6GmMuJDxnedx6jdoDQnh9tx1axpc8W1i+X4jjxb3QrFD/dr4mbMiRxf+xGGCy8q+U2nNyEv7Rk4J/wDivj52UYYbMilddjI8LIuitCbgaVcvBXYP88IHK+8imMc1xdJHlNNw6i6E/bl9PmAYZPZSiV5wG7EtvfSUZ3tWvP66rPBDKe/whMsghh8TgqAbalepzqP4E30o9OcPVR2vpCAJMWSSS7TwNLeJ/25IF1rsOmJVSVaGuq22XCYjTf+58CeUklaYugitUtS4H8mRn98g5BNq02b/+RZRf9Xuxzny4gEr3KXXPAvgFP2I0jQB3Ycrtji4co1CqkxfwuMNjDlDLoSq0sCl7vpBi8VTT68JyzdXB1fNnP0zzwyRlCjSgGDt3i6TrHToMfx2ogU/vWJtt7n4JxMhejYMhs0GGf5C3NLKzzeqaeS0vv/B6J6/XmgaeSC5qiiSEbb0ZJFf0XUIS4dRVBlTR9eC//VRWEyQTFvt0twZsStdHhPc/ZOVH3YZxP0lwHYKtFbjJPS3fMQpWt87H6hnyF3CVhxRrDV34602+WnkQ1c7jkbPR4l+eBlqWY573vsRUkkjLu5TFqytiztN4Xz7MTBn2fes1U4vL9axRTKvCUb0h5CJ+OAklPjB1bvpsa6/c7OuH8NXsfKByXwh3EARSdx3KzFlaDYGIPcPCBbPUaQme7+1IX/w9Bvw3Aok/Lj9bM2PgLiREqcAU+YzDmt+XK75+1J4iwJdml+c0VVX1/p2Uudwwy7mTMWScoJXv/UWtFFeJn3qKiRldZG0UNkSBIQT/uHrDXUQ4HZIchrXRrhku0CSe1uTOAJ8JN0lIwdEd2QfxhfMZTAo0VFnY38QqdAmbK3vuStE799wqzqDhbQ8tf3dFqOh/9fs8rWUvEJZPLC1SMBJhEgdW7ePWkLEn1iU/T+MBGmkpPgSj2WqAE077LGIEH9uaZw3Zhgf9+HUMOaDcf4t0Stq+q0yBiZsfbcNUM7laCx09cN1eGLGGTRjXGIXsJBvNiz25K62riVVOdp0+qGVDXScNrFTV74HlJBUXgMtFg1qG10DcJzVCVnBUDRXam3saBe6tgq6Cc2DETPGXWcC/4ebvuOIOmD96KBAhXH3+i+AL/NzYxEj7Pkkx9oQzosrejQ4dE/ueZyCi6pM5MZzusUpp2RlH9X13vEyH3Z272eOkX0YwOS4LFQFPr57dtx9FVu/mh18u959dolBv3tNK7ui0LyZgf+lYTD+NgAtGQw9COg7ZbaxRaYGYTizccCCzfB7NYxokS2dr1H9IgoBw/PB+81kkoyu0aVdAm2irE7v6y4Wm+H49fGv1Fwhhav2ji9bl/pk4OBgS7iUX0eIdU0v7x1l6yptV6MwADsXKFYc/8ylzgw8kswdkpBq1ozsUhKrM6CI6GTO7WsaIoMRPszYn45Rx4fxCRQ+Cp/720jTjVwcyS9to3k2dX4b7y+Fy9s+c4elOkm9gpFbKPACoxcUuhTy2wfP/LFb13uFvxhCDp5IdEkOjpDloCNzL414ROJbwjJgFl552jgLuFZUtlw72rSJUCz3NOuzR8IUFGlHFRqAeOtw/G79UMnwn66Xt3NCo1zmqtptO2pUyq7Nce+iDuVkH29V9N4hGN+aX7OlMewlnz26SSq6ckdavaxUwvZn2hql6yfXHQqqgfOKh5aJsjE/yZN0eg17qQqn43OaWvhR3aJ6imkizDdlJfUPq9a266u/wiIwKD9xzT2AcS7swVcgCngMgyJhd96qPVrEiZQSIObxmTQuAJdE7ZmzxsygskO36bop8GUKmRxc63inNpR8+aWqLytJcwFDcDVAqX+MYrAIsDBNqQE/uXD1f2Gnly1NwdADME86CZpS5LnAKv1OT9K3jDcSo+UJYtuVFZEb69UE/YlYOA+DNBJx3vlY5c08VfuSbQ85coSJzdU7VZJ68VQzzqXHPj3mg6N/kgRXzZRLQpNqFaFf2ddQfauq91esKch1Lf5NyW3KRfR5cuYy8s56CeJ4RMAXns6GXk43KiS3AAClwP71gus3I8Pbaybnk8PMMFirVknWy2E2tdPT/+K1dWP7oHh98X+raq4PKhr/deNNTyReos6FEYA2X+GVb+j+v4nep9HzimIZ2EcsHbXIcGtG7rNU9wIvnbK6BcfFd428a7uN+KHupK3lsyIM6rw1/DFoOZthsdU66XZVllTtL2wq/V5RKNHY6UX3WOLjpRC6BOl1+y2Tfe5gWtSO1PZgMdN6/ZKQ8Ki9friXy+MRejXxKOV6yHcm1LmY7y+8JPRCmRovQkG08L7qA4GBw5//aoJmdCKRDVSvmC5/QaqnOBQGHdfXQGbCW+vNLg+w9dcnCJb2Hiy3+CqJKP5w1qjXicXUX0V6x130K+66UQNw5zzmeaThiz6iCr4pLZRP4F+xXt8Mjf32fcKGOIr/IPS1vJbiMU0tkVxwG7zUf65SJqyDP9+7ucLljnOwK2wE7z/cBjSegI++PXaWe6Z+sbZeVgsOhbY6DchJ56BqQsAj8ydp5G1dd1NChhdnIIWLrurb/CczIp4QeHc3OZV+dhQoyv/JzwcKhPzC/KsYORSGfKIcdGwCHWd8eqcEluLnhjT+JFIShSuf4R6NVWNtA0HlvsLqeEl4+R2vkGnUuWwLNP+W79hstr6iwGGH/eYqFbOlE7LboEffh8oI4K2uFqiQqQVmvn8YJXCLxq3MTyNg4ARNUcoFJCouGBilQzFB2YdOWK8tqyWrvws9NmwZUfaQKq/oerL2SmBqO2LvmnEtFtOKxVIrhyOeALOyJoANzBAAIOYrgGtVp+MmjaaqcGaqQI8dhy260hNhfSQy2YdiWnoo83PibORKE4UzoP6879B9VxCLVUTiUAECX4QCq7uWj4V+pRS5YaQjM0zB5R4dJC578TpJa9o17OTZTpYRiEG1WgDKVDWXzpBC2hVesPLxBTUbbL/0PNOo7w1xtQ9NFZKba63ALrVzUBTMS9fYpQ14Ztc1/m7Jap9dsO+eOwuXNoG8yR2Fd9f55YeOM+nJenLFTdLgn0JOTCAdko9VEWXwHuaAqoRo1vdvALP3SQV78Egehoijp45+rtVsxboztg1Hw7mW5oCmXadRyjk6j7LsOqvGTquojaT0QRz7RSQ3QgFlxiB2jeAI4taxs6FNzVnf7RWxoaLBwwGirPIJwuUfWpwNnhlbUrfvvu3HkwpK2b+Nwn06GVBhgedUD/PSQJ5f0QD/R7rJ4vL1s7rmzPPT/nA7+T1QFr45+W7ESTZvBM6rRexVWZuWzxAc01+rmjkcqIU6j3s0t/ASBPRFntQb7MQiTLL784wXi8EdCOCrsfuDGZdEgYETjx39OznFa3ylCkG/7+AEuX9jjMNVjIwZkI7zjF/+uRPpEj/c4oq6jKvmtnS1bjEDQ/YsJJ2qnETFFQULdqE9n/7Tj+az8OMdnup2DyZktA+rkH1D7X/2lP0CUx7EukTzctPDIt7Llji/W1OAXv+J/YSh9lfaC7LBiedph3XqJwf2qTFh7hZIKAvgyHV0Ecy9MLtdzpnBTUzSXaTL/INmgYGPr1I3IY2hOY4zT86Aa3ogOtDfBvnidGuCGjhDUfw/ZQ3lxZRX3RAHe2wRolQNJBcLYYc2nnxp2QxHvZqiSRyx2G5EGvUS/Fs5IyTKRZYn4sqXurW9qKSJxNdj/ZbGnv13nmhbsOVg51AUnfuHXI65psd+K/EnCkVn1sPdA+w/trVpR085u/pUok4/1p3T/YEJ+6+Ey7EDr+dh7Qufpit00VB7fBRFfUpYex+CXKBM+uI1L21ZFuXzk64lXCmwcE1J/GtMS/1MNxay7CFoqJ8dq/TK8XqO3qMTCHPUrYDFqeEKBzIDc1HqXipFo0GIMMyY9P11DkJLSPhT1GzIUvqv4XhW5HJ2CXt3kkKRtmDq7O4T5XDqLovhgJjp3rSdBnCh5ohjQALMZkpn4lhoijAy1xrdfrINPwSN0xWoC5jZ417EQX5WkBIMtMrPd2+biRD+gviWiFvcFjVex7hVoIgfgBO+VdVK4Airf1Q0L4H/96L+bCpBh6aUW0jfdv+imVyAjh43uLC53eRY6C14OKlmOE99+uZxPefo4WSbzU9+/ryGzcKMRLozFrmC/1ozV0vNKu62q3iQk6BxxDPi5kRBkf3UcLWnVXqO9Q5un6zu7XPyWJTzW/M2jNWZTcwz8J0POlVlzvelEgyCKGodnVWYNbp0VF5B3aAH+UK6C0BVxN88Ite9LKv7hDttLaLI0+bKP4drcRGZteTaFzPwY2AsnbNqNNyRA5/YLsmsikUZ7VBFUQuNx40yR0YNtIrEWOamgx/em1zkiPKetIs2V720TuvWNLAib0Io/pqxYWBZDR+yH1GQQDRSWTBc4af1sHGUPuw8cbNHWQcThLbUmmmUyiHscVELwNESR8571B4xEGghgk1Kh5zWQe6iHOoSh1iU0OtLTUkpXYJ2p/OI6nb4SlxnySErYoHHe5318pY0zq4IlQdwyr9xhN71VeBWE5L0BzFQZ4ykUyavSK6GOE8+wMsT4Iem9foWItC59YRyuJQsNAl/LfOHRW9n6Xtr/D5Z1yVAi4KXUd8ffpNi/VlJgSENMq0rRcGKuDgbVO3tpy0eMnGIxK9HSx9Gcl9KEY3oYxIfc6VsKFusklazcSWKev2jyfjkqpgMRPH0NAoWArTxO7s4Ht8xZMY5AYTg4lUCdZ356m9wmNBSOmhyxqgIIUoPJjThevd485IhcELj6QiN/nThe5SP+LS3ZWUy/2R730zU1Uebt2Zcgca+RuZ1L7PTM4WPAEq8VpnFxEjvrBE7ek1nkwoYwS+aMUGQ2QCgqaqwnRa/N/fjAcT5LicD6gC7Nsl4EungPx5VCtXQ7pdfiuUhPtORAffrSliPElmohBIrHIfymBsNBkywuotX/1agSp2a8F3vybqrF73gW6V4/HyVDeVzD35ATgcVNnzuzeCzp9NAdTqAdLDHywFgCVqRB/5s/6QjVHvwCUroEt5MT8tGybxhRwKwjjI+A6BDwUBlRs0/zWjNfpLnvt0hI6R3GDnEAfafZ9X0h0MhfhArH+zWs5GdE9LlLztjKAywI72j1P6FAK1I0es8Hmr4tA/H+2k1tGKjAmMkbAZOmTsL6bB9gYNbjtbDGtDsIGhW8gb/2kjFYzxiiFuVscOyvo1XIbZhN3/gWfOsupK9usQiafrgz7DezKk1PNPL+13njr/JjEz0XmiQUteftxan+waU3l+otiB0dklAfUzvUwl2FG3UJxZxoGYziOUVtPw+Pd7v1hBfg421MALArqyfzS8lZ7ca2a71OByXJpm1tztTH1/p0sJ2z/9IB6aofilBcUySie5Lq+jy6qAPvARpOv12rurbhI/D7IAVHoYZsAeqy6mVvHq6AFwSCNYD89KfZJ1Db22XutvoiG/dwR1hqbkgFaeuGfyn09AAAAAABMRjc/vrZ9EuxKG8yfyzTaxC9XnBvVTI/pZTnBKHKWXBjKXdkCNrKwWsV0iJ5qt+3YvtK6Nbi771vioTtzFOdO2FCYpw5YTswunogAAc4AAMkqnkHrxlnGnrEEmtpGaEP/IQUWde8CtlKBPHnRRGhjlhaXp0FkBjn6vEEuKLvWjbRuTBSZbT52OsOLPTHfsa4uU5msdqW5nBDgAAACF+BGn8PcYFfTje/DXTePmKdwopgAByvkX0a+bNf/yN42UYphaApr0QGSVrlRsUPe5zZlH/Czco9I30ZCf/vCbuTStlZJQHjy0WngAALpitICmyTlBjtSG9gAio40cBX4AAAAAABFWElGugAAAEV4aWYAAElJKgAIAAAABgASAQMAAQAAAAEAAAAaAQUAAQAAAFYAAAAbAQUAAQAAAF4AAAAoAQMAAQAAAAIAAAATAgMAAQAAAAEAAABphwQAAQAAAGYAAAAAAAAASAAAAAEAAABIAAAAAQAAAAYAAJAHAAQAAAAwMjEwAZEHAAQAAAABAgMAAKAHAAQAAAAwMTAwAaADAAEAAAD//wAAAqAEAAEAAACTAgAAA6AEAAEAAAB4AQAAAAAAAA==" + } + }, + "cell_type": "markdown", + "id": "4eff7563-3a93-493d-8788-bb055a23d07a", + "metadata": {}, + "source": [ + "Let's look at another backpropagation using a neuron\n", + "\n", + "![neuron_model.webp](attachment:34f0ebc5-cb10-4db9-b7a9-9f501a3732da.webp)\n", + "\n", + "activation function is usually sigmoid or tanh [[more here]](https://en.wikipedia.org/wiki/Activation_function)" + ] + }, + { + "cell_type": "code", + "execution_count": 153, + "id": "29f8a9d0-16a6-49e3-b1db-52084db2ce24", + "metadata": {}, + "outputs": [], + "source": [ + "# inputs x1,x2\n", + "x1 = Value(2.0, label='x1')\n", + "x2 = Value(0.0, label='x2')\n", + "# weights w1,w2\n", + "w1 = Value(-3.0, label='w1')\n", + "w2 = Value(1.0, label='w2')\n", + "# bias of the neuron\n", + "b = Value(6.8813735870195432, label='b')\n", + "# x1*w1 + x2*w2 + b\n", + "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n", + "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n", + "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n", + "n = x1w1x2w2 + b; n.label = 'n'\n", + "o = n.tanh(); o.label = 'output'\n", + "o.backward()" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "id": "1a1bdc0d-23e1-44bc-8b8e-1e88cdb64b2c", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738898448\n", + "\n", + "x1*w1 + x2*w2\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738897168+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "1315738898448->1315738897168+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738898448+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "1315738898448+->1315738898448\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738898960\n", + "\n", + "w2\n", + "\n", + "data 1.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "1315738896848*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "1315738898960->1315738896848*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738898512\n", + "\n", + "x2\n", + "\n", + "data 0.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738898512->1315738896848*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738899088\n", + "\n", + "b\n", + "\n", + "data 6.8814\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738899088->1315738897168+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315731070736\n", + "\n", + "output\n", + "\n", + "data 0.7071\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "1315731070736tanh\n", + "\n", + "tanh\n", + "\n", + "\n", + "\n", + "1315731070736tanh->1315731070736\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738897168\n", + "\n", + "n\n", + "\n", + "data 0.8814\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738897168->1315731070736tanh\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738897168+->1315738897168\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738897680\n", + "\n", + "x1\n", + "\n", + "data 2.0000\n", + "\n", + "grad -1.5000\n", + "\n", + "\n", + "\n", + "1315738897872*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "1315738897680->1315738897872*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738909968\n", + "\n", + "w1\n", + "\n", + "data -3.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "1315738909968->1315738897872*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738896848\n", + "\n", + "x2*w2\n", + "\n", + "data 0.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738896848->1315738898448+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738896848*->1315738896848\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738897872\n", + "\n", + "x1*w1\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738897872->1315738898448+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738897872*->1315738897872\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 154, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "draw_dot(o)" + ] + }, + { + "cell_type": "code", + "execution_count": 155, + "id": "7d33e845-75ed-4aec-9310-0f55e866e065", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=3.0)" + ] + }, + "execution_count": 155, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(2.0)\n", + "a + 1" + ] + }, + { + "cell_type": "markdown", + "id": "59924a5f-ae4c-4d0f-b45d-611b32ba6df7", + "metadata": {}, + "source": [ + "Let's add this QoL improvement." + ] + }, + { + "cell_type": "code", + "execution_count": 156, + "id": "5ec7a25c-1540-47d3-aefc-eea6cfac8e8a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Value(data=4.0)\n", + "Value(data=4.0)\n" + ] + } + ], + "source": [ + "print(a * 2)\n", + "print(2 * a)" + ] + }, + { + "cell_type": "code", + "execution_count": 157, + "id": "048d58a9-e70d-45b7-885f-e34317323283", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=54.598150033144236)" + ] + }, + "execution_count": 157, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(4.0)\n", + "a.exp()" + ] + }, + { + "cell_type": "markdown", + "id": "8830493e-47d3-44dd-957f-9c84a9fafa99", + "metadata": {}, + "source": [ + "a / b\n", + "a * 1/b\n", + "a * b**-1\n", + "\n", + "so, let's implement n**k and have a spl case for division" + ] + }, + { + "cell_type": "code", + "execution_count": 158, + "id": "c75d090e-6f68-41d8-b874-fd72a5c2303a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=125.0)" + ] + }, + "execution_count": 158, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "z = Value(5.0)\n", + "z**3" + ] + }, + { + "cell_type": "code", + "execution_count": 159, + "id": "ff12ee1b-b022-4790-81dd-b88527719e29", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=0.5)" + ] + }, + "execution_count": 159, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(4.0)\n", + "b = Value(8.0)\n", + "a / b" + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "id": "b313cee1-dd94-40bc-a414-ac6475443585", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=-4.0)" + ] + }, + "execution_count": 160, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = Value(4.0)\n", + "b = Value(8.0)\n", + "a - b" + ] + }, + { + "cell_type": "code", + "execution_count": 161, + "id": "7079d013-6e83-4b81-b1d1-d4860d0bf3e7", + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738821136\n", + "\n", + " \n", + "\n", + "data 4.8284\n", + "\n", + "grad 0.1464\n", + "\n", + "\n", + "\n", + "1315738812816*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "1315738821136->1315738812816*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738821136+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "1315738821136+->1315738821136\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738816592\n", + "\n", + "x1\n", + "\n", + "data 2.0000\n", + "\n", + "grad -1.5000\n", + "\n", + "\n", + "\n", + "1315738816144*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "1315738816592->1315738816144*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738816912\n", + "\n", + "w2\n", + "\n", + "data 1.0000\n", + "\n", + "grad 0.0000\n", + "\n", + "\n", + "\n", + "1315738814800*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "1315738816912->1315738814800*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738812496\n", + "\n", + " \n", + "\n", + "data 6.8284\n", + "\n", + "grad -0.1036\n", + "\n", + "\n", + "\n", + "1315738827216**-1\n", + "\n", + "**-1\n", + "\n", + "\n", + "\n", + "1315738812496->1315738827216**-1\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738812496+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "1315738812496+->1315738812496\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738825808\n", + "\n", + " \n", + "\n", + "data 1.0000\n", + "\n", + "grad -0.1036\n", + "\n", + "\n", + "\n", + "1315738825808->1315738812496+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738816144\n", + "\n", + "x1*w1\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738813264+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "1315738816144->1315738813264+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738816144*->1315738816144\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738819984\n", + "\n", + "x2\n", + "\n", + "data 0.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738819984->1315738814800*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738827984\n", + "\n", + " \n", + "\n", + "data -1.0000\n", + "\n", + "grad 0.1464\n", + "\n", + "\n", + "\n", + "1315738827984->1315738821136+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738861968\n", + "\n", + " \n", + "\n", + "data 5.8284\n", + "\n", + "grad 0.0429\n", + "\n", + "\n", + "\n", + "1315738861968->1315738821136+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738861968->1315738812496+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738861968exp\n", + "\n", + "exp\n", + "\n", + "\n", + "\n", + "1315738861968exp->1315738861968\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738827216\n", + "\n", + " \n", + "\n", + "data 0.1464\n", + "\n", + "grad 4.8284\n", + "\n", + "\n", + "\n", + "1315738827216->1315738812816*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738827216**-1->1315738827216\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738814224\n", + "\n", + "w1\n", + "\n", + "data -3.0000\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "1315738814224->1315738816144*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738820432\n", + "\n", + " \n", + "\n", + "data 2.0000\n", + "\n", + "grad 0.2203\n", + "\n", + "\n", + "\n", + "1315738813840*\n", + "\n", + "*\n", + "\n", + "\n", + "\n", + "1315738820432->1315738813840*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738828624\n", + "\n", + "b\n", + "\n", + "data 6.8814\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738813392+\n", + "\n", + "+\n", + "\n", + "\n", + "\n", + "1315738828624->1315738813392+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738813264\n", + "\n", + "x1*w1 + x2*w2\n", + "\n", + "data -6.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738813264->1315738813392+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738813264+->1315738813264\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738814800\n", + "\n", + "x2*w2\n", + "\n", + "data 0.0000\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738814800->1315738813264+\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738814800*->1315738814800\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738813840\n", + "\n", + " \n", + "\n", + "data 1.7627\n", + "\n", + "grad 0.2500\n", + "\n", + "\n", + "\n", + "1315738813840->1315738861968exp\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738813840*->1315738813840\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738813392\n", + "\n", + "n\n", + "\n", + "data 0.8814\n", + "\n", + "grad 0.5000\n", + "\n", + "\n", + "\n", + "1315738813392->1315738813840*\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738813392+->1315738813392\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "1315738812816\n", + "\n", + "output\n", + "\n", + "data 0.7071\n", + "\n", + "grad 1.0000\n", + "\n", + "\n", + "\n", + "1315738812816*->1315738812816\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 161, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# inputs x1,x2\n", + "x1 = Value(2.0, label='x1')\n", + "x2 = Value(0.0, label='x2')\n", + "# weights w1,w2\n", + "w1 = Value(-3.0, label='w1')\n", + "w2 = Value(1.0, label='w2')\n", + "# bias of the neuron\n", + "b = Value(6.8813735870195432, label='b')\n", + "# x1*w1 + x2*w2 + b\n", + "x1w1 = x1*w1; x1w1.label = 'x1*w1'\n", + "x2w2 = x2*w2; x2w2.label = 'x2*w2'\n", + "x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'\n", + "n = x1w1x2w2 + b; n.label = 'n'\n", + "# Let's define tanh using exp and divide and subtract\n", + "e = (2*n).exp()\n", + "o = (e - 1) / (e + 1) \n", + "#---\n", + "o.label = 'output'\n", + "o.backward()\n", + "draw_dot(o)" + ] + }, + { + "cell_type": "markdown", + "id": "c817b8f7-a9e5-4193-a759-27973d83868d", + "metadata": {}, + "source": [ + "Now let's look at the same thing in pytorch" + ] + }, + { + "cell_type": "code", + "execution_count": 162, + "id": "961afc05-1958-41e0-b434-40d8b0a148e3", + "metadata": {}, + "outputs": [], + "source": [ + "import torch" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "id": "67c41ab6-8b5d-4908-a868-0d63c196ae5f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.7071066904050358\n", + "---\n", + "x2 0.5000001283844369\n", + "w2 0.0\n", + "x1 -1.5000003851533106\n", + "w1 1.0000002567688737\n" + ] + } + ], + "source": [ + "x1 = torch.Tensor([2.0]).double() ; x1.requires_grad = True # This is false by default because no one needs to see gradients on the input layer\n", + "x2 = torch.Tensor([0.0]).double() ; x2.requires_grad = True\n", + "w1 = torch.Tensor([-3.0]).double() ; w1.requires_grad = True\n", + "w2 = torch.Tensor([1.0]).double() ; w2.requires_grad = True\n", + "b = torch.Tensor([6.8813735870195432]).double() ; b.requires_grad = True\n", + "n = x1*w1 + x2*w2 + b\n", + "o = torch.tanh(n)\n", + "\n", + "print(o.data.item())\n", + "o.backward()\n", + "\n", + "print('---')\n", + "print('x2', x2.grad.item())\n", + "print('w2', w2.grad.item())\n", + "print('x1', x1.grad.item())\n", + "print('w1', w1.grad.item())" + ] + }, + { + "cell_type": "markdown", + "id": "43f61380-db69-4b53-9a87-e6121fe2eae1", + "metadata": {}, + "source": [ + "Let's now build the a neural net library (multi-layer perceptron) in mircograd" + ] + }, + { + "cell_type": "code", + "execution_count": 175, + "id": "9399d558-a3c6-44ae-8b3c-a995d750f20e", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "class Neuron:\n", + "\n", + " def __init__(self, nin): # nin is number of inputs\n", + " self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n", + " self.b = Value(random.uniform(-1,1))\n", + "\n", + " def __call__(self, x):\n", + " # w * x + b\n", + " act = sum((wi * xi for wi, xi in zip(self.w, x)), self.b)\n", + " out = act.tanh()\n", + " return out\n", + "\n", + " def parameters(self):\n", + " return self.w + [self.b]\n", + "\n", + "class Layer:\n", + "\n", + " def __init__(self, nin, nout): # nin is the number of inputs on each neuron, nout is the number of neurons in the layer\n", + " self.neurons=[Neuron(nin) for _ in range(nout)]\n", + "\n", + " def __call__(self, x):\n", + " outs = [n(x) for n in self.neurons]\n", + " return outs[0] if len(outs) == 1 else outs # QoL \n", + "\n", + " def parameters(self):\n", + " return [p for neuron in self.neurons for p in neuron.parameters()]\n", + " # params = []\n", + " # for neuron in self.neurons:\n", + " # ps = neurons.parameters()\n", + " # params.extend(ps)\n", + " # return params\n", + "\n", + "class MLP:\n", + "\n", + " def __init__(self, nin, nouts): # nout is the list of the numbers of neurons in each layer\n", + " sz = [nin] + nouts\n", + " self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n", + "\n", + " def __call__(self, x):\n", + " for layer in self.layers:\n", + " x = layer(x)\n", + " return x\n", + "\n", + " def parameters(self):\n", + " return [p for layer in self.layers for p in layer.parameters()]\n", + "\n", + "# x = [2.0, 3.0,]\n", + "# n = Layer(2, 3) # three 2-dimensional neutrons in this Layer\n", + "# n(x)" + ] + }, + { + "attachments": { + "ee0672be-bf08-4422-b9a4-24bd40eea517.jpg": { + "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxQSEhUSEhMWFRUXFx0WGRUXGBgdGhgXHRYXGBkaGRsaHSoiGCElHRYVIjEiJSotLi8uFx8zODMtNyguLisBCgoKDg0OFxAQGy8lHyIwKy0tLS0tLS0tLS0tListLS0tKy0tKy0tLS0uLSstKy0tLS0tKy0tLSs1Ly0tLS0tLf/AABEIALoBDwMBIgACEQEDEQH/xAAbAAACAwEBAQAAAAAAAAAAAAAABQMEBgIBB//EAEoQAAIBAwIDBAYGBgkCBAcAAAECAwAEERIhBTFBEzJRYQYUIlJxgRUjM0JikVNjcpKh8BYkQ1SCorHB4dHTNERzkwdkdIOytML/xAAYAQADAQEAAAAAAAAAAAAAAAAAAgMBBP/EACwRAAICAQMCBAYCAwAAAAAAAAABAhEDEiExQVEEMtHwEyJhcZHBobEUgeH/2gAMAwEAAhEDEQA/APuNFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFQ3M2hc4zU1U+J9z5imiraRj4IvpQe6fz/wCKPpQe6fz/AOKW1U4lxOK3UNM4XJ0qNyzt7qKuWc+QBNdHw4Etch79KD3T+f8AxR9KD3T+f/FZf6UnO6cOu2XoxEKE/wCCSVWHzAqS043G8nYuHhmOcRTIUZscymfZkA8UJrFHGzbkaT6UHun8/wDiu4eIamA08/OlVTWffX40SxxpmKbse0UUVzFgooooAKKKKACiiigAooooAKKKKACiiigAryvaSXx+sb+egrG6HhDU6HVFZ7NGaXUV+B9TQ0Vns0Zo1B8D6mhr2s7mmvCu4f2v9hWqViTxaVdl2iiimJBRRRQAVT4n3PmKuVnfSrj8UMTYdHkGD2Qb2sZ3zjONs86aLqSMfBW4jerBFJM/djUucc8AZwB1J5D41P6K8BKf1q5Aa7kHtHmIUO4gi91V2yR3myT0Aw1/6UC5EUPZFA1xb6jq1DT6zFkEYGx2Hzr67T5Z3wLCNchVHjPCYrqIxTJqU7g8mVhydGG6MDuGG4q9RUhzE8HlkVpbWdtcsBA7TYdrEwJikIHIkBlb8UbY2pzZ99fjSL0pvlg4gshUsDa6ZNIyVHbfVsfIFpB/iNN+E3aSsrIwYZwfEHwI5g+RrpjK4EWqkaOiiiuYsFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFJL/wC0b+egp3SS/wDtG/noKWRbB5hZxTiKQJrfJyQqooy7udlRF6k/88hXFtwK8uBqnuPVFPKG3EbSKP1k0ispPkigDlk8694FAJ7+aVt1tQsMY6CWRBJK/mdDxKD0BbxrYURQZMjukZST0TmXeHiE+r3ZlhkjPxARXH+FhVWz4hIsvq11GIp8FlKkmOZBsWiY77ZGpDuuRzBBO1pB6b8PMtq7p9tB9fC3USIC2M+DDUh8Q5rWhY5GmR024T3D+1/sKzg4tF2aSM6qHVXUE7kMARgczzq/w3ijspEFvI+/efEadOre0fkppY8lszWk0NcSyqoLMwUDmSQAPmaVm0upPtJ1iHuwIC3/ALkmR+SCu4vR+AHU6GVueqZmkIPlrJC/ICnOU5PpBE20Iec/qlyv75wg/OvNd3JyWKAeLEyP+S6VH5mmwGKgvr1IULyNpA28SSeQUDdiegG9AC5uCKRm4mlmxuQ76E+aR6VI/azSPi9wpgKW8YjtmIQFFCtcMfuQjGynfMh6ZI8abSwtMDLd/VQKNQgJG4G+qcjY+OgbDrnpEkLSj1qQFekMZ27OM/eI6O+2fAYHjlo7tGPgzFp6HqI5kZ95Y9GV/smzqypPPDBCCfdrXeinHfWEMcuEuosLPF4NjaRPGN+an5cwRUNUOJcIjmKuS8cqdyaJikiZ5gMO8p6qwKnqDV54k1sSjOuTZVXv72OGNpZXVI0GpnY4AHnWRJv1wq38ZA6y2ylz8SkiL/lrkcF1usl1NJdOp1IJNIjQ9CkSALkdGbUw8aksUh3NHnC1ad5ryVCvbhUSNxutsmrs1YHkWLyOR01gHlUUdgsUyBmZFJCxXCnDxnpG5Ozr7uoEdD0p3XSQLIdDqGVtip5EHpV3FKNE1LcsevTQbXCdpGP7aJTkD9ZHuR8VyPIU0tblJFDxsrqeTKQR+YpVZztbutvMxZGOIZW5n9VIfeHQ/eA8RvLdcFUsZIWMEp3LJjD/APqIfZf497wIrkLjWikw4s8O12gUfp0yYj+0O9F88j8VN45AwBUgg7gg5BHkaAOqKKpcR+78/wDasbpGN0XaKTZopdYuoc0Uk1jx/jXVGsNQ5opLmrNj3vlQp2apDGiiinGCkl/9o389BTukl/8AaN/PQUsi2DzC70RfRdX0J5s8dyvmjwJCfye3b4ZHjWsrF8Us5C0dxbsq3EWdOruSI2NcUmNwraVORupVTvjBYcP9M7dsJOfVJuRhuCFOfwPnRKPNSfPB2oixckGnZpKVelN+ILS4lO+mJsD3mIwijxJYqAPOo730qs4hl7mLyVXDM3XCouWY+QFZ+4llv5UkkjaG1ibXHE+0k0g7skq/cVeaod84Y4wBWt0LGLk6RnLLhUlkyLqfDIvtx6dWoKNS4ZSGxudOxI5csVtuCvcNHrhnhlTJ2eNlbO3NkbAPkUri8tllQo3I9RzBG4YHoQd81Bwi3YswV+yu13LY+ruI9gGdOvgSMMpPPBwVTsrkjpQ5+kbhPtLQnzhkV/4PoP8ACvR6RQDaQvEf1qOg/eYaf413YcV1N2Uq9lNjOgnIcDmY2++P4jqBUV5xFpGMNsAzjZ5W3ji8j77/AIB8yOrkDu647GAoiInkfuRxsDnzJ5Io6sf4naix4W2vtrhhJN0x3IgekYP8WO58htUcHoxbhfajDuTlpW77MeZ1LjT8BgDpS6+4ee0FtbTzocZlPaF1jjORj6zUQzclwRjBPTcAvN/W5cf+WibfwmlU8vNEI38WHgN2HE+58xS+2srqBVSJ4HjUBVVo2jIA6alZh/lql6RcZnht3aS20kcmWRXTVyXOdLYJI5Cmi6aMfBNSnhtrLxHMnaPDZZIQxnTLc42L6+cUWc6dOGbnkDGUl/6SJc2xhXUkszxwFQdwssqRuyMOeFdj0Ir6jBEqKEUAKoCgDkABgAfKq5MnRCQj3ECegvDgMGygc9WkQO5+LvlifMnNVL30SMOZOHuY2G/qzszW8n4QGyYD4MmAOqtWtoqKbXBQyHCuICePWFZCCUeNu9HIpwyN5g/IjBGxplZ99fjSXisqW3EZCzqiXEKSHOw7aNjGTnxZDGP/ALVWrXjlsHUmePn7wrpUriRaqRpr20SVGjkGpWG4/iCCNwQcEEbgil3D7t4nFtO2on7KU/2qjo3hIBzHUbjqB3/SS0/vMX76/wDWq3EeLWMyFHuIsHcESAMrDcMp6MDuDXKWHpFKJOC6CXtX7FjuUxqhY/ijyNPxQqfjVLhXpVDvFPcRF05ShgFlXow6K3ivzGxph/SS0/vMX76/9aAOYuM6CEuk7FjsHzmJj+GTAwfJgD8atcR+78/9qqy+kFmwKtcQkHYgspBHgQedY/0o4nFbxD1G5BDEqYlYOqAjvJk5j+A9nyrGrRj4NPe3aQxvLIwVEUszHoAMk0u4fwWa9Ha3jSQwtvHaIzI2nobiRCGZjz0KQq5wdR3pFw7jHr72ls4Ge31zDo6RRvKhHxkWPI8j0NfTqWMTIoz39BuHf3G3znOrsl1Z8dWM5+dL+IcFmsh2tmzywrvJaSMzto6m3kcllYe4xKtjA0netjRTUMZ+yu0mjSWNgyOoZWHIqRkGr1h3vlWe4HF2U17bjZEuNaDwWaNJWHkO0aUjyNaGw73yqaVMmlTGVFFFVKBSS/8AtG/noKd0kv8A7Rv56ClkWweYr0lk4us+pLe2kvADhiip2QPIjtJWVGIxghSxHWpLm1N5cCzyRAiiW5IJBcMSI4QRyDaWZsb6VA+9W1ghVFCIoVVGAqgAADkAByFYoj5MtOkYSMtbHJ4XJEvMvCsD426rE2s/JTTSwvo50EkTh1O2R4jYgjmCOoO4rV1jvS7hwtmPEYRp049aQbLLDkAyEcu0jHtauZUMpz7ONcRYZne5eqWWyEkOoOI5I2LpKeSEKM6vFSNmHUfI1TurtI11sduQxuWJ5BQO8T4Ci0hEi9reMscKtlYGYBc7YaU5w55YXkPM8ljyPmfynEM54moQqIo0KmQ85C/MdkeaIeYk5kHbHOrtsXsVCMuu2HKVF9uMfrVUe0PxqPiOtVuIXweRJrNJJZV9klUIjkjzujO+FOOakZIPkSK7+lZ5ITcF4reIZzgNLKCDgrg6Qr5204bfbeqHKM+JcVCxqYtMjybRAHZjjOokckA3J8B44qlYX1vbgoJDPMx1SGNS7u55khAdI6AHYAAUntPRFtInbDyksxt5cdnpYg6DoACPtkkKVz0POtNwe9iYGNE7J070JAVl+Q2I8GXINAEPrlzJ9nbrEPenff8A9uPOfmwpJ6UWrNGYZbh5ZXHswxKqIPxsN2Cg9Wb4ZNPLriTyOYbXBYHDzHeOI+H6x/wjl1I6+HhyQxHGWZiC8jbu58WP+gGAOgFNFW0Y+D5zD6LSRwSyFdU8bLJEgPeMMiyjB/GU0/A+dfVOGX6XEUc8Tao5FDqfEEZHwPlSalUEU9m7PaBZIXYvJas2nDndngfBCknco3sk75Uk5rPFtsJGfc3FFZhPTWMbSW15G3uG3Z/LvRak/wA1U7zi93dAxwxtZxnZppdJmI69jGpIQn33OR7lSUW+g9ogu0S8vZ3ZQ8UKLbKfGXUZJip6acxLkdVcdKs20M0LqU+vTPcbAkH7LnZ/g2D+Kp7CySGNYoxpRRgDJJ8SSTuSTkknckk1ds++vxro0VElqtl7h/EYpshdnXvRsNLr8VO/z5Vd0DwH5VV4hwyObGtfaHddSVdT+FhuKo67m37wNzF7y4EyjzXZZfiNJ8jXKWLvE+GpMmk+yQdSOuNSOOTL8PDkdwdjVfhV4STDOqrOgycD2ZF5CRPI9RzU7eBM0fGYGjMvaqFXZix0lT4MGwVPkRmld8JLzQYUMIRtSXMgIYHr2cexII2OvAI6GgBxf3kUK6pGVRyG27HwUDdj5Csb6apJcxKOy7JdX1SvvLK+MABF2jGCTliTjmBTPhlxFCHe4DG7TCvqOuRydl7HYey55BQANwcYNWUs31CefHasCAgOVhTb2F8T7zdT5AVjdIx8GG4Tw1rJYr6UFTFcfWjwtyrwux8lZxIT7qV9ZVs7jlSV0BBBAIIwQeRB5g0lsjc2ACQxm6tR3YtQWeBfcjLkLMg6BmUqNstsKWMu5ikbWvGOBk8qzf8ATOPl6tea/c9Wkznw1dz/ADYpfem5vxolQ2tqe9FqVp5192QoSsKHqFZmYbZXcFm0bZ76OS9s1xdjuXE2qP8A9FESFGHk2hnHk4rRWHe+VVkQAAAAADAA5ADkBVmw73yqadsRPcZUUUVUoFJL/wC0b+egp3SS/wDtG/noKWRbB5hf6F47fiOe96yn7nqdvo//AL/jWrrDXF36lc+tnPq8iCK4xv2ekkxT490amV/AFTyU1topAwDKQykZBByCDyII5itXAmRVJndUeOBfVp9fd7J9Wfd0HP8ACr1Y/wBMeIC4J4bCdTOB6yw5Q25xqVj0eQeyq88Fm6b6Klbox/B+3+q7btNRiTs9JjDFNA2Uvtq97BDfKthwiS0j9qWGRHz9pNG7Ecv7Q6lH51YuLVJF0OoK+HhjljwI8RXfD55bZfbDTQZ74GZY9h31H2i/iHtDqDzCJ7lskaiOrTiUMo+rljf9llOPyO1ZiZS0xvkTVbK4YoM5kKgqblV5HTtgc2C6huFzbv4Ib6RY41jeMYeWcBScc1iRh1bYsRyXzYYmv+Hw2yalmmhGwVY5GbU3RUjfUGPkBTkB4LlNHaal0adWvI06cZznljHWs/dWxvypUNFCudM49mZ8/o+qIfE97wxvSM8NuYezMzL6uZC3ZSjKxOxBTtezwMFvDKqzcuo1Pr9yn2lpq84JVb+EgT8qAILK5NoFhnVVjHspOgxGfASD+ybz3U+IO1M+Jn6v5iqb8ehwVmWSLIwRLE4XHUFsFD+dKL2ZIImltpkkt13aHWDoH6ps7fsHbwxypo7NGPguUsueKEym3tojPOACyg6Y4geRmkwQmeigFjzAxvS5fSkPbSyxr9ah0pGT3ndgkP7zMoPnnyrZ+jnBltIBEDqYkvJIe9LK27u3mT+QAA2Aq88lLYnGHcSLwXiLbm5tIz7gt5ZAPLWZlJ+OkfCqt3eXFpvexIYet1AWKJ5zRt7US/iBcDqRzrc1yygjBGQdsHrUVkkh9KM6rAjIOQdwR1FT2ffX40j4bb+q3E1kPslVZ4OfsxSFw0YPgjo2B0V1HSrUPEy0gW3TtWzjVnESn8T4OfgoJ+FdGpONkqpmtkkCgsxAA3JJwAPMnlSg8Veba0QMP08mREP2B3pflhfxV1HwXWQ90/bMNwmMRKfwx59o+bEn4U2ArkLiFvReNz2ssjvPsROMKVI5aFA0gDJ2IPnmup+Jy2ozcgSRcu3jGCM7APETzJwMoTk9BTi6uFjRndgqqMljyApXY27TutxMpVV3hhbmv6yQe+RyH3QfHNAC+3smuwLwSKso+wAIZYh1WTHeLfe8OQ5ZN6PiHarhl0SodMkZ5q23I/eU8w3UfMCxd8GUsZImMMp5umMN+2h9l/nv4EVmvSi8kgCzTIFlT2Vmj3jmHPs5FJ1Jnc9QDuD0OSVox8D6k/0rJNI0VlEJijaZJnYpAjDmoYAtK46qowORYHak8PpA93AsKHs555/V9S844yGkZ1z1ESOAfeAr6Dw6xjgiSGJAkaKFVRyAH886SMe4qj3M59EcS5+tWmfc9Vlx+96xn+FQ/S0kDrFfRCEu2mOZGLwOx5KWIBiY9FcYPIMTtWyqtxGxjnieGVA8bqVZTyIP88+lM4obShdViw73yrP+jUrhJLeVi8ltKYGc83XSrxOfMxPHn8QatBYd75VNKmIuRlRRRVigUkv/ALRv56CndJL/AO0b+egpZFsHmK5FJ04P2GWtbiS1XOTGuhocnniOQEJk7+wVzU/Fb11aOCBQ9xNkIGzpRVxrlkxvoXI2G5JVRzyL1l6FW+z3Q9cl5l5wGUH9XF3Ih4YGfEk71kUymScVs1YqVJ59m4k7L1W3WKPI8Cy6mH+Eg1f4dw+OBOziQIucnHMsebMTuzHqTuav3nofYyc7WJWHJ41Ebr+y6YZfkaRypLYypFM5mtpW0RTtjXHIe7FMRswb7snj7LbkFhpi45xvgbVI18yoIYcGeQnTncIoA1SN5Lnl1JA61UvboRIXOT0CjmzHYKPEk1BwNpG1iLBmc4muDvHCByjj/SMuTsNs5LHkCR5GzPaiWe19UZEs2Zp2wWhO6yDPtSy/ojz9sc+WG5Vb4AVeQtOT62BujjHZqekIyQU/GCSep6BpYWMcCnHMnU8jnLOfedjz/wBB0pXxS/tZ8KC0zqcq1uGZkbxEi+yh+JAPWnOUezwq6lGAZWBBB5EHYg0p4fKbdxbSklD9hIeoH9kx95RyP3gPEGln0teRKomRIkJI9Zk9rSPu9qkTaUJ97Vp8cUwn4B264uLiSUHcBSEQHoyhN8jmMk0AX77i8ER0ySqG9wbufgi5Y/IVlfS2L1mBzFZsDjUJnREbY5IVT9YSQMbgc6d+jqrETbNGkcqDOVUKJk5CQefRh0PkQSx4n3PmK2KtpGPg+Rj0feCJbqU6dE0EjJ4RrcRs7P8ABQWx5b+FfaKzFzAsiNG4DKylWU8ipGCPyNVOA8e9V02d6+nThIbpz7EyclV35JKBgEHGrGRnJApkx1wLGV8myorwHNJuP+ksVtiPeW4YfV20e8jnxI/s18XbCjx6VIcz3pNZrc8Q7PUwEVqC+nHOSY6FbIII+qc4I8Ku2XDGDKq3MqjkMCHb4fV1FwazdA8kxDTzN2kpGdIOAFRM/dRQFHjgnmTTaz76/GulQqJFytlj6Jk/vlx+UH/armXhzqpZr2dVUEknsAABuST2WwpszYGTsB1pFGPXWDEf1VTlQf7dhycj9GDyH3iM8sZ5ixStOFS3X1klxOIQwaFWEQZiOUjL2eMHmqkZGx2OwbfRMn98uPyg/wC1TWqPEOKxwkKxJdu7GgLO3wUb48zsOpoAg+ipP75cflB/2qxn/wAQ4Pq1QXMkzo2to2CHQuD7R7OMaf8AEa14guLj7Qm3j/RoQZSPxyDZPgmf2qklsI4kCRoqrvkAc+WSx5sfM1jdIxnzv0V4W1tcWVxKCBLK0YX3dUEpRm8CSMAfiHjivrNZjjHDhcRNESVJwyuOaOpDI6+asFPyrrgPpOGYW15pguxtpJwk+P7S3JPtg8yneXcEcicjKzE7NLRRWa476TBWa2s9M12dtIOUgyNpLgg+wBzC95uQHMhhipwx9d3fuO72yRg9CyW8eoj4FtJ81p9Yd75Ur4Pw8W8SxAliMlnPN5GJZ3bzZix+dNLDvfKpXcifUZUUUVUoFJL/AO0b+egp3SS/+0b+egpZFsHmF3okmu6vZjzR47ZfJVhSZseGXnOfHSPCtZWP4BOIL+eFthdBZ4z0MkcaxSoPPQkLY6gt4VsK1cE5+ZhSn0rsBPZzxHbVGxU+66jUjDzVgpHmKbUh9NuIGG0kVPtZh2EK+MsgKrt4KMufBUY9K0UwFpxV710YqcKg+rV0U6mUayWLbZ3GVycbDma3PCLO4MekPHboDgLGupgMDkz4UfumqkXDIhGkTIrqiKg1KDsqgDn8KvcN4FHpzG0sRz/ZyOByH3clf4Uie5fJFqO5bT0fhJzLqnbxmYuM+Sn2R8hTREAGAAAOQGwpV6jcp3LkOPCaJT/mjK/6Gj1u7Tv26SecUu/7sgUf5qcgNmGdjypM3D5Lf2rUao+ZtmOB59ix+zP4T7J/Dzrv6fRftY54vN4nK/N0DKPzq1Z8Wgm+ymjc+CupPzAORQBRlZLxNUTFJojldQw8T47rqd9JGxHIjkeRr1OIdtEQy6JUYLJH1VvLxUjcN1Bq1xHhSykOpMcq92VO8PI9HX8J2rO8amkRlaRQk49kOu0Nymc6CT3H6hW5HOCQTTQ8yMfBfqvxGWJY2M5RYse0ZCoTHnq2qrBx6BkkkD4WJNchII0jBJz8NJzU/o1wIzFb28TMre1DA262yHu+zyMpG7OdwTpGw36J5EkSjFszUa2GPqYLzs//AJeG/EP5RKEI+GRTb0bkssOlmI1IwZEVdMgJ5GRWAcHzYVvaUcf9HobsAuCsqbxzpgSxHxVvDxU7HkQakstPgdwKdTWffX40o4NdyN2kM4AnhbRJp7rZGpJFHRXUg46HUOlR3F2sjFWcR26nEspONZ6xof4MR8BvnF3JONk0tx059dbSP/CqcMf07A7qP1YPM/eO3LOWd9fxQKDIwUclXmWPuoo3Y+QFLoriaUBbaPsIgMCWRcHSOXZxbEfF8fA1dsOExxHXu8p2MrnU58s/dHkuBXGXKuu5uO6DbR+8wBmYeS7rF88nyFXeH8NjhB0Lu3eckl2PizHdquUUAFUuI/d+f+1XajmhDc+lY1aMfApqC+so5kMc0aSIeauoYfkacepr5/nR6mvn+dT0sTSzKf0VtcadD6P0fbTaP3NenHlimVjZRwoI4Y0jQclRQo/IU59TXz/Oj1NfP863SzdLF1WLDvfKrPqa+f513FbqpyKFF2CiyaiiiqDhSS/+0b+egp3Si8t2LkhSR/xSyLYXUhPxXhq3CBWLKysHSRDh45B3XQ9CN/IgkHIJotfSC7gwlzbm4A2E9toBI8ZIXYFT+wWG3TlTL1V/dNHqr+6aVNorKMJdSnP6WOdoLG4dj1k0RID+JmbVj9lWqla2EjzC6u3WSYKVjVAeygVsaljzuxOF1OdzgYCjanPqr+6aPVX901rbZkYQjvZDTbhPcP7X+wpf6q/ummXDYyqkEY3/ANhRHkMzTiW6KKKc5QqpecNhl+1ijf8AaVT/AKirdFACn+j8a/ZPNF+xK+n9xiV/hSj0q4fc+rSKs3bBsLoeNdZycbMpAHxIrW1T4n3PmK2KtpGPg+PDgtxCF7VQkUs0EUgJU5R7mIYwCefL4E19trHccsDPBJEraWIyj+7IpDRt8mVT8qd+jXGRdQh8aJFOiaIneKUd5D/qD1UgjY0+SGkWMrG1FFcySBQWYgAAkknAAG5JPSpjmF9K7aQ34WBgjTWoEjHOyRzHBGPvfXMPgfKr/BeDpG8ZPtsuylsYX9heSfLfzNU+FXHrM819uI5AsUGesEZY9pjp2ju5H4QlPLPvr8a6Ix+Tck38w9ooornKhUVzOsas7sFVRqZicAAcyTUtLvSGxae3kjQgMQCurkSrBgG8jjB+NZJtLYGRR+kNuyyNrYCNDIweORW7MDJcKyhmXbmoNTX3Fo4xvksY2lVQrEsq6c90H3l257+VJuIcOnuy5aPsP6tLCoZlYs8oUZ9gkBRp67nPIV6lrcSyxs8PZKtvJFu6Mdbdng+yT7PsnB5+IFTcpUJbLNn6UQmKFn1h5IxJ2axTM2NgxVQhZlB64wRg8jmmVvxWKRkVHDGRDImM4KKVBOcYGCyjB3/Ksxw1p4ZbdTblnSz7NkDx59l1UMpJ0kHGeYOD8qsWfC57doZRGJSFnDorqNJmmEw0l8AquCvj1xQpsxSZff0iQzQxoGZZTIpYJJs0baCNlx3gck7ADPIg1cvb8pNDHgaZA5YnpoUHb86R8N4ZPGbZ2jBKS3HaBWX2RNIWDAkjUAOnPflTXi9pI80LxgewsoycYDMgCZHMjPhQnKrf0/RqbaOrb0it5GCq7HILKTHIFcAZJjYqBJtv7JNWBxaIrEwfab7M4PtDQXydvZGlScnHhzNZyw4ZcdtaySRyfVlu1aSZWGTC65jjU6QuT0AbcbYzXHDuEmVrqLUOzjWSCFhuB2wEj/u5Rfhms1y7GKUupobPj0EraUc5wWGUdQ6jmY2ZQJAPFSa74VxiK5BaFiwGN9DqDkZGCwAbzxy5GkHDuDSaotcLqYlbLvcySLrMbRjsU1kAHUe8BgbY8H3o7bNFawRONLJEisMg4YKARkbHemi5Pk1NvktXVxoXVpZvaVcINR9pgucDoM5J6AE1NSH00/8ADr/9Rb//ALMVZtuHxiC4uNP1y376ZPvL/WgCFP3QcnIHPJzRKbTqjvw+GjPGpN1bri+31R9CzRmvnHEPV83w/wDOdu3q/wCk16I9PY+A1d7Hnq2qbjN6kUfFIZTiWX2kTBy4NrEhZR1AZWBI2GDml+KUXgG2km966d9O/PG/P0NzDeq0jxqTqj06tuWoEj47CvIb5WkkiGdUYUttt7YJGPHkaz/AI4UvrgaUWVo4nXYByhUByDjJGoLnzxVHj8yrJealypNsrZZlQA6smRlGdG2/iNjtWubSv7iLwsZTcFfEX/t6fVm3or5zw6zEpvYoDFtHBPD2KlYu1VpWVkBJByUUFl2OPKu72R7iFr0DSk00SNqVzptYwwIdVIJUysxbBHs89qz4u3Hvgp/gLVWrt07pNdfv+D6HQK+btAjQzCOSNoGmtVxboyQhjcoGMTayMkEZ04AIHXNfQbCyjhQRxIEReSryGTk/xJpoT1EM/h44l5r37V0T/f8A0sUUUVQ5QqnxPufMVcqvewl1wPGmi6kjHwJKXXnC8ydvDI0E4AXtEwQ6jcLKh9mReeM7jJwRT36Nfy/Oj6Nfy/Oulyg+SKUkJ04nxFRgrZyH38zR9OejD/8A5VUueGzXRHr0yyRg59WiUpCfDtMsWmx4Ehfw1o/o5/L86Po5/L86RLGhrkUwKms++vxqb6Nfy/OpLexYMCcbGmlONPcVRdjOiiiuUuFFFFABRRRQBH2C6tekasadWBnTnOM88Z3xUlFFABRRRQB4RUVrapEoSNFRRyVQAB12A2qaigAooooAMV5ivaKAKVhw9YjKVJPayGU5xsSqrgYHL2RVzFe0UGyk5O2eYoxXtFBhW4hbGRGRZHiJ++mnUN+moEb8uXWurG1WKNYkGFRQoHkBj51PRWV1G1OtPQ8xXtFFaKFFFZni3Gpor5IUjkmVrZpOyjEYOsSqNRZyuNiRjPXlVMeNzbS+4NmmorLD0riLQzF5Eia2mnZGRcARNGG1/eDLkjC7HfwFRD0nkN1CJI5beEwTTMJRH7Sp2RVsqzFSAzZXY771ReFyPp3/AIv0M1I11FZFPSeSS4tB2UsEMqyyEyiPEkax6lOxJjI2Ok4OD8at2HpYkhizDKkc+exlfRpk9kuuwYsmpVJGoDOPHaiXhsiV1739GFo0dFZjhPpis/q59XmjjuDpilcR6WcKzacByw2R8EjB0+Yzb45xCSO5skRsJLLIsgwDqVbeVwNxke0qnbwpXgmpaWqe7/F3/QWh5RWdg9KMvErW08azFlidwg1sEZwCmrXGSqsRqUct8VJD6URMls4Vh6xq2OMxhEZpDJvsFK6Tz9ogVjwZF097+j/AWh9RWesPSpZGiDQyxpPkQyPo0yHSXAwrFkLKrMNQHLx2qf0Z9IPXU7VYJY4yoKPJow+c50hWJ2IxkgZ6ZolgyRTbXHv9MLQ6oooqRoUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFFFFABRRRQAUUUUAFK24a3rgucjSIDDp3zkyK+fDGBTQ15Wxk43XUDGD0JLIsckg0+r3MDac5+vlVwy5GPZC9etTzejtxcOvrckTRi3ltyIlYFu1CKXySdJwvd6eJ6ayiuj/Ly8336d+RdKMpH6P3Lvbi4liaOFHjOhWVpNcRj1nJIQ4+6NtzvyAqcE9CeweH2LMCHP1qWyCebClU1sQdB3yzKcsR0GQdtRWPxeSq6fb7+r90GlGYs/Rp0g4fEXUm0kV2O+HAhlj9nw3kB38KYca4S00ttIrhexeRj4+3BJENPmC4O/hTeileeberrv8Azd/2bRhuG+hckclrK3q2u3fU8wRzNP8AVuhZ5GyQfazpyQT12FHozwpJ57yVWL2za4odiAO2CvdFD1BkA38QwrcGiqPxmRp3y9vtvfv6NoxRRjOB+hZgkhJSyAhz9bHbIJpvZKrrYj6s75YqcsR0BIrQ+jHDGtbSC3ZgzRoELDOCR1GaZ17U8niJ5PN759WakkFFFFRNCiiigAooooAKKKKACiiigAooooA//9k=" + } + }, + "cell_type": "markdown", + "id": "b7f37088-3ee3-466b-974b-bdf7a1df0d25", + "metadata": {}, + "source": [ + "![images.jpg](attachment:ee0672be-bf08-4422-b9a4-24bd40eea517.jpg)\n", + "\n", + "Now let's define a layer and a MLP (multi-layer perceptron)" + ] + }, + { + "cell_type": "markdown", + "id": "7e885e5c-3de4-4d81-89c5-02423da45778", + "metadata": {}, + "source": [ + "Let's train a NN! (finally!) This is a simple binary classifer NN" + ] + }, + { + "cell_type": "code", + "execution_count": 240, + "id": "58ab19a9-8f0f-41a5-b270-95f31a1127e4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=0.5295324547878887)" + ] + }, + "execution_count": 240, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = [2.0, 3.0, -1.0]\n", + "n = MLP(3, [4, 4, 1]) # implementing the example image\n", + "n(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 184, + "id": "78f114fb-4736-458e-9eb0-3e03405dc63c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "41" + ] + }, + "execution_count": 184, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(n.parameters())" + ] + }, + { + "cell_type": "code", + "execution_count": 177, + "id": "e095ac0b-5aa5-4224-a2d7-67a5af1c9742", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Value(data=0.07700064719309661),\n", + " Value(data=0.6874724332729629),\n", + " Value(data=-0.7191542943909137),\n", + " Value(data=-0.12016611563222777)]" + ] + }, + "execution_count": 177, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# when we feed the first inputs, we should get the first output (1.0) and so on.\n", + "\n", + "xs = [\n", + " [2.0, 3.0, -1.0],\n", + " [3.0, -1.0, 0.5],\n", + " [0.5, 1.0, 1.0],\n", + " [1.0, 1.0, -1.0],\n", + "]\n", + "ys = [1.0, -1.0, -1.0, 1.0] # desired targets\n", + "ypred = [n(x) for x in xs]\n", + "ypred" + ] + }, + { + "cell_type": "markdown", + "id": "c9676e56-fff4-4d14-8120-955a9a9622ee", + "metadata": {}, + "source": [ + "We want `ypred` to go as close to as `ys`. We do this (and in deep learning) by calculating a single number for perfomance called the *loss*. We are going to implement a mean-square loss" + ] + }, + { + "cell_type": "code", + "execution_count": 178, + "id": "584c98fa-5fef-4310-88b1-3638b41821e8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=5.033137455307795)" + ] + }, + "execution_count": 178, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n", + "loss" + ] + }, + { + "cell_type": "code", + "execution_count": 179, + "id": "7e1b37ad-e268-4cbb-8f98-fa7e5daf44c8", + "metadata": {}, + "outputs": [], + "source": [ + "loss.backward()" + ] + }, + { + "cell_type": "code", + "execution_count": 180, + "id": "4768716a-2ff9-4e50-b7c1-fff9691b2ef5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.7368026977666375" + ] + }, + "execution_count": 180, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "n.layers[0].neurons[0].w[0].grad # this is now NOT zero!" + ] + }, + { + "cell_type": "code", + "execution_count": 185, + "id": "1eb237df-ca08-4e5f-b22f-1c6d4d10341e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-0.32973208920024066" + ] + }, + "execution_count": 185, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "n.layers[0].neurons[0].w[0].data" + ] + }, + { + "cell_type": "code", + "execution_count": 187, + "id": "fcfcb22d-b973-45b5-aee4-78d7d22e6d5b", + "metadata": {}, + "outputs": [], + "source": [ + "for p in n.parameters():\n", + " p.data += -0.01 * p.grad # negative sign because we want to minimise the loss. We think of grad as a vector pointing towards increased loss. This is gradient descent.\n", + " # https://en.wikipedia.org/wiki/Gradient_descent#An_analogy_for_understanding_gradient_descent" + ] + }, + { + "cell_type": "markdown", + "id": "033750e6-d9ca-4ce7-9582-e5a2841f953a", + "metadata": {}, + "source": [ + "Let's look at the new loss (It has decreased)" + ] + }, + { + "cell_type": "code", + "execution_count": 189, + "id": "a78061fa-fca6-4a04-9a15-ebe3908dc021", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=3.7015356808242585)" + ] + }, + "execution_count": 189, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ypred = [n(x) for x in xs]\n", + "loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n", + "loss" + ] + }, + { + "cell_type": "markdown", + "id": "5922cb79-0965-4f6b-b6c4-e30253cc57cb", + "metadata": {}, + "source": [ + "Let backprop this loss and update the weights" + ] + }, + { + "cell_type": "code", + "execution_count": 190, + "id": "3bd72e52-5dec-4e1b-84d5-b7435f90d19f", + "metadata": {}, + "outputs": [], + "source": [ + "loss.backward()" + ] + }, + { + "cell_type": "code", + "execution_count": 191, + "id": "3dd28be9-dc33-415b-bf2c-3da6c8d59160", + "metadata": {}, + "outputs": [], + "source": [ + "for p in n.parameters():\n", + " p.data += -0.01 * p.grad # negative sign because we want to minimise the loss. We think of grad as a vector pointing towards increased loss. This is gradient descent.\n", + " # https://en.wikipedia.org/wiki/Gradient_descent#An_analogy_for_understanding_gradient_descent" + ] + }, + { + "cell_type": "markdown", + "id": "c2b0b435-a7c0-4ab2-b907-55709bfb7342", + "metadata": {}, + "source": [ + "Now let's find the new loss" + ] + }, + { + "cell_type": "code", + "execution_count": 192, + "id": "8cc95aa5-ecb4-4a24-b631-1daf8e9f7b78", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Value(data=3.193172494954373)" + ] + }, + "execution_count": 192, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ypred = [n(x) for x in xs]\n", + "loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n", + "loss" + ] + }, + { + "cell_type": "markdown", + "id": "3faa653d-3787-4ab3-9abc-ea8df46153a1", + "metadata": {}, + "source": [ + "So the flow is forward pass -> backward pass -> update" + ] + }, + { + "cell_type": "code", + "execution_count": 241, + "id": "f530b51e-9e4b-47b2-9fb0-335c48c3c424", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step # 0 6.063052689312497\n", + "Step # 1 3.4913909241492496\n", + "Step # 2 1.2084297159180784\n", + "Step # 3 0.4656671369410741\n", + "Step # 4 0.24785984660117516\n", + "Step # 5 0.17205836536367008\n", + "Step # 6 0.13140359668124182\n", + "Step # 7 0.10607370364195885\n", + "Step # 8 0.08881662744244229\n", + "Step # 9 0.07632247962333527\n", + "Step # 10 0.06686857143197884\n", + "Step # 11 0.05947134159212191\n", + "Step # 12 0.05352900687658706\n", + "Step # 13 0.04865302737738714\n", + "Step # 14 0.04458150502268218\n", + "Step # 15 0.04113156601356174\n", + "Step # 16 0.03817169980897338\n", + "Step # 17 0.03560494239910917\n", + "Step # 18 0.0333582524439753\n", + "Step # 19 0.031375575848589486\n", + "Step # 20 0.02961318784557092\n", + "Step # 21 0.028036486100278167\n", + "Step # 22 0.026617733994873168\n", + "Step # 23 0.025334441397667242\n", + "Step # 24 0.024168182466436462\n", + "Step # 25 0.023103718921689623\n", + "Step # 26 0.022128340591535754\n", + "Step # 27 0.021231362959825167\n", + "Step # 28 0.020403739813356792\n", + "Step # 29 0.01963776138740571\n", + "Step # 30 0.0189268167944545\n", + "Step # 31 0.01826520532701045\n", + "Step # 32 0.017647985303954276\n", + "Step # 33 0.017070852033527407\n", + "Step # 34 0.016530038559047344\n", + "Step # 35 0.016022234379549593\n", + "Step # 36 0.015544518462321941\n", + "Step # 37 0.015094303701612632\n", + "Step # 38 0.014669290606996854\n", + "Step # 39 0.014267428481870884\n", + "Step # 40 0.01388688271714819\n", + "Step # 41 0.013526007106099663\n", + "Step # 42 0.013183320304248127\n", + "Step # 43 0.01285748572855482\n", + "Step # 44 0.012547294324111991\n", + "Step # 45 0.01225164973260155\n", + "Step # 46 0.011969555481211223\n", + "Step # 47 0.011700103878302559\n", + "Step # 48 0.011442466356546074\n", + "Step # 49 0.011195885048268026\n", + "Step # 50 0.010959665413551208\n", + "Step # 51 0.01073316977087405\n", + "Step # 52 0.010515811604061303\n", + "Step # 53 0.010307050539087767\n", + "Step # 54 0.01010638790062819\n", + "Step # 55 0.009913362771829845\n", + "Step # 56 0.009727548492106068\n", + "Step # 57 0.009548549537221928\n", + "Step # 58 0.009375998733897512\n", + "Step # 59 0.009209554767852004\n", + "Step # 60 0.00904889994987455\n", + "Step # 61 0.008893738209305405\n", + "Step # 62 0.008743793288391685\n", + "Step # 63 0.008598807114459579\n", + "Step # 64 0.008458538329820275\n", + "Step # 65 0.00832276096187455\n", + "Step # 66 0.008191263218074911\n", + "Step # 67 0.00806384639228904\n", + "Step # 68 0.007940323870742451\n", + "Step # 69 0.007820520227127632\n", + "Step # 70 0.007704270397693438\n", + "Step # 71 0.007591418928192805\n", + "Step # 72 0.007481819285496838\n", + "Step # 73 0.007375333227492005\n", + "Step # 74 0.0072718302255883925\n", + "Step # 75 0.007171186934788122\n", + "Step # 76 0.0070732867068092455\n", + "Step # 77 0.006978019142241778\n", + "Step # 78 0.0068852796781358255\n", + "Step # 79 0.006794969207796124\n", + "Step # 80 0.00670699372988889\n", + "Step # 81 0.006621264024259348\n", + "Step # 82 0.006537695352119681\n", + "Step # 83 0.006456207178497548\n", + "Step # 84 0.006376722915042138\n", + "Step # 85 0.006299169681467748\n", + "Step # 86 0.0062234780840792975\n", + "Step # 87 0.006149582009970493\n", + "Step # 88 0.006077418435616704\n", + "Step # 89 0.0060069272487023664\n", + "Step # 90 0.005938051082127848\n", + "Step # 91 0.005870735159236637\n", + "Step # 92 0.0058049271493877865\n", + "Step # 93 0.005740577033077032\n", + "Step # 94 0.005677636975877802\n", + "Step # 95 0.005616061210537649\n", + "Step # 96 0.005555805926620974\n", + "Step # 97 0.005496829167141412\n", + "Step # 98 0.0054390907316724645\n", + "Step # 99 0.005382552085468513\n", + "Step # 100 0.005327176274165607\n", + "Step # 101 0.005272927843666831\n", + "Step # 102 0.005219772764848314\n", + "Step # 103 0.005167678362751362\n", + "Step # 104 0.005116613249951687\n", + "Step # 105 0.005066547263821784\n", + "Step # 106 0.005017451407423288\n", + "Step # 107 0.00496929779378706\n", + "Step # 108 0.004922059593356691\n", + "Step # 109 0.004875710984387967\n", + "Step # 110 0.00483022710611221\n", + "Step # 111 0.004785584014485676\n", + "Step # 112 0.004741758640360024\n", + "Step # 113 0.00469872874992065\n", + "Step # 114 0.004656472907251062\n", + "Step # 115 0.004614970438891062\n", + "Step # 116 0.004574201400266073\n", + "Step # 117 0.004534146543873533\n", + "Step # 118 0.004494787289119742\n", + "Step # 119 0.004456105693708702\n", + "Step # 120 0.004418084426490032\n", + "Step # 121 0.004380706741680469\n", + "Step # 122 0.004343956454378153\n", + "Step # 123 0.004307817917295212\n", + "Step # 124 0.0042722759986379205\n", + "Step # 125 0.004237316061069736\n", + "Step # 126 0.004202923941695134\n", + "Step # 127 0.00416908593300756\n", + "Step # 128 0.0041357887647473605\n", + "Step # 129 0.004103019586619787\n", + "Step # 130 0.004070765951825515\n", + "Step # 131 0.0040390158013597155\n", + "Step # 132 0.00400775744903826\n", + "Step # 133 0.003976979567211711\n", + "Step # 134 0.003946671173131104\n", + "Step # 135 0.003916821615930658\n", + "Step # 136 0.0038874205641950834\n", + "Step # 137 0.003858457994081638\n", + "Step # 138 0.0038299241779675714\n", + "Step # 139 0.0038018096735966287\n", + "Step # 140 0.0037741053136988835\n", + "Step # 141 0.003746802196060106\n", + "Step # 142 0.003719891674018089\n", + "Step # 143 0.0036933653473650166\n", + "Step # 144 0.003667215053634969\n", + "Step # 145 0.003641432859758701\n", + "Step # 146 0.00361601105406707\n", + "Step # 147 0.0035909421386263116\n", + "Step # 148 0.003566218821889728\n", + "Step # 149 0.0035418340116498666\n", + "Step # 150 0.003517780808277683\n", + "Step # 151 0.003494052498234713\n", + "Step # 152 0.0034706425478457192\n", + "Step # 153 0.0034475445973193913\n", + "Step # 154 0.003424752455006112\n", + "Step # 155 0.0034022600918815187\n", + "Step # 156 0.0033800616362457997\n", + "Step # 157 0.0033581513686285923\n", + "Step # 158 0.003336523716890975\n", + "Step # 159 0.003315173251514751\n", + "Step # 160 0.0032940946810716405\n", + "Step # 161 0.003273282847863682\n", + "Step # 162 0.0032527327237279312\n", + "Step # 163 0.0032324394059978774\n", + "Step # 164 0.0032123981136149104\n", + "Step # 165 0.003192604183383576\n", + "Step # 166 0.0031730530663640144\n", + "Step # 167 0.0031537403243963964\n", + "Step # 168 0.003134661626750936\n", + "Step # 169 0.003115812746899099\n", + "Step # 170 0.003097189559400266\n", + "Step # 171 0.003078788036899308\n", + "Step # 172 0.0030606042472306025\n", + "Step # 173 0.003042634350623824\n", + "Step # 174 0.0030248745970078003\n", + "Step # 175 0.0030073213234078264\n", + "Step # 176 0.0029899709514333668\n", + "Step # 177 0.0029728199848519754\n", + "Step # 178 0.00295586500724621\n", + "Step # 179 0.002939102679750448\n", + "Step # 180 0.002922529738863839\n", + "Step # 181 0.002906142994337289\n", + "Step # 182 0.0028899393271306856\n", + "Step # 183 0.002873915687438391\n", + "Step # 184 0.0028580690927797886\n", + "Step # 185 0.002842396626152884\n", + "Step # 186 0.002826895434248074\n", + "Step # 187 0.0028115627257202106\n", + "Step # 188 0.0027963957695165496\n", + "Step # 189 0.0027813918932584844\n", + "Step # 190 0.0027665484816751626\n", + "Step # 191 0.002751862975086844\n", + "Step # 192 0.002737332867936595\n", + "Step # 193 0.002722955707367939\n", + "Step # 194 0.0027087290918473375\n", + "Step # 195 0.002694650669829513\n", + "Step # 196 0.002680718138464186\n", + "Step # 197 0.0026669292423429185\n", + "Step # 198 0.002653281772284076\n", + "Step # 199 0.0026397735641553854\n", + "Step # 200 0.0026264024977318504\n", + "Step # 201 0.0026131664955885165\n", + "Step # 202 0.0026000635220264513\n", + "Step # 203 0.0025870915820308447\n", + "Step # 204 0.0025742487202602853\n", + "Step # 205 0.002561533020065842\n", + "Step # 206 0.0025489426025392856\n", + "Step # 207 0.002536475625589039\n", + "Step # 208 0.002524130283043393\n", + "Step # 209 0.0025119048037794776\n", + "Step # 210 0.0024997974508777603\n", + "Step # 211 0.0024878065208007068\n", + "Step # 212 0.002475930342594991\n", + "Step # 213 0.0024641672771165517\n", + "Step # 214 0.0024525157162775986\n", + "Step # 215 0.002440974082314912\n", + "Step # 216 0.0024295408270786777\n", + "Step # 217 0.002418214431341317\n", + "Step # 218 0.0024069934041255226\n", + "Step # 219 0.002395876282050965\n", + "Step # 220 0.0023848616286989387\n", + "Step # 221 0.0023739480339946685\n", + "Step # 222 0.0023631341136063117\n", + "Step # 223 0.002352418508360469\n", + "Step # 224 0.002341799883673461\n", + "Step # 225 0.0023312769289979738\n", + "Step # 226 0.002320848357284638\n", + "Step # 227 0.0023105129044579276\n", + "Step # 228 0.002300269328906075\n", + "Step # 229 0.002290116410984435\n", + "Step # 230 0.0022800529525321095\n", + "Step # 231 0.002270077776401008\n", + "Step # 232 0.00226018972599765\n", + "Step # 233 0.0022503876648364727\n", + "Step # 234 0.002240670476104961\n", + "Step # 235 0.0022310370622400065\n", + "Step # 236 0.0022214863445151186\n", + "Step # 237 0.0022120172626381566\n", + "Step # 238 0.0022026287743595026\n", + "Step # 239 0.0021933198550899\n", + "Step # 240 0.0021840894975282666\n", + "Step # 241 0.0021749367112986412\n", + "Step # 242 0.0021658605225963992\n", + "Step # 243 0.0021568599738431664\n", + "Step # 244 0.002147934123350419\n", + "Step # 245 0.0021390820449913844\n", + "Step # 246 0.00213030282788102\n", + "Step # 247 0.0021215955760638706\n", + "Step # 248 0.0021129594082095427\n", + "Step # 249 0.002104393457315599\n", + "Step # 250 0.0020958968704177764\n", + "Step # 251 0.0020874688083069564\n", + "Step # 252 0.002079108445253255\n", + "Step # 253 0.0020708149687364515\n", + "Step # 254 0.002062587579183047\n", + "Step # 255 0.002054425489709562\n", + "Step # 256 0.0020463279258717145\n", + "Step # 257 0.0020382941254198286\n", + "Step # 258 0.0020303233380597743\n", + "Step # 259 0.00202241482521953\n", + "Step # 260 0.0020145678598212833\n", + "Step # 261 0.002006781726058751\n", + "Step # 262 0.001999055719179698\n", + "Step # 263 0.00199138914527338\n", + "Step # 264 0.001983781321063024\n", + "Step # 265 0.0019762315737029182\n", + "Step # 266 0.001968739240580211\n", + "Step # 267 0.00196130366912113\n", + "Step # 268 0.0019539242166017054\n", + "Step # 269 0.0019466002499626315\n", + "Step # 270 0.0019393311456283979\n", + "Step # 271 0.0019321162893303435\n", + "Step # 272 0.001924955075933836\n", + "Step # 273 0.0019178469092690424\n", + "Step # 274 0.001910791201965695\n", + "Step # 275 0.001903787375291321\n", + "Step # 276 0.0018968348589930967\n", + "Step # 277 0.0018899330911432088\n", + "Step # 278 0.0018830815179874684\n", + "Step # 279 0.0018762795937972963\n", + "Step # 280 0.001869526780724939\n", + "Step # 281 0.0018628225486617025\n", + "Step # 282 0.0018561663750993143\n", + "Step # 283 0.00184955774499431\n", + "Step # 284 0.0018429961506351163\n", + "Step # 285 0.0018364810915121475\n", + "Step # 286 0.0018300120741906545\n", + "Step # 287 0.0018235886121860377\n", + "Step # 288 0.0018172102258421216\n", + "Step # 289 0.001810876442211629\n", + "Step # 290 0.001804586794939505\n", + "Step # 291 0.00179834082414832\n", + "Step # 292 0.001792138076326316\n", + "Step # 293 0.001785978104217711\n", + "Step # 294 0.0017798604667150894\n", + "Step # 295 0.0017737847287542753\n", + "Step # 296 0.0017677504612111706\n", + "Step # 297 0.001761757240800724\n", + "Step # 298 0.0017558046499780197\n", + "Step # 299 0.0017498922768413362\n", + "Step # 300 0.0017440197150370828\n", + "Step # 301 0.001738186563666814\n", + "Step # 302 0.0017323924271959753\n", + "Step # 303 0.0017266369153644931\n", + "Step # 304 0.0017209196430992434\n", + "Step # 305 0.0017152402304281022\n", + "Step # 306 0.0017095983023958218\n", + "Step # 307 0.0017039934889815072\n", + "Step # 308 0.0016984254250177687\n", + "Step # 309 0.0016928937501113808\n", + "Step # 310 0.0016873981085655373\n", + "Step # 311 0.001681938149303642\n", + "Step # 312 0.0016765135257945742\n", + "Step # 313 0.0016711238959792873\n", + "Step # 314 0.0016657689221990137\n", + "Step # 315 0.0016604482711246983\n", + "Step # 316 0.0016551616136878556\n", + "Step # 317 0.0016499086250127352\n", + "Step # 318 0.0016446889843497237\n", + "Step # 319 0.0016395023750101637\n", + "Step # 320 0.0016343484843021818\n", + "Step # 321 0.001629227003467954\n", + "Step # 322 0.0016241376276219487\n", + "Step # 323 0.001619080055690501\n", + "Step # 324 0.001614053990352399\n", + "Step # 325 0.001609059137980624\n", + "Step # 326 0.0016040952085851895\n", + "Step # 327 0.001599161915756965\n", + "Step # 328 0.0015942589766126342\n", + "Step # 329 0.001589386111740561\n", + "Step # 330 0.0015845430451477908\n", + "Step # 331 0.0015797295042078328\n", + "Step # 332 0.0015749452196096151\n", + "Step # 333 0.0015701899253071257\n", + "Step # 334 0.0015654633584701642\n", + "Step # 335 0.0015607652594359178\n", + "Step # 336 0.0015560953716613338\n", + "Step # 337 0.0015514534416765074\n", + "Step # 338 0.0015468392190387274\n", + "Step # 339 0.0015422524562874749\n", + "Step # 340 0.0015376929089001685\n", + "Step # 341 0.0015331603352486678\n", + "Step # 342 0.0015286544965566408\n", + "Step # 343 0.0015241751568575948\n", + "Step # 344 0.0015197220829536045\n", + "Step # 345 0.0015152950443749508\n", + "Step # 346 0.0015108938133403076\n", + "Step # 347 0.001506518164717592\n", + "Step # 348 0.0015021678759856764\n", + "Step # 349 0.001497842727196576\n", + "Step # 350 0.0014935425009383841\n", + "Step # 351 0.0014892669822988897\n", + "Step # 352 0.0014850159588296608\n", + "Step # 353 0.0014807892205109291\n", + "Step # 354 0.0014765865597169416\n", + "Step # 355 0.0014724077711819657\n", + "Step # 356 0.0014682526519668873\n", + "Step # 357 0.0014641210014262843\n", + "Step # 358 0.0014600126211761822\n", + "Step # 359 0.0014559273150622164\n", + "Step # 360 0.0014518648891284985\n", + "Step # 361 0.0014478251515867807\n", + "Step # 362 0.0014438079127864097\n", + "Step # 363 0.0014398129851845107\n", + "Step # 364 0.0014358401833168212\n", + "Step # 365 0.0014318893237690515\n", + "Step # 366 0.001427960225148534\n", + "Step # 367 0.0014240527080565143\n", + "Step # 368 0.0014201665950608262\n", + "Step # 369 0.001416301710669022\n", + "Step # 370 0.001412457881301926\n", + "Step # 371 0.0014086349352676292\n", + "Step # 372 0.001404832702735981\n", + "Step # 373 0.0014010510157132982\n", + "Step # 374 0.001397289708017771\n", + "Step # 375 0.0013935486152549387\n", + "Step # 376 0.0013898275747938728\n", + "Step # 377 0.0013861264257434972\n", + "Step # 378 0.0013824450089294398\n", + "Step # 379 0.0013787831668711824\n", + "Step # 380 0.0013751407437595923\n", + "Step # 381 0.0013715175854347953\n", + "Step # 382 0.001367913539364437\n", + "Step # 383 0.0013643284546222562\n", + "Step # 384 0.0013607621818670114\n", + "Step # 385 0.0013572145733217174\n", + "Step # 386 0.0013536854827532383\n", + "Step # 387 0.0013501747654521238\n", + "Step # 388 0.0013466822782129332\n", + "Step # 389 0.0013432078793146056\n", + "Step # 390 0.0013397514285014057\n", + "Step # 391 0.0013363127869639735\n", + "Step # 392 0.001332891817320703\n", + "Step # 393 0.0013294883835995312\n", + "Step # 394 0.0013261023512197853\n", + "Step # 395 0.0013227335869745427\n", + "Step # 396 0.0013193819590130663\n", + "Step # 397 0.001316047336823662\n", + "Step # 398 0.0013127295912166706\n", + "Step # 399 0.001309428594307766\n", + "Step # 400 0.001306144219501601\n", + "Step # 401 0.0013028763414754797\n", + "Step # 402 0.0012996248361635194\n", + "Step # 403 0.0012963895807408523\n", + "Step # 404 0.0012931704536082113\n", + "Step # 405 0.0012899673343766713\n", + "Step # 406 0.0012867801038525559\n", + "Step # 407 0.0012836086440227708\n", + "Step # 408 0.001280452838040121\n", + "Step # 409 0.0012773125702089847\n", + "Step # 410 0.0012741877259711454\n", + "Step # 411 0.001271078191891917\n", + "Step # 412 0.001267983855646318\n", + "Step # 413 0.0012649046060055527\n", + "Step # 414 0.0012618403328237514\n", + "Step # 415 0.001258790927024721\n", + "Step # 416 0.0012557562805890292\n", + "Step # 417 0.001252736286541282\n", + "Step # 418 0.0012497308389374862\n", + "Step # 419 0.0012467398328526748\n", + "Step # 420 0.0012437631643686506\n", + "Step # 421 0.0012408007305620295\n", + "Step # 422 0.0012378524294922606\n", + "Step # 423 0.0012349181601899752\n", + "Step # 424 0.0012319978226454458\n", + "Step # 425 0.0012290913177972365\n", + "Step # 426 0.0012261985475209183\n", + "Step # 427 0.0012233194146180962\n", + "Step # 428 0.0012204538228054985\n", + "Step # 429 0.0012176016767041437\n", + "Step # 430 0.0012147628818288965\n", + "Step # 431 0.001211937344577902\n", + "Step # 432 0.0012091249722223463\n", + "Step # 433 0.001206325672896266\n", + "Step # 434 0.0012035393555866172\n", + "Step # 435 0.0012007659301232624\n", + "Step # 436 0.0011980053071693644\n", + "Step # 437 0.0011952573982117374\n", + "Step # 438 0.0011925221155513496\n", + "Step # 439 0.0011897993722939958\n", + "Step # 440 0.001187089082341147\n", + "Step # 441 0.00118439116038077\n", + "Step # 442 0.0011817055218784353\n", + "Step # 443 0.0011790320830684352\n", + "Step # 444 0.0011763707609451254\n", + "Step # 445 0.0011737214732542571\n", + "Step # 446 0.0011710841384845044\n", + "Step # 447 0.0011684586758591653\n", + "Step # 448 0.001165845005327818\n", + "Step # 449 0.0011632430475582091\n", + "Step # 450 0.0011606527239282583\n", + "Step # 451 0.0011580739565180418\n", + "Step # 452 0.0011555066681020623\n", + "Step # 453 0.0011529507821414794\n", + "Step # 454 0.0011504062227765052\n", + "Step # 455 0.0011478729148188818\n", + "Step # 456 0.0011453507837445055\n", + "Step # 457 0.001142839755686052\n", + "Step # 458 0.001140339757425801\n", + "Step # 459 0.0011378507163885086\n", + "Step # 460 0.001135372560634334\n", + "Step # 461 0.0011329052188519735\n", + "Step # 462 0.00113044862035173\n", + "Step # 463 0.0011280026950588058\n", + "Step # 464 0.0011255673735066606\n", + "Step # 465 0.0011231425868303204\n", + "Step # 466 0.0011207282667599853\n", + "Step # 467 0.0011183243456145617\n", + "Step # 468 0.0011159307562953504\n", + "Step # 469 0.0011135474322797852\n", + "Step # 470 0.0011111743076152691\n", + "Step # 471 0.0011088113169130636\n", + "Step # 472 0.001106458395342307\n", + "Step # 473 0.0011041154786240565\n", + "Step # 474 0.001101782503025445\n", + "Step # 475 0.00109945940535386\n", + "Step # 476 0.0010971461229512618\n", + "Step # 477 0.0010948425936885542\n", + "Step # 478 0.0010925487559599748\n", + "Step # 479 0.0010902645486776347\n", + "Step # 480 0.001087989911266041\n", + "Step # 481 0.0010857247836567896\n", + "Step # 482 0.0010834691062832588\n", + "Step # 483 0.0010812228200753333\n", + "Step # 484 0.0010789858664543004\n", + "Step # 485 0.001076758187327722\n", + "Step # 486 0.0010745397250843912\n", + "Step # 487 0.0010723304225894125\n", + "Step # 488 0.0010701302231791916\n", + "Step # 489 0.0010679390706566816\n", + "Step # 490 0.0010657569092865494\n", + "Step # 491 0.0010635836837904397\n", + "Step # 492 0.0010614193393423154\n", + "Step # 493 0.0010592638215638258\n", + "Step # 494 0.0010571170765197595\n", + "Step # 495 0.0010549790507135449\n", + "Step # 496 0.0010528496910827633\n", + "Step # 497 0.0010507289449948108\n", + "Step # 498 0.0010486167602425077\n", + "Step # 499 0.0010465130850398144\n" + ] + } + ], + "source": [ + "xs = [\n", + " [2.0, 3.0, -1.0],\n", + " [3.0, -1.0, 0.5],\n", + " [0.5, 1.0, 1.0],\n", + " [1.0, 1.0, -1.0],\n", + "]\n", + "ys = [1.0, -1.0, -1.0, 1.0] # desired targets\n", + "ls = []\n", + "\n", + "for k in range(500):\n", + " # forward pass\n", + " ypred = [n(x) for x in xs]\n", + " loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n", + "\n", + " # backward pass\n", + " for p in n.parameters():\n", + " p.grad = 0.0 # YOU NEED TO clear out your gradient before every backward pass or it will just accumulate\n", + " loss.backward()\n", + "\n", + " # update (the gradient descent)\n", + " for p in n.parameters():\n", + " p.data += -0.05 * p.grad\n", + "\n", + " print('Step #', k, ' ', loss.data)\n", + " ls += [loss.data]" + ] + }, + { + "cell_type": "code", + "execution_count": 245, + "id": "8615e709-2365-4988-bad8-473590ff22b9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Value(data=0.9868926891382584),\n", + " Value(data=-0.9822284602914287),\n", + " Value(data=-0.9834981983414189),\n", + " Value(data=0.9831334891456814)]" + ] + }, + "execution_count": 245, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ypred = [n(x) for x in xs]\n", + "ypred" + ] + }, + { + "cell_type": "code", + "execution_count": 246, + "id": "046c7e20-5384-476f-abb7-d6d313561ebb", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 246, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAixElEQVR4nO3de3DU1f3/8ddns9klkVy4hZCyUbxSxKACppHaakm1VK366/hzGDrys5121Hih2k5NO/UynTb02/l1sNUftbaV/lGNlyniWMFSFKiVayQVsEawKFFu3siG2+ay5/dHsh+y3DfZ3ZOcPB8zO5vdz+ezn7MnzOTF+bw/53jGGCMAAIA0CNhuAAAAcAfBAgAApA3BAgAApA3BAgAApA3BAgAApA3BAgAApA3BAgAApA3BAgAApE0w2yeMx+PasWOHCgoK5Hletk8PAAB6wRij1tZWlZWVKRA4/rhE1oPFjh07FIlEsn1aAACQBs3NzRo7duxxt2c9WBQUFEjqalhhYWG2Tw8AAHohGo0qEon4f8ePJ+vBInH5o7CwkGABAMAAc7IyBoo3AQBA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2hAsAABA2mR9EbJM+b9/b1LroQ7ddvlZGl04xHZzAAAYlFIesfjwww/1rW99SyNGjFBeXp4uuOACrV+/PhNtS0n9umYteP09fbq/zXZTAAAYtFIasfjss880bdo0XXHFFVq8eLFGjRqlLVu2aNiwYZlq3ykLdK/iGjfGbkMAABjEUgoWv/zlLxWJRPTEE0/4740bNy7tjeqNQPf68OQKAADsSelSyAsvvKApU6boxhtvVElJiS666CI9/vjjJzwmFospGo0mPTKhe8CCEQsAACxKKVj897//1fz583XOOefo5Zdf1m233aa77rpLf/7zn497TF1dnYqKivxHJBLpc6OPxWPEAgAA6zxjTv1PcSgU0pQpU/T666/77911111at26dVq1adcxjYrGYYrGY/zoajSoSiailpUWFhYV9aHqyy/7nFTV/elALb79UF5Xbr/kAAMAl0WhURUVFJ/37ndKIxZgxYzRhwoSk9z7/+c9r+/btxz0mHA6rsLAw6ZEJiRqLOCMWAABYk1KwmDZtmpqampLee+edd3T66aentVG9kaixSGEABgAApFlKweL73/++Vq9erV/84hfaunWrnnzySf3+979XTU1Nptp3yvy7Qiy3AwCAwSylYDF16lQtXLhQTz31lCZOnKif/exnmjdvnmbNmpWp9p0yLzGPBddCAACwJuUpva+55hpdc801mWhLn3jUWAAAYJ0zi5AlZt40XAwBAMAah4IF81gAAGCbM8Hi8KUQkgUAALa4Eyy6n6mxAADAHmeCRaD7mzCPBQAA9rgTLKixAADAOmeCBaubAgBgnzvBghELAACscyZYJOaxYMQCAAB7HAoWzLwJAIBtzgSLxFoh3BUCAIA9DgULVjcFAMA2Z4IFNRYAANjnTLDwRI0FAAC2ORMsmHkTAAD73AkWzGMBAIB1zgQLVjcFAMA+d4JF9zM1FgAA2ONMsAgwjwUAANY5FCyosQAAwDZnggU1FgAA2OdQsOh6psYCAAB7nAkWfo0Fk3oDAGCNQ8GCmTcBALDNmWDB6qYAANjnULDgrhAAAGxzJlgEuCsEAADrHAoWXc/UWAAAYI8zwSIxpTc1FgAA2ONMsGDmTQAA7HMmWDDzJgAA9jkULLqeqbEAAMAeZ4LF4eJNkgUAALY4FCy8k+8EAAAyyplg4ddYcC0EAABrHAoWXc/kCgAA7HEmWLC6KQAA9jkULFjdFAAA25wLFsy8CQCAPc4EiwRuNwUAwB5nggVTegMAYJ9DwaLrmRoLAADscSZYJG43pcYCAAB7nAkW/qUQy+0AAGAwSylYPPjgg/I8L+kxfvz4TLUtJcy8CQCAfcFUDzj//PP1j3/84/AHBFP+iIygxgIAAPtSTgXBYFClpaWZaEufeKxuCgCAdSnXWGzZskVlZWU688wzNWvWLG3fvv2E+8diMUWj0aRHJrC6KQAA9qUULCorK7VgwQItWbJE8+fP17Zt23TZZZeptbX1uMfU1dWpqKjIf0QikT43+lj8GgtGLAAAsCalYDFjxgzdeOONqqio0FVXXaWXXnpJe/fu1TPPPHPcY2pra9XS0uI/mpub+9zoY0mMVxAsAACwp0+Vl8XFxTr33HO1devW4+4TDocVDof7cppTwsybAADY16d5LPbt26d3331XY8aMSVd7eo27QgAAsC+lYPGDH/xAK1as0HvvvafXX39dN9xwg3JycjRz5sxMte+UBQKsbgoAgG0pXQr54IMPNHPmTH3yyScaNWqUvvjFL2r16tUaNWpUptqXMmosAACwJ6VgUV9fn6l29Bk1FgAA2OfQWiFdz9RYAABgj0PBghoLAABscyZYMKU3AAD2ORQsWDYdAADbnAkW1FgAAGCfM8GCKb0BALDPmWCRmCCLayEAANjjTLBgdVMAAOxzJlgEuCsEAADrnAkWnhIjFpYbAgDAIOZMsPBLLAgWAABY41CwYOZNAABscyZYiBoLAACscyZYBJh5EwAA6xwKFl3PFG8CAGCPQ8GCGgsAAGxzJliwuikAAPY5FCwSIxaWGwIAwCDmTLBg5k0AAOxzKFgw8yYAALY5EywSy6ZTvAkAgD3uBAtqLAAAsM6ZYEGNBQAA9jkTLDxqLAAAsM6ZYOGvbmq3GQAADGoOBQtm3gQAwDZnggUzbwIAYJ9DwaK7xiJuuSEAAAxizgQLaiwAALDPoWBBjQUAALY5EywSM29SYwEAgD3uBAtm3gQAwDpnggUzbwIAYJ87wSLAiAUAALY5EyyosQAAwD53gkWixsJyOwAAGMycCRbUWAAAYJ8zwYKZNwEAsM+ZYOHPvMmIBQAA1jgULKixAADANmeCBaubAgBgnzvBovuG0zi5AgAAa5wJFoHub8KABQAA9rgTLFjdFAAA6xwKFl3P1FgAAGBPn4LF3Llz5Xme5syZk6bm9AU1FgAA2NbrYLFu3To99thjqqioSGd7eo15LAAAsK9XwWLfvn2aNWuWHn/8cQ0bNizdbeqVwzUWlhsCAMAg1qtgUVNTo6uvvlrV1dUn3TcWiykajSY9MoF5LAAAsC+Y6gH19fV64403tG7dulPav66uTg899FDKDUsVM28CAGBfSiMWzc3Nuvvuu/WXv/xFQ4YMOaVjamtr1dLS4j+am5t71dCTYcQCAAD7UhqxaGho0J49e3TxxRf773V2dmrlypV65JFHFIvFlJOTk3RMOBxWOBxOT2tPIDFiwV0hAADYk1KwmD59ujZu3Jj03i233KLx48frRz/60VGhIps87goBAMC6lIJFQUGBJk6cmPTeaaedphEjRhz1frZxVwgAAPY5M/MmNRYAANiX8l0hR1q+fHkamtF3rG4KAIB9zoxYJGbelKizAADAFoeCxeFkQa4AAMAOJ4MFdRYAANjhTLBQj0sh1FkAAGCHM8EiqcaCib0BALDCoWBBjQUAALY5GSyosQAAwA5ngoVHjQUAANY5GSyYxwIAADucCRbJl0IsNgQAgEHMmWDRY8CCEQsAACxxJlhwVwgAAPY5EyySizdJFgAA2OBQsPB6LJ1uty0AAAxWzgQL6XCdBTUWAADY4VSwSNRZECsAALDDyWBBjQUAAHY4FSxEjQUAAFY5FSwSK5xSYwEAgB2OBYvuGgtyBQAAVjgZLKixAADADqeCReJ2U2osAACww61gQY0FAABWORUsAoHEpRDLDQEAYJByK1j4xZskCwAAbHAqWFBjAQCAXW4FC39Kb5IFAAA2OBUsEhNkxeN22wEAwGDlVLA4vGw6IxYAANjgVLBIFG8CAAA7nAwWjFgAAGCHU8HCY3VTAACscjRYkCwAALDBqWDB6qYAANjlaLAgWQAAYINTwYKZNwEAsMutYMHqpgAAWOVUsDh8u6nlhgAAMEg5GSwYsQAAwA6nggXzWAAAYJdjwYLVTQEAsMmpYBFgxAIAAKucChbMvAkAgF1OBQuKNwEAsCulYDF//nxVVFSosLBQhYWFqqqq0uLFizPVtpR5TOkNAIBVKQWLsWPHau7cuWpoaND69ev1la98Rdddd502b96cqfalhBoLAADsCqay87XXXpv0+uc//7nmz5+v1atX6/zzz09rw3rj8JTeJAsAAGxIKVj01NnZqWeffVb79+9XVVXVcfeLxWKKxWL+62g02ttTnhSrmwIAYFfKxZsbN27U0KFDFQ6Hdeutt2rhwoWaMGHCcfevq6tTUVGR/4hEIn1q8IlQvAkAgF0pB4vzzjtPjY2NWrNmjW677TbNnj1bb7311nH3r62tVUtLi/9obm7uU4NPhJk3AQCwK+VLIaFQSGeffbYkafLkyVq3bp0efvhhPfbYY8fcPxwOKxwO962Vp4h5LAAAsKvP81jE4/GkGgqb/EshltsBAMBgldKIRW1trWbMmKHy8nK1trbqySef1PLly/Xyyy9nqn0pocYCAAC7UgoWe/bs0c0336ydO3eqqKhIFRUVevnll/XVr341U+1LCZdCAACwK6Vg8cc//jFT7UgLZt4EAMAux9YK6XrmrhAAAOxwLFh0JQsuhQAAYIdTwSIxpTfFmwAA2OFWsKDGAgAAq5wKFtRYAABgl1PBgttNAQCwy6lgwQRZAADY5WawsNwOAAAGK6eChX8phCILAACscCxYJOaxsNwQAAAGKaeCReKuEHIFAAB2OBYsKN4EAMAmp4IFt5sCAGCXW8FC1FgAAGCTU8HCr7EgWAAAYIVjwYLVTQEAsMmpYOH5IxYECwAAbHAsWLC6KQAANjkVLFjdFAAAuxwLFtRYAABgk1PBghoLAADscipYsLopAAB2ORUsmHkTAAC73AoWzLwJAIBVTgWLACMWAABY5VawYN10AACscipYUGMBAIBdbgULaiwAALDKqWDB6qYAANjlWLBg5k0AAGxyLFh0PTPzJgAAdjgVLORRYwEAgE1OBYvDd5uSLAAAsMGxYMGIBQAANjkVLLoHLKixAADAEqeCRWLmTXIFAAB2OBUsmHkTAAC7nAoW1FgAAGCXU8EiUWPBiAUAAHY4FSwSIxbcbQoAgB1OBQtqLAAAsMuxYEGNBQAANjkVLAJcCQEAwCrHggWrmwIAYJNjwaLrmZk3AQCwI6VgUVdXp6lTp6qgoEAlJSW6/vrr1dTUlKm2pS4xYhG33A4AAAaplILFihUrVFNTo9WrV2vp0qVqb2/XlVdeqf3792eqfSlhdVMAAOwKprLzkiVLkl4vWLBAJSUlamho0Je+9KW0Nqw3mHkTAAC7UgoWR2ppaZEkDR8+/Lj7xGIxxWIx/3U0Gu3LKU+I1U0BALCr18Wb8Xhcc+bM0bRp0zRx4sTj7ldXV6eioiL/EYlEenvKk2LEAgAAu3odLGpqarRp0ybV19efcL/a2lq1tLT4j+bm5t6e8qQ87goBAMCqXl0KueOOO/Tiiy9q5cqVGjt27An3DYfDCofDvWpcqhixAADArpSChTFGd955pxYuXKjly5dr3LhxmWpXr7BWCAAAdqUULGpqavTkk09q0aJFKigo0K5duyRJRUVFysvLy0gDU+GvbgoAAKxIqcZi/vz5amlp0eWXX64xY8b4j6effjpT7UsJIxYAANiV8qWQ/izAzJsAAFjl1FohjFgAAGCXU8EiMWJBrAAAwA7HgkXXc3+/ZAMAgKucChaJSb2ZxwIAADucChaMWAAAYJdjwYIRCwAAbHIrWHR/G0YsAACww6lg4VFjAQCAVW4Fi0SNBTecAgBghVPBgpk3AQCwy6lgwcybAADY5VSw8GfeJFcAAGCFU8GCGgsAAOxyKlgwjwUAAHY5FSy6ByyosQAAwBKngkXAn9PbbjsAABis3AoW3BUCAIBVTgULjxoLAACscitYdD8zYgEAgB1OBQvmsQAAwC5HgwXJAgAAG5wKFoen9LbbDgAABisngwUzbwIAYIdTwYKZNwEAsMvJYEGNBQAAdjgVLKixAADALqeChT+jNyMWAABY4VSwYOZNAADscitYdD8z8yYAAHY4FSyYeRMAALscDRYkCwAAbHAqWHBXCAAAdjkaLEgWAADY4FSw8C+FWG4HAACDlZvBghELAACscCxYdD1TYwEAgB1OBYvEBFmdJAsAAKxwKljk5nj+z4QLAACyz6lgEcw5/HXaO+MWWwIAwODkVLDoOWJBsAAAIPvcChaBniMWXAoBACDbnAoWgYCnnO5bQxixAAAg+5wKFtLhyyEECwAAss+9YNF9OYRLIQAAZF/KwWLlypW69tprVVZWJs/z9Pzzz2egWb2XG+z6Sh2MWAAAkHUpB4v9+/dr0qRJevTRRzPRnj4LdtdYtBEsAADIumCqB8yYMUMzZszIRFvSIjeHSyEAANiScrBIVSwWUywW819Ho9GMni/EpRAAAKzJePFmXV2dioqK/EckEsno+bgUAgCAPRkPFrW1tWppafEfzc3NGT1f4lJIB5dCAADIuoxfCgmHwwqHw5k+jY95LAAAsMe9eSz84k2CBQAA2ZbyiMW+ffu0detW//W2bdvU2Nio4cOHq7y8PK2N6w3uCgEAwJ6Ug8X69et1xRVX+K/vueceSdLs2bO1YMGCtDWst4JcCgEAwJqUg8Xll18uY/rvaECISyEAAFjjXI3F4RGL/ht+AABwlXPBguJNAADscS5YhJjHAgAAa5wLFolLIcy8CQBA9jkXLLgUAgCAPc4GCy6FAACQfQ4GC+axAADAFgeDBTNvAgBgi3PBIkiNBQAA1jgXLEJcCgEAwBrngkWQSyEAAFjjXLDgdlMAAOxxLlhwKQQAAHucCxZcCgEAwB7nggWXQgAAsMfBYNF1KaQjTrAAACDbHAwW3SMWHVwKAQAg25wNFqxuCgBA9jkXLIJcCgEAwBrngkWISyEAAFjjXLDwaywYsQAAIOucCxZBJsgCAMAa54JF4lJIWwfBAgCAbHMuWBQMCUqSogc7LLcEAIDBx7lgUZwfkiQdbO/UofZOy60BAGBwcS5YFA4JKifQVWex90C75dYAADC4OBcsPM9TcV6uJOmzA22WWwMAwODiXLCQpGGndV0OIVgAAJBdbgaL/K4RCy6FAACQXU4Gi0QBJyMWAABkl5PBIjFi8dl+ggUAANnkaLBIjFhwKQQAgGxyM1hQvAkAgBVuBguKNwEAsMLJYJEo3vxkX8xySwAAGFycDBZnjTpNkvTO7n3qYJVTAACyxslgcebIoSoYEtTB9k417W613RwAAAYNJ4NFIOBp0thiSVJj816rbQEAYDBxMlhI0oWRYklSw/uf2W0IAACDiLPB4tKzR0iSlm7erQNtHZZbAwDA4OBssPjCuBE6fUS+WmMdWtS4w3ZzAAAYFJwNFoGAp1mV5ZKkXy99Ry3MaQEAQMY5Gywk6eaqM3TmqNP0UWtMNU++oUPtnbabBACA05wOFkNyczTvpguVH8rRa1s/1v/6f69r7bZPbTcLAABn9SpYPProozrjjDM0ZMgQVVZWau3ateluV9pUjC3WE/9nqoafFtJbO6P634+t0szfr1b92u3aEz1ku3kAADjFM8aYVA54+umndfPNN+t3v/udKisrNW/ePD377LNqampSSUnJSY+PRqMqKipSS0uLCgsLe93wVO2JHtK8ZVv0zLpmdcQPf+WyoiG6sLxYZ40aqsiwfEWG5+tzxXkadlquhoaD8jwva20EAKC/OtW/3ykHi8rKSk2dOlWPPPKIJCkejysSiejOO+/Ufffdl7aGZcoHnx3QosYdWrxppzbviOpE3z43x1NxfkjD80Mqzu8KGkNCOcrPzVF+KKf756DyQgHlhYIK5wSUG/QUDASUm+MpNyegYM7hn3NzAgoGPIWCXc+5OQEFAp4CnpTjefI8Tzndr3v+HPA8BY7YBgBANmUkWLS1tSk/P1/PPfecrr/+ev/92bNna+/evVq0aFHaGpYN+2Id2vhBizZ+uFfvf3JAzZ8d1AefHtCOloM61N5/1xjx/LCRHDo8T93hw5PXvZ/U9X7itZf02vM/71jbug+X55/XO+pzjnr/iM9R0jE9ztnjc9Sjff53VI8XR247Ilf13PdEmevIQOYlbTv2+0ced/S2433iyT6z57befdfj/Nh93Km1OZXjjn7j1Bz5/U75uF6fL5vnyvJ36/X/Kfr/76Bv5xso3y97/ym898pzVTAkN62feap/v4OpfOjHH3+szs5OjR49Oun90aNH6+233z7mMbFYTLHY4VVGo9FoKqfMqKHhoKrOGqGqs0Ycte1gW6c+O9DW9djfrs8OtOlAW4cOtnXqQHunDrZ1Jv18oK1D7Z1G7Z1xtXfG1eH/3PXcETdq64irI961ra17n7hJPKTO+KllPGOkTmPUdY9LSgNOAIBB4PYrzkp7sDhVKQWL3qirq9NDDz2U6dOkXV4oR3mhPJUV52X1vMYYdca7gkYidPiv492vjekKF92v43H1eP/wscZ0hRAj41/y6fnadJ/PdL+vpPcPb+t5nHpuP8bnqOf7xzzHMc7To33dH3FUnxy/v3oeZ46/7RQ//6gznern97ZdR53v+G1J/j6nuN9R2/rel6nq7aF9O2fvD+7LefsixavSPY7rwzl7f+iA+/306dfahy9rq4/zQxn/835cKZ155MiRysnJ0e7du5Pe3717t0pLS495TG1tre655x7/dTQaVSQS6UVTBwfP8xTMoYYCADAwpXS7aSgU0uTJk7Vs2TL/vXg8rmXLlqmqquqYx4TDYRUWFiY9AACAm1IeK7nnnns0e/ZsTZkyRZdcconmzZun/fv365ZbbslE+wAAwACScrC46aab9NFHH+n+++/Xrl27dOGFF2rJkiVHFXQCAIDBJ+V5LPqqP91uCgAATs2p/v12eq0QAACQXQQLAACQNgQLAACQNgQLAACQNgQLAACQNgQLAACQNgQLAACQNgQLAACQNgQLAACQNllfVzUx0Wc0Gs32qQEAQC8l/m6fbMLurAeL1tZWSWLpdAAABqDW1lYVFRUdd3vW1wqJx+PasWOHCgoK5Hle2j43Go0qEomoubmZNUgyjL7ODvo5O+jn7KGvsyNT/WyMUWtrq8rKyhQIHL+SIusjFoFAQGPHjs3Y5xcWFvIPNkvo6+ygn7ODfs4e+jo7MtHPJxqpSKB4EwAApA3BAgAApI0zwSIcDuuBBx5QOBy23RTn0dfZQT9nB/2cPfR1dtju56wXbwIAAHc5M2IBAADsI1gAAIC0IVgAAIC0IVgAAIC0cSZYPProozrjjDM0ZMgQVVZWau3atbabNKCsXLlS1157rcrKyuR5np5//vmk7cYY3X///RozZozy8vJUXV2tLVu2JO3z6aefatasWSosLFRxcbG+853vaN++fVn8Fv1fXV2dpk6dqoKCApWUlOj6669XU1NT0j6HDh1STU2NRowYoaFDh+qb3/ymdu/enbTP9u3bdfXVVys/P18lJSX64Q9/qI6Ojmx+lX5t/vz5qqio8CcIqqqq0uLFi/3t9HFmzJ07V57nac6cOf579HV6PPjgg/I8L+kxfvx4f3u/6mfjgPr6ehMKhcyf/vQns3nzZvPd737XFBcXm927d9tu2oDx0ksvmZ/85Cfmr3/9q5FkFi5cmLR97ty5pqioyDz//PPm3//+t/nGN75hxo0bZw4ePOjv87Wvfc1MmjTJrF692vzzn/80Z599tpk5c2aWv0n/dtVVV5knnnjCbNq0yTQ2Npqvf/3rpry83Ozbt8/f59ZbbzWRSMQsW7bMrF+/3nzhC18wl156qb+9o6PDTJw40VRXV5sNGzaYl156yYwcOdLU1tba+Er90gsvvGD+9re/mXfeecc0NTWZH//4xyY3N9ds2rTJGEMfZ8LatWvNGWecYSoqKszdd9/tv09fp8cDDzxgzj//fLNz507/8dFHH/nb+1M/OxEsLrnkElNTU+O/7uzsNGVlZaaurs5iqwauI4NFPB43paWl5le/+pX/3t69e004HDZPPfWUMcaYt956y0gy69at8/dZvHix8TzPfPjhh1lr+0CzZ88eI8msWLHCGNPVr7m5uebZZ5/19/nPf/5jJJlVq1YZY7pCYCAQMLt27fL3mT9/viksLDSxWCy7X2AAGTZsmPnDH/5AH2dAa2urOeecc8zSpUvNl7/8ZT9Y0Nfp88ADD5hJkyYdc1t/6+cBfymkra1NDQ0Nqq6u9t8LBAKqrq7WqlWrLLbMHdu2bdOuXbuS+rioqEiVlZV+H69atUrFxcWaMmWKv091dbUCgYDWrFmT9TYPFC0tLZKk4cOHS5IaGhrU3t6e1Nfjx49XeXl5Ul9fcMEFGj16tL/PVVddpWg0qs2bN2ex9QNDZ2en6uvrtX//flVVVdHHGVBTU6Orr746qU8l/j2n25YtW1RWVqYzzzxTs2bN0vbt2yX1v37O+iJk6fbxxx+rs7MzqbMkafTo0Xr77bcttcotu3btkqRj9nFi265du1RSUpK0PRgMavjw4f4+SBaPxzVnzhxNmzZNEydOlNTVj6FQSMXFxUn7HtnXx/pdJLahy8aNG1VVVaVDhw5p6NChWrhwoSZMmKDGxkb6OI3q6+v1xhtvaN26dUdt499z+lRWVmrBggU677zztHPnTj300EO67LLLtGnTpn7XzwM+WAADVU1NjTZt2qTXXnvNdlOcdN5556mxsVEtLS167rnnNHv2bK1YscJ2s5zS3Nysu+++W0uXLtWQIUNsN8dpM2bM8H+uqKhQZWWlTj/9dD3zzDPKy8uz2LKjDfhLISNHjlROTs5R1a+7d+9WaWmppVa5JdGPJ+rj0tJS7dmzJ2l7R0eHPv30U34Px3DHHXfoxRdf1KuvvqqxY8f675eWlqqtrU179+5N2v/Ivj7W7yKxDV1CoZDOPvtsTZ48WXV1dZo0aZIefvhh+jiNGhoatGfPHl188cUKBoMKBoNasWKFfvOb3ygYDGr06NH0dYYUFxfr3HPP1datW/vdv+kBHyxCoZAmT56sZcuW+e/F43EtW7ZMVVVVFlvmjnHjxqm0tDSpj6PRqNasWeP3cVVVlfbu3auGhgZ/n1deeUXxeFyVlZVZb3N/ZYzRHXfcoYULF+qVV17RuHHjkrZPnjxZubm5SX3d1NSk7du3J/X1xo0bk4Lc0qVLVVhYqAkTJmTniwxA8XhcsViMPk6j6dOna+PGjWpsbPQfU6ZM0axZs/yf6evM2Ldvn959912NGTOm//2bTmspqCX19fUmHA6bBQsWmLfeest873vfM8XFxUnVrzix1tZWs2HDBrNhwwYjyfz61782GzZsMO+//74xput20+LiYrNo0SLz5ptvmuuuu+6Yt5tedNFFZs2aNea1114z55xzDrebHuG2224zRUVFZvny5Um3jR04cMDf59ZbbzXl5eXmlVdeMevXrzdVVVWmqqrK3564bezKK680jY2NZsmSJWbUqFHcntfDfffdZ1asWGG2bdtm3nzzTXPfffcZz/PM3//+d2MMfZxJPe8KMYa+Tpd7773XLF++3Gzbts3861//MtXV1WbkyJFmz549xpj+1c9OBAtjjPntb39rysvLTSgUMpdccolZvXq17SYNKK+++qqRdNRj9uzZxpiuW05/+tOfmtGjR5twOGymT59umpqakj7jk08+MTNnzjRDhw41hYWF5pZbbjGtra0Wvk3/daw+lmSeeOIJf5+DBw+a22+/3QwbNszk5+ebG264wezcuTPpc9577z0zY8YMk5eXZ0aOHGnuvfde097enuVv0399+9vfNqeffroJhUJm1KhRZvr06X6oMIY+zqQjgwV9nR433XSTGTNmjAmFQuZzn/ucuemmm8zWrVv97f2pn1k2HQAApM2Ar7EAAAD9B8ECAACkDcECAACkDcECAACkDcECAACkDcECAACkDcECAACkDcECAACkDcECAACkDcECAACkDcECAACkDcECAACkzf8HeQBqSQlwzRAAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(np.arange(0, 500, 1), ls)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}