![\"weather\"](\"images/MnistExamples.png\")
\n",
"\n",
"**Figure:** 28 x 28 pixels images of handwritten numbers from 0 to 9. Author: Suvanjanprasai. License: [CC BY-SA](https://creativecommons.org/licenses/by-sa/4.0/deed.en).\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "7e91f51b",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"In this figure:\n",
"- One has no issue recognizing each digit: for each image, we immediately attach a label to our visual impression.\n",
"- Yet, the task of writing a computer program that is able to identify each individual image seems complicated.\n",
"- Each digit can indeed be written in many different ways and there are no clear logical instructions on how to proceed to identify the digit pattern.\n",
"\n",
"The following code cells read and plot the MNIST data."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "575f3c1a",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"outputs": [],
"source": [
"import gzip\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from pathlib import Path\n",
"\n",
"# MNIST image size\n",
"MNIST_IM_SIZE = 28\n",
"\n",
"# MNIST directory\n",
"DIR_MNIST = Path('data', 'mnist')\n",
"\n",
"# MNIST training set configuration\n",
"N_TRAIN = 60000\n",
"FILE_TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'\n",
"FILE_TRAIN_LABELS = 'train-labels-idx1-ubyte.gz'\n",
"\n",
"# MNIST test set configuration\n",
"N_TEST = 10000\n",
"FILE_TEST_IMAGES = 't10k-images-idx3-ubyte.gz'\n",
"FILE_TEST_LABELS = 't10k-labels-idx1-ubyte.gz'"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "7235cb04",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"outputs": [],
"source": [
"def load_mnist_images(filename, num_images, mnist_im_size=MNIST_IM_SIZE):\n",
" '''\n",
" return flatten images\n",
" '''\n",
" f = gzip.open(filename,'r')\n",
" f.read(16)\n",
" buf = f.read(num_images * mnist_im_size**2)\n",
" images = np.frombuffer(buf, dtype=np.uint8) / 255.\n",
" images = images.reshape(num_images, mnist_im_size**2)\n",
" f.close()\n",
"\n",
" return images\n",
"\n",
"def load_mnist_labels(filename, num_labels):\n",
"\n",
" f = gzip.open(filename, 'r')\n",
" f.read(8)\n",
" buf = f.read(num_labels)\n",
" labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)\n",
" f.close()\n",
" \n",
" return labels\n",
"\n",
"def plot_mnist_image(image, cmap=plt.cm.gray_r):\n",
" n_side = int(np.sqrt(len(image)))\n",
" plt.figure()\n",
" plt.imshow(image.reshape((n_side, n_side)), cmap=cmap)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e8d041da",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"outputs": [],
"source": [
"# Read training set\n",
"X_train = load_mnist_images(Path(DIR_MNIST, FILE_TRAIN_IMAGES), N_TRAIN)\n",
"y_train = load_mnist_labels(Path(DIR_MNIST, FILE_TRAIN_LABELS), N_TRAIN)\n",
"\n",
"# Read test set\n",
"X_test = load_mnist_images(Path(DIR_MNIST, FILE_TEST_IMAGES), N_TEST)\n",
"y_test = load_mnist_labels(Path(DIR_MNIST, FILE_TEST_LABELS), N_TEST)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "7ef1c805",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGdCAYAAAC7EMwUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAatklEQVR4nO3dfWiV9/3/8dfR6qm1yWGZJudkpiGUyIaKmzf1ZlWjw2BGrdYWYktHZExsjW4SRWbdMKvFiKNiIav7rgynW1NlYK1Uqc3QxBbratO4BlesxTgzTBZ0NidGd4L18/tDPL8eE2+u4zm+c06eD7jAc53rneudyw/n5cfr5vicc04AABgYYN0AAKD/IoQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABg5gHrBm527do1nTt3ThkZGfL5fNbtAAA8cs6ps7NTubm5GjDg9nOdPhdC586dU15ennUbAIB71NLSohEjRtx2mz4XQhkZGZKuN5+ZmWncDQDAq3A4rLy8vOjn+e0kLYRef/11/fa3v1Vra6tGjRqlLVu2aNq0aXesu/FfcJmZmYQQAKSwuzmlkpQLE3bt2qUVK1Zo7dq1amxs1LRp01RSUqKzZ88mY3cAgBTlS8ZTtCdNmqRx48Zp69at0XXf+973NH/+fFVVVd22NhwOKxAIqKOjg5kQAKQgL5/jCZ8JdXd3q6GhQcXFxTHri4uLdeTIkR7bRyIRhcPhmAUA0D8kPITOnz+vr7/+Wjk5OTHrc3Jy1NbW1mP7qqoqBQKB6MKVcQDQfyTtZtWbT0g553o9SbVmzRp1dHREl5aWlmS1BADoYxJ+ddywYcM0cODAHrOe9vb2HrMjSfL7/fL7/YluAwCQAhI+Exo8eLDGjx+v2tramPW1tbWaOnVqoncHAEhhSblPqKKiQj/5yU80YcIETZkyRX/4wx909uxZvfDCC8nYHQAgRSUlhEpLS3XhwgW9/PLLam1t1ejRo7V//37l5+cnY3cAgBSVlPuE7gX3CQFAajO9TwgAgLtFCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzD1g3AODuNDQ0eK6prq6Oa1/bt2/3XFNWVua5Zvny5Z5rxo0b57kGfRczIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGZ8zjln3cQ3hcNhBQIBdXR0KDMz07odICmOHz/uuWbmzJmea8LhsOea+ykQCHiu+e9//5uETpBIXj7HmQkBAMwQQgAAMwkPocrKSvl8vpglGAwmejcAgDSQlC+1GzVqlP72t79FXw8cODAZuwEApLikhNADDzzA7AcAcEdJOSd06tQp5ebmqqCgQAsXLtTp06dvuW0kElE4HI5ZAAD9Q8JDaNKkSdqxY4cOHDigN954Q21tbZo6daouXLjQ6/ZVVVUKBALRJS8vL9EtAQD6qKTfJ9TV1aVHH31Uq1evVkVFRY/3I5GIIpFI9HU4HFZeXh73CSGtcZ/QddwnlJ683CeUlHNC3zR06FCNGTNGp06d6vV9v98vv9+f7DYAAH1Q0u8TikQi+vzzzxUKhZK9KwBAikl4CK1atUr19fVqbm7W3//+dz3zzDMKh8MqKytL9K4AACku4f8d9+9//1vPPvuszp8/r+HDh2vy5Mk6evSo8vPzE70rAECKS3gI7dy5M9E/EujTPv74Y881Tz/9tOeajo4OzzU+n89zjaS4LgoaPHiw55rz5897rvnoo48814wfP95zjRTf7wRveHYcAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM0n/UjvAwuXLl+Oq+/TTTz3XPP/8855rzp0757nmfiosLPRcs3r1as81paWlnmt++MMfeq555ZVXPNdI0ksvvRRXHe4eMyEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBmeoo20tGTJkrjqampqEtxJampoaPBcc+nSJc81M2bM8FxTV1fnuaapqclzDe4PZkIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDM8ABT9HnxPEzz3XffjWtfzrm46rwqKiryXPPEE094rlm1apXnGknKzc31XPODH/zAc823vvUtzzWHDh3yXHO//l7hHTMhAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZnyujz3ZLxwOKxAIqKOjQ5mZmdbtIMGOHz/uuWbmzJmea8LhsOeaeP34xz/2XPPWW295rqmrq/Nc09TU5LlGkn72s595rhk+fHhc+/JqwADv/3YeOnRoXPuqr6/3XDNu3Li49pVOvHyOMxMCAJghhAAAZjyH0OHDhzV37lzl5ubK5/Npz549Me8751RZWanc3FwNGTJERUVFOnHiRKL6BQCkEc8h1NXVpbFjx6q6urrX9zdt2qTNmzerurpax44dUzAY1OzZs9XZ2XnPzQIA0ovnb1YtKSlRSUlJr+8557RlyxatXbtWCxYskCRt375dOTk5qqmp0ZIlS+6tWwBAWknoOaHm5ma1tbWpuLg4us7v92vGjBk6cuRIrzWRSEThcDhmAQD0DwkNoba2NklSTk5OzPqcnJzoezerqqpSIBCILnl5eYlsCQDQhyXl6jifzxfz2jnXY90Na9asUUdHR3RpaWlJRksAgD7I8zmh2wkGg5Kuz4hCoVB0fXt7e4/Z0Q1+v19+vz+RbQAAUkRCZ0IFBQUKBoOqra2Nruvu7lZ9fb2mTp2ayF0BANKA55nQpUuX9OWXX0ZfNzc36/jx48rKytIjjzyiFStWaMOGDSosLFRhYaE2bNighx56SM8991xCGwcApD7PIfTJJ5/EPMuroqJCklRWVqY//elPWr16ta5cuaKlS5fq4sWLmjRpkt5//31lZGQkrmsAQFrgAaaI2xdffOG5prKy0nPNzp07PdfE+zDNb57LvFu/+tWvPNc888wznmtwXTwPML3VhVF3Ulpa6rmmpqYmrn2lEx5gCgBICYQQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAMwn9ZlWkpkgkElfdqlWrPNfs27fPc008T1PfsWOH5xpJmjBhgueaK1euxLUv9H0tLS3WLaQ9ZkIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDM8ABT6NNPP42rLp6HkcbjnXfe8VwzY8aMJHQCINGYCQEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADDDA0yhioqKuOqcc55rioqKPNfwMFJ8UzzjLhX21V8xEwIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGB5immXfffddzzfHjx+Pal8/n81zz5JNPxrUv4IZ4xl08NZL0/e9/P6463D1mQgAAM4QQAMCM5xA6fPiw5s6dq9zcXPl8Pu3Zsyfm/UWLFsnn88UskydPTlS/AIA04jmEurq6NHbsWFVXV99ymzlz5qi1tTW67N+//56aBACkJ88XJpSUlKikpOS22/j9fgWDwbibAgD0D0k5J1RXV6fs7GyNHDlSixcvVnt7+y23jUQiCofDMQsAoH9IeAiVlJTozTff1MGDB/Xqq6/q2LFjmjVrliKRSK/bV1VVKRAIRJe8vLxEtwQA6KMSfp9QaWlp9M+jR4/WhAkTlJ+fr3379mnBggU9tl+zZo0qKiqir8PhMEEEAP1E0m9WDYVCys/P16lTp3p93+/3y+/3J7sNAEAflPT7hC5cuKCWlhaFQqFk7woAkGI8z4QuXbqkL7/8Mvq6ublZx48fV1ZWlrKyslRZWamnn35aoVBIZ86c0UsvvaRhw4bpqaeeSmjjAIDU5zmEPvnkE82cOTP6+sb5nLKyMm3dulVNTU3asWOHvvrqK4VCIc2cOVO7du1SRkZG4roGAKQFzyFUVFQk59wt3z9w4MA9NYR7c+XKFc813d3dce0rOzvbc803L1xBernVFbC3U1lZmfhGevGjH/0orrqNGzcmuBPcjGfHAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMJP2bVZG+HnzwQc81fLlhaojnidivvPKK55pNmzZ5rsnLy/Ncs3LlSs81kvTwww/HVYe7x0wIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGR5girg9+eST1i3gDo4fPx5XXTwPFt21a5fnmnnz5nmu2b17t+ca9F3MhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJjhAaZpxjl3X2okac+ePZ5rXnvttbj2BWnz5s2ea9avXx/Xvjo6OjzXPP/8855rduzY4bkG6YWZEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADM8wDTN+Hy++1IjSW1tbZ5rfv7zn3uu+elPf+q55tvf/rbnGkk6evSo55o///nPnmv+8Y9/eK5paWnxXJOfn++5RpLmzJnjuWbp0qVx7Qv9GzMhAIAZQggAYMZTCFVVVWnixInKyMhQdna25s+fr5MnT8Zs45xTZWWlcnNzNWTIEBUVFenEiRMJbRoAkB48hVB9fb3Ky8t19OhR1dbW6urVqyouLlZXV1d0m02bNmnz5s2qrq7WsWPHFAwGNXv2bHV2dia8eQBAavN0YcJ7770X83rbtm3Kzs5WQ0ODpk+fLuectmzZorVr12rBggWSpO3btysnJ0c1NTVasmRJ4joHAKS8ezondOMrgLOysiRJzc3NamtrU3FxcXQbv9+vGTNm6MiRI73+jEgkonA4HLMAAPqHuEPIOaeKigo9/vjjGj16tKT/f8luTk5OzLY5OTm3vJy3qqpKgUAguuTl5cXbEgAgxcQdQsuWLdNnn32mt956q8d7N9934py75b0oa9asUUdHR3SJ514IAEBqiutm1eXLl2vv3r06fPiwRowYEV0fDAYlXZ8RhUKh6Pr29vYes6Mb/H6//H5/PG0AAFKcp5mQc07Lli3T7t27dfDgQRUUFMS8X1BQoGAwqNra2ui67u5u1dfXa+rUqYnpGACQNjzNhMrLy1VTU6N33nlHGRkZ0fM8gUBAQ4YMkc/n04oVK7RhwwYVFhaqsLBQGzZs0EMPPaTnnnsuKb8AACB1eQqhrVu3SpKKiopi1m/btk2LFi2SJK1evVpXrlzR0qVLdfHiRU2aNEnvv/++MjIyEtIwACB9+JxzzrqJbwqHwwoEAuro6FBmZqZ1Oynnr3/9q+eahQsXJqGTxLnV+cTbCQQCce3riy++iKvufpgyZYrnmlmzZsW1r5dffjmuOkDy9jnOs+MAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGbi+mZV9F3xPGn5sccei2tfH3/8cVx1Xt343iov/vOf/yShk94NGzbMc008Ty5/7bXXPNcAfR0zIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGZ4gGmaGTFihOea3bt3x7Wv//u///Ncs379+rj2db/84he/8Fzz4osveq4pLCz0XAOkI2ZCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzPicc866iW8Kh8MKBALq6OhQZmamdTsAAI+8fI4zEwIAmCGEAABmCCEAgBlCCABghhACAJghhAAAZgghAIAZQggAYIYQAgCYIYQAAGYIIQCAGUIIAGCGEAIAmCGEAABmCCEAgBlPIVRVVaWJEycqIyND2dnZmj9/vk6ePBmzzaJFi+Tz+WKWyZMnJ7RpAEB68BRC9fX1Ki8v19GjR1VbW6urV6+quLhYXV1dMdvNmTNHra2t0WX//v0JbRoAkB4e8LLxe++9F/N627Ztys7OVkNDg6ZPnx5d7/f7FQwGE9MhACBt3dM5oY6ODklSVlZWzPq6ujplZ2dr5MiRWrx4sdrb22/5MyKRiMLhcMwCAOgffM45F0+hc07z5s3TxYsX9cEHH0TX79q1Sw8//LDy8/PV3NysX//617p69aoaGhrk9/t7/JzKykr95je/6bH+br6bHADQ94TDYQUCgbv6HI87hMrLy7Vv3z59+OGHGjFixC23a21tVX5+vnbu3KkFCxb0eD8SiSgSicQ0n5eXRwgBQIryEkKezgndsHz5cu3du1eHDx++bQBJUigUUn5+vk6dOtXr+36/v9cZEgAg/XkKIeecli9frrffflt1dXUqKCi4Y82FCxfU0tKiUCgUd5MAgPTk6cKE8vJy/eUvf1FNTY0yMjLU1tamtrY2XblyRZJ06dIlrVq1Sh999JHOnDmjuro6zZ07V8OGDdNTTz2VlF8AAJC6PJ0T8vl8va7ftm2bFi1apCtXrmj+/PlqbGzUV199pVAopJkzZ2r9+vXKy8u7q314+b9EAEDfk7RzQnfKqyFDhujAgQNefiQAoB/j2XEAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAzhBAAwAwhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADMPWDdwM+ecJCkcDht3AgCIx43P7xuf57fT50Kos7NTkpSXl2fcCQDgXnR2dioQCNx2G5+7m6i6j65du6Zz584pIyNDPp8v5r1wOKy8vDy1tLQoMzPTqEN7HIfrOA7XcRyu4zhc1xeOg3NOnZ2dys3N1YABtz/r0+dmQgMGDNCIESNuu01mZma/HmQ3cByu4zhcx3G4juNwnfVxuNMM6AYuTAAAmCGEAABmUiqE/H6/1q1bJ7/fb92KKY7DdRyH6zgO13Ecrku149DnLkwAAPQfKTUTAgCkF0IIAGCGEAIAmCGEAABmUiqEXn/9dRUUFOjBBx/U+PHj9cEHH1i3dF9VVlbK5/PFLMFg0LqtpDt8+LDmzp2r3Nxc+Xw+7dmzJ+Z955wqKyuVm5urIUOGqKioSCdOnLBpNonudBwWLVrUY3xMnjzZptkkqaqq0sSJE5WRkaHs7GzNnz9fJ0+ejNmmP4yHuzkOqTIeUiaEdu3apRUrVmjt2rVqbGzUtGnTVFJSorNnz1q3dl+NGjVKra2t0aWpqcm6paTr6urS2LFjVV1d3ev7mzZt0ubNm1VdXa1jx44pGAxq9uzZ0ecQpos7HQdJmjNnTsz42L9//33sMPnq6+tVXl6uo0ePqra2VlevXlVxcbG6urqi2/SH8XA3x0FKkfHgUsRjjz3mXnjhhZh13/3ud90vf/lLo47uv3Xr1rmxY8dat2FKknv77bejr69du+aCwaDbuHFjdN3//vc/FwgE3O9//3uDDu+Pm4+Dc86VlZW5efPmmfRjpb293Uly9fX1zrn+Ox5uPg7Opc54SImZUHd3txoaGlRcXByzvri4WEeOHDHqysapU6eUm5urgoICLVy4UKdPn7ZuyVRzc7Pa2tpixobf79eMGTP63diQpLq6OmVnZ2vkyJFavHix2tvbrVtKqo6ODklSVlaWpP47Hm4+DjekwnhIiRA6f/68vv76a+Xk5MSsz8nJUVtbm1FX99+kSZO0Y8cOHThwQG+88Yba2to0depUXbhwwbo1Mzf+/vv72JCkkpISvfnmmzp48KBeffVVHTt2TLNmzVIkErFuLSmcc6qoqNDjjz+u0aNHS+qf46G34yClznjoc0/Rvp2bv9rBOddjXTorKSmJ/nnMmDGaMmWKHn30UW3fvl0VFRWGndnr72NDkkpLS6N/Hj16tCZMmKD8/Hzt27dPCxYsMOwsOZYtW6bPPvtMH374YY/3+tN4uNVxSJXxkBIzoWHDhmngwIE9/iXT3t7e4188/cnQoUM1ZswYnTp1yroVMzeuDmRs9BQKhZSfn5+W42P58uXau3evDh06FPPVL/1tPNzqOPSmr46HlAihwYMHa/z48aqtrY1ZX1tbq6lTpxp1ZS8Siejzzz9XKBSybsVMQUGBgsFgzNjo7u5WfX19vx4bknThwgW1tLSk1fhwzmnZsmXavXu3Dh48qIKCgpj3+8t4uNNx6E2fHQ+GF0V4snPnTjdo0CD3xz/+0f3zn/90K1ascEOHDnVnzpyxbu2+Wblypaurq3OnT592R48edU888YTLyMhI+2PQ2dnpGhsbXWNjo5PkNm/e7BobG92//vUv55xzGzdudIFAwO3evds1NTW5Z5991oVCIRcOh407T6zbHYfOzk63cuVKd+TIEdfc3OwOHTrkpkyZ4r7zne+k1XF48cUXXSAQcHV1da61tTW6XL58ObpNfxgPdzoOqTQeUiaEnHPud7/7ncvPz3eDBw9248aNi7kcsT8oLS11oVDIDRo0yOXm5roFCxa4EydOWLeVdIcOHXKSeixlZWXOueuX5a5bt84Fg0Hn9/vd9OnTXVNTk23TSXC743D58mVXXFzshg8f7gYNGuQeeeQRV1ZW5s6ePWvddkL19vtLctu2bYtu0x/Gw52OQyqNB77KAQBgJiXOCQEA0hMhBAAwQwgBAMwQQgAAM4QQAMAMIQQAMEMIAQDMEEIAADOEEADADCEEADBDCAEAzBBCAAAz/w/T1GfDQy2TBQAAAABJRU5ErkJggg==",
"text/plain": [
"\n",
" Warning\n",
" \n",
"This dataset is interesting for teaching purposes but it may also convey the idea that neural networks are simple and always work.\n",
" \n",
"Dealing with more serious machine learning problems, one soon realizes that datasets tend to be more complex.\n",
"
"
]
},
{
"cell_type": "markdown",
"id": "143c0ec3",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"## Building blocks of Neural Networks"
]
},
{
"cell_type": "markdown",
"id": "9a02f79c",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"### Neurons\n",
"\n",
"In neural network terminology:\n",
"- $\\boldsymbol x$ is the the input layer;\n",
"- each component of $\\boldsymbol x$ is a *neuron*;\n",
"- the numerical value held by the neuron is a real number called the *activation*;\n",
"- a neuron is *active* (resp. *inactive*) when it holds a positive (resp. non-positive) value.\n",
"\n",
"Here, the activation of the input layer corresponds to the grayscale value of the corresponding pixel on the image: white pixels have activation equal to zero and black pixels have activation equal to 1."
]
},
{
"cell_type": "markdown",
"id": "950c0468",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"#### The MNIST input layer\n",
"\n",
"![\"weather\"](\"images/mnist_52.png\")
\n",
"\n",
"\\begin{align}\n",
" \\boldsymbol{x} &=\n",
" \\begin{pmatrix}\n",
" x_0 \\\\\n",
" x_1 \\\\\n",
" x_2 \\\\\n",
" \\vdots \\\\\n",
" x_{p - 1} \\\\\n",
" \\end{pmatrix}\n",
"\\end{align}\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "5f439702",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"#### The MNIST output layer\n",
"\n",
"The values of the neurons of the output layer $\\boldsymbol y$ can be interpreted as the probabilities of each class given the input.\n",
"\n",
"In the MNIST example, the $k$th class corresponds to the digit $k - 1$). For instance, if\n",
"$$\n",
"\\boldsymbol y^\\top = \\left(0.01, 0.9, 0.2, 0.1, 0.05, 0.09, 0.05, 0.03, 0.25, 0.1\\right)\n",
"$$\n",
"the most probable digit is $k = 1$ (the second class).\n",
"\n",
"\n",
" Remark\n",
" \n",
"The output does not have to be between 0 and 1. In that case, one simply selects the neuron that is the most active.\n",
"
\n",
"\n"
]
},
{
"cell_type": "markdown",
"id": "83b2d2f0",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"#### The simplest neural network for the MNIST example\n",
"\n",
"![\"weather\"](\"images/nnet_mnist.png\")
\n",
"\n",
"The simplest neural network that one can build for this problem is represented in the following figure (mind the differences in notations with this course).\n",
"\n",
"Each link represents a contribution of an input to an output.\n",
"\n",
"Here, each contribution is linear because no activation function has been applied yet.\n",
"\n",
"Thus, in this network, all outputs are linear combinations of all features."
]
},
{
"cell_type": "markdown",
"id": "bc6b98f3",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"### A two-layer linear network\n",
"\n",
"In the simplest construction, the outputs are linear functions of the inputs, so that\n",
"\n",
"\\begin{equation}\n",
"\\hat y_k = \\sum_{j = 1}^{p} w_{kj} x_j + b_k,\n",
"\\end{equation}\n",
"\n",
"for some matrix $\\boldsymbol W$ with elements $w_{kj}$, or *weights*, and some vector $\\boldsymbol b$ with elements $b_k$, or *biases*, for $0 \\le k \\le K -1 $ and $1 \\le j \\le p$.\n",
"\n",
"\n",
"In matrix form,\n",
"\n",
"\\begin{equation}\n",
"\\hat {\\boldsymbol y} = \\boldsymbol W \\boldsymbol x + \\boldsymbol b.\n",
"\\end{equation}"
]
},
{
"cell_type": "markdown",
"id": "3fbd5418",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"This is a linear regression model with multiple outputs applied to classification.\n",
"\n",
"If we use the Mean-Squared Error (MSE) as loss function, this is just an Ordinary Least Squares (OLS) problem.\n",
"\n",
"\n",
" Remark\n",
" \n",
"More layers could be added to this two-layer linear model.\n",
"\n",
"However, since a linear function of a linear function is itself linear, the resulting model would linearly link all outputs to all inputs and thus be equivalent to the two-layer linear model.\n",
"
"
]
},
{
"cell_type": "markdown",
"id": "76db83b5",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"### From a linear model to a non-linear model\n",
"\n",
"- Non-linear functions are now applied to the linear combination of the features.\n",
"- These functions are referred to as *activation function*, since their role is to turn on or off a neuron, and the value they return (the state of the neuron) as the *activation*.\n",
"- Their role is crucial since their non-linearity is what distinguishes a linear model that is able to represent linear relationships only from a neural network that can capture nonlinear interactions between features and combinations of features.\n",
"\n",
"Let us introduce some possible activation functions (see [Table of activation functions on Wikipedia](https://en.wikipedia.org/wiki/Activation_function#Table_of_activation_functions) for other examples)."
]
},
{
"cell_type": "markdown",
"id": "e22d2e30",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"#### Heaviside distribution\n",
"\n",
"The Heaviside distribution (or step function, or binary step) is defined as\n",
"\n",
"\\begin{equation}\n",
"f(x)={\\begin{cases}1,&x>0\\\\0,&x\\leq 0\\end{cases}}\n",
"\\end{equation}\n",
"\n",
"If we apply this function to the weighted sum, we then get a binary neuron: its activation state is either 0 or 1.\n",
"\n",
"In neural network terminology, the association of the weighted sum and Heaviside activation function is called a *perceptron*."
]
},
{
"cell_type": "markdown",
"id": "e6bba5f8",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"---\n",
"**Perceptron**\n",
"\n",
"A perceptron is a function that maps an input vector $\\boldsymbol x$ to a single binary value depending on the value of the affine transformation of $\\boldsymbol x$:\n",
"\n",
"\\begin{equation}\n",
"f(\\boldsymbol x) =\n",
"\\begin{cases}\n",
" 1 & {\\text{if }}\\ \\boldsymbol w_k^\\top \\boldsymbol x + b_k > 0\\\\\n",
" 0 & {\\text{otherwise}}.\n",
"\\end{cases}\n",
"\\end{equation}\n",
"\n",
"---\n",
"\n",
"In this case, the activation state of a neuron can only be 0 or 1."
]
},
{
"cell_type": "markdown",
"id": "32b2a61c",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"For multiple classes, we can set the output of the perceptron to the class label for the largest $\\boldsymbol w_k \\boldsymbol x + b_k$.\n",
"\n",
"We then have a single output $\\hat{y}$ taking as many values as class labels.\n",
"\n",
"We are left with an optimization problem where we need to adjust these weights and biases to minimize the error in the image recognition."
]
},
{
"cell_type": "markdown",
"id": "d50be2cf",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"> ***Question***\n",
"> - For this simple network, do you have an intuition for how the weights should combine to recognize a \"0\"? a \"9\"? (again think of the perceptron as a weighted sum)\n",
"\n",
"Let's check it with scikit-learn. For now we will let scikit-learn handle all the heavy lifting (i.e. finding the best set of weights and biases)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7b93f547",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"outputs": [],
"source": [
"from sklearn.linear_model import Perceptron\n",
"\n",
"clf = Perceptron(tol=1e-3, random_state=0)\n",
"clf.fit(X_train, y_train)\n",
"train_score = clf.score(X_train, y_train)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "c9342dde",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train score (% of correct classification): 89 %\n"
]
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAGdCAYAAAC7EMwUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjjklEQVR4nO3de2zV9f3H8dcB2mPBcqBCb1K6Oi4qZSgXQS4KXho7ZUOcAU0WSDaickkYGifjD8mSUeMmYRmTbcYx2GASIyALTKxCiw7rEEEZA4JQoEhroUIvXFrafn9/EPpb5fr+2NNPL89HchJ6zvfF99MvX/riyznnfUJBEAQCAMCDDr4XAABovyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN508r2Ab6qvr9exY8cUHx+vUCjkezkAAKMgCFRZWanU1FR16HD1a50WV0LHjh1TWlqa72UAAL6loqIi9erV66rbtLgSio+PlyRNnDhRMTExnlcDALA6f/68Vq9e3fDz/GqiVkKvvvqqfv3rX6u4uFgDBgzQokWLNGbMmGvmLv4XXExMjGJjY6O1PABAlF3PUypReWHCqlWrNHv2bM2bN087duzQmDFjlJ2drSNHjkRjdwCAVioqJbRw4UL95Cc/0U9/+lPddtttWrRokdLS0rRkyZJo7A4A0Eo1eQnV1NRo+/btysrKanR/VlaWtm7desn21dXVqqioaHQDALQPTV5CJ06cUF1dnZKSkhrdn5SUpJKSkku2z8nJUSQSabjxyjgAaD+i9mbVbz4hFQTBZZ+kmjt3rsrLyxtuRUVF0VoSAKCFafJXx/Xo0UMdO3a85KqntLT0kqsjSQqHwwqHw029DABAK9DkV0KxsbEaMmSIcnNzG92fm5urkSNHNvXuAACtWFTeJzRnzhz9+Mc/1tChQ3X33XfrT3/6k44cOaKnn346GrsDALRSUSmhSZMmqaysTL/85S9VXFyszMxMbdiwQenp6dHYHQCglYraxITp06dr+vTp0frt0Y64DLKtqalx2tfZs2fNmRtuuMGciYuLM2dOnTplzlzuFanX46abbjJnEhISnPaF9o2PcgAAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAb6I2wBTfnsvgTpcPCOzYsaM5I0mdOtlPn8t9sGE09lNRUWHOSG7DSLt3727OdO7c2ZxxGSraoYPbvzPr6urMmXPnzpkz1dXV5szJkyfNmQMHDpgzkttA2/r6+mbJBEFgzrREXAkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG6ZoO3CZbu0yqbq5pjN369bNnJGkmJgYc+bOO+80Z5KTk82Z5cuXmzOSlJKSYs7s2bPHnElISDBnTpw4Yc5UVlaaM5J0/Phxc2bkyJHmjMsk6KqqKnNmzJgx5owklZaWmjMu54PLBPLY2FhzRnKbXO4yVf16cSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN4wwNRBhw727u7UyX6ok5KSzBmXgZAuAw0laefOnebM1q1bzZmsrCxzZsOGDeaMJP3gBz8wZ95//31z5uGHHzZn1qxZY84sXLjQnJGkX//61+aMy7lXVlZmzpw9e9acKS4uNmckt0GuLsNpXQb7uqzNNXf69GmnfV0ProQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwJt2PcDUZaioJIVCIXPmxhtvNGc6d+5szuTm5pozffr0MWck6cyZM+ZMx44dzZnf/va35sztt99uzkhSfX29OdO1a1dzxuXPaciQIeaMy7GTpP79+5szu3fvNmdGjRplzrgM0+zWrZs5I7n9va2srDRnwuGwORMXF2fOSG6DZq0/iyw/W7kSAgB4QwkBALxp8hKaP3++QqFQo1tycnJT7wYA0AZE5TmhAQMG6L333mv42uV5AABA2xeVEurUqRNXPwCAa4rKc0L79+9XamqqMjIyNHnyZB08ePCK21ZXV6uioqLRDQDQPjR5CQ0fPlzLly/Xxo0b9dprr6mkpEQjR4684mfJ5+TkKBKJNNzS0tKaekkAgBaqyUsoOztbjz32mAYOHKgHHnhA69evlyQtW7bsstvPnTtX5eXlDbeioqKmXhIAoIWK+ptVu3TpooEDB2r//v2XfTwcDju9UQsA0PpF/X1C1dXV2rNnj1JSUqK9KwBAK9PkJfTcc88pPz9fhYWF+vjjj/WjH/1IFRUVmjJlSlPvCgDQyjX5f8cdPXpUTzzxhE6cOKGePXtqxIgRKigoUHp6elPvCgDQyjV5Cb3xxhtN/VtGjcuwSsltgKnLkEuXgZDf+c53zBmXgYuS2wDTF154wZz585//bM64/tmeO3fOnDl58qQ5k5WVZc7s3bvXnHEZgitJhYWF5ozLsM+6ujpzpra21pz561//as5IUt++fc2ZzMxMc6Znz57mjMvfdUmKj483Z0pKSkzbV1dXX/e2zI4DAHhDCQEAvKGEAADeUEIAAG8oIQCAN5QQAMAbSggA4A0lBADwhhICAHhDCQEAvKGEAADeUEIAAG+i/qF2LVmHDm4d3LFjR3PGZWigy8DKmpoac8ZlIKTkNoTzN7/5jTnTqZP9NK2qqjJnJGnw4MHmjMtnZR0+fNiccRmC6zIYU3Ib+HnHHXeYM5s3bzZnRo0aZc5MnjzZnJGkmJgYc8Y67FNyO1+TkpLMGVfHjh0zbW/5OcSVEADAG0oIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALxpsVO0a2pqFATBdW8fDofN+6ivrzdnJLepzi7TeEePHm3OrF692pzp1q2bOSNJGzZsMGcGDBhgzlgn+ErSwIEDzRnJ7fjdf//95ozL9OibbrrJnPnHP/5hzkjSLbfcYs64TJd/4IEHzJn8/HxzJiEhwZyRpLq6OnPG5eeD5WfdRUOGDDFnJOnTTz81Z6zfk+VnK1dCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOBNix1gGhsbq9jY2Kjuo7q62inXvXt3c+b8+fPmzNatW82ZPn36mDNlZWXmjCTdc8895szRo0fNmZKSEnPm9ttvN2ckady4cebM+vXrzZnMzExzplevXuaM63DaAwcOmDNff/21ObN27VpzpkePHuaMy7GTpMGDB5szb731ljlz6tQpc+bgwYPmjCRVVVWZM+fOnTNtX1NTc93bciUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN602AGmzaG8vNwpF4lEzJljx46ZMy6DRePi4pplP5JUW1trzhQXF5szI0aMMGfy8vLMGUlKSkoyZx555BFzZseOHebMkSNHzJkzZ86YM5LbcN+6ujpz5plnnjFn3nnnHXPm+PHj5owk/fznPzdnevbsac7s3bvXnBk9erQ5I7n9/IomroQAAN5QQgAAb8wltGXLFo0fP16pqakKhUKXfB5IEASaP3++UlNTFRcXp7Fjx2r37t1NtV4AQBtiLqHTp09r0KBBWrx48WUff/nll7Vw4UItXrxY27ZtU3Jysh588EFVVlZ+68UCANoW8wsTsrOzlZ2dfdnHgiDQokWLNG/ePE2cOFGStGzZMiUlJWnlypV66qmnvt1qAQBtSpM+J1RYWKiSkhJlZWU13BcOh3Xvvfde8aOqq6urVVFR0egGAGgfmrSESkpKJF36MtekpKSGx74pJydHkUik4ZaWltaUSwIAtGBReXVcKBRq9HUQBJfcd9HcuXNVXl7ecCsqKorGkgAALVCTvlk1OTlZ0oUropSUlIb7S0tLr/gmwHA4rHA43JTLAAC0Ek16JZSRkaHk5GTl5uY23FdTU6P8/HyNHDmyKXcFAGgDzFdCVVVV+uKLLxq+Liws1M6dO5WQkKDevXtr9uzZWrBggfr27au+fftqwYIF6ty5s5588skmXTgAoPUzl9Ann3yicePGNXw9Z84cSdKUKVP0l7/8Rc8//7zOnj2r6dOn6+TJkxo+fLjeffddxcfHN92qAQBtgrmExo4dqyAIrvh4KBTS/PnzNX/+/G+zLrPz58+bM//7vJXFqVOnzBmXYYMuxf3pp5+aMy5DTyXps88+M2f+9+X716ugoMCc6dy5szkjSXfeeac5U1paas787z/krtfhw4fNGZeBrJL08ccfmzNDhw41Z1avXm3OuLyNIyYmxpyR3IbTnjhxwpxxGSLsOgDg3LlzTrloYXYcAMAbSggA4A0lBADwhhICAHhDCQEAvKGEAADeUEIAAG8oIQCAN5QQAMAbSggA4A0lBADwhhICAHhDCQEAvGnST1b1yWVKbn19vdO+XCZ2u3xs+b///W9zZu/evebM9OnTzRlJ2rp1qzmzf/9+c6ZLly7mzPHjx80Zye176tixozkzZswYc6ZDB/u/GV0nLbs4dOhQs+yne/fu5ozLn5HkNhHbZX179uwxZ1yPd11dnVMuWrgSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABv2swAUxeug/x69Ohhzrz99tvmzNixY82ZwsJCc+bo0aPmjOQ23NHl2O3atcucue2228wZSUpKSjJn7rzzTnPmo48+Mmeys7PNmd27d5szkttwX5dhqenp6ebMI488Ys68++675owkHThwwJxxGdI7atQoc+bNN980ZySpd+/eTrlo4UoIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALxp1wNMO3Vy+/YHDhxozvTr18+cee2118wZl2GaK1asMGckaeLEiebMkSNHzJm77rrLnDl8+LA5I0kFBQXmzMcff2zO/PGPfzRnFi9ebM48/vjj5owk5ebmmjMDBgwwZ44fP27OLFiwwJxxGUwrSbfeeqs5s2/fPnPG5TicOXPGnJHchtNGE1dCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOBNux5g6io5Odmceeutt8yZ733ve+bMww8/bM5s377dnJGkr7/+2pwpKSkxZ7788ktzZvLkyeaMJJ08edKcefTRR82Z6upqc6aqqsqc2b9/vzkjSTfffLM5U1paas506dLFnOnevbs5c/vtt5szklRcXGzOVFZWmjMuQ0WHDh1qzkjS+fPnzZmzZ8+atg+FQte9LVdCAABvKCEAgDfmEtqyZYvGjx+v1NRUhUIhrV27ttHjU6dOVSgUanQbMWJEU60XANCGmEvo9OnTGjRo0FU/YOuhhx5ScXFxw23Dhg3fapEAgLbJ/MKE7OxsZWdnX3WbcDjs9OQ9AKB9icpzQnl5eUpMTFS/fv00bdq0q75qprq6WhUVFY1uAID2oclLKDs7WytWrNCmTZv0yiuvaNu2bbrvvvuu+JLUnJwcRSKRhltaWlpTLwkA0EI1+fuEJk2a1PDrzMxMDR06VOnp6Vq/fr0mTpx4yfZz587VnDlzGr6uqKigiACgnYj6m1VTUlKUnp5+xTfNhcNhhcPhaC8DANACRf19QmVlZSoqKlJKSkq0dwUAaGXMV0JVVVX64osvGr4uLCzUzp07lZCQoISEBM2fP1+PPfaYUlJSdOjQIf3iF79Qjx49nEabAADaNnMJffLJJxo3blzD1xefz5kyZYqWLFmiXbt2afny5Tp16pRSUlI0btw4rVq1SvHx8U23agBAm2AuobFjxyoIgis+vnHjxm+1oOZUW1vbbLlOnexPv7kM7iwoKDBnXIc7btu2rVn2FYlEzJkPP/zQnJGk/v37mzOXe8HNtbgMjXUZenrkyBFzRpIOHjxoztxyyy3mjGXQ5UUu59DOnTvNGUmaMGGCOePyPcXGxpozLueqJB06dMicsQ4rvlpHfBOz4wAA3lBCAABvKCEAgDeUEADAG0oIAOANJQQA8IYSAgB4QwkBALyhhAAA3lBCAABvKCEAgDeUEADAG0oIAOBN1D9ZtS06c+aMOeMyzdhl8vYHH3xgzvTp08eckaTvf//75syVPmH3ao4ePWrOjBw50pyRpLy8PHNmz5495ox1KrEk3XzzzeZMz549zRlJSk1NNWfq6+vNGZep7x07djRnXM5VSXr99dfNmcGDB5szFRUV5ozrhPSvvvrKnDl58qRp+/Pnz1/3tlwJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA37XqAaSgUcsolJiaaMwMGDDBnYmNjzZlevXqZMwcOHDBnJGnv3r3mTFVVlTkTiUTMmerqanNGkh5//HFzxmUYqcuxcxn+WllZac5IUu/evc0Z65BLSRo3bpw543KOuxxvSbr//vudclbJycnmTHl5udO+LMNFL7rppptM29fU1Fz3tlwJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3bWaAqWVg3kUuA0Il6ezZs+ZMUlKSOVNWVmbObNq0yZxxGTwpSSkpKeZMVlaWOXPw4EFzpqioyJyRpDVr1pgzL7zwgjlz/Phxc6Znz57mTEVFhTkjSfX19ebM0aNHzRmXwZ2rV682Z7p06WLOSNJXX31lztxxxx3mzOnTp82ZHj16mDOS21Bbl/PhenElBADwhhICAHhDCQEAvKGEAADeUEIAAG8oIQCAN5QQAMAbSggA4A0lBADwhhICAHhDCQEAvKGEAADetJkBpi4DAOPi4pz29Z///Mec+ec//2nOJCQkmDMuAyFdhqtKbsMxt2/fbs4cOXLEnHE5HySpT58+5syWLVvMmWHDhjXLfh577DFzRpJ27dplzrj8fXIZNOsyIPRf//qXOSNJaWlp5ky3bt3MGZdhyidOnDBnJKm2ttYpZ2EZeMqVEADAG0oIAOCNqYRycnI0bNgwxcfHKzExURMmTNC+ffsabRMEgebPn6/U1FTFxcVp7Nix2r17d5MuGgDQNphKKD8/XzNmzFBBQYFyc3NVW1urrKysRv///vLLL2vhwoVavHixtm3bpuTkZD344INOH6QEAGjbTC9MeOeddxp9vXTpUiUmJmr79u265557FASBFi1apHnz5mnixImSpGXLlikpKUkrV67UU0891XQrBwC0et/qOaHy8nJJ//8qrsLCQpWUlDT6COdwOKx7771XW7duvezvUV1drYqKikY3AED74FxCQRBozpw5Gj16tDIzMyVJJSUlki59yW9SUlLDY9+Uk5OjSCTScHN5SSQAoHVyLqGZM2fq888/19///vdLHguFQo2+DoLgkvsumjt3rsrLyxtuLu8bAAC0Tk5vVp01a5bWrVunLVu2qFevXg33X3yjZElJiVJSUhruLy0tveIbIsPhsMLhsMsyAACtnOlKKAgCzZw5U6tXr9amTZuUkZHR6PGMjAwlJycrNze34b6amhrl5+dr5MiRTbNiAECbYboSmjFjhlauXKm3335b8fHxDc/zRCIRxcXFKRQKafbs2VqwYIH69u2rvn37asGCBercubOefPLJqHwDAIDWy1RCS5YskSSNHTu20f1Lly7V1KlTJUnPP/+8zp49q+nTp+vkyZMaPny43n33XcXHxzfJggEAbUcoCILA9yL+V0VFhSKRiCZNmuQ01M+iR48eTrmYmBhzpri42Jy5+KpDi/fff9+c6dy5szkjSUOGDDFnPvvsM3PmSi9quRrX4Y6RSMSc6dDB/vqe/v37mzMu59CBAwfMGUn61a9+Zc64DOk9d+6cOeMycPf8+fPmjCQVFBSYMy4DVi0DPy/6+uuvzRlJ+vLLL82Z7t27m7avqanRqlWrVF5erq5du151W2bHAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBunT1ZtK8rLy51yLtO3n376aXPmd7/7nTnjMi3YZSqxJL333nvmTF1dnTnzzQ9PvB4XP+XXKj093Zx56aWXzBmXKdUux85lErt04SNZrCZPnmzOrFu3zpy51lTmy7FOgb7IZUK6y7TztLQ0c8Zl8rYkdevWzSkXLVwJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA37XqAaW1trVPu1KlT5ozLsFSXwZ1fffWVOXPy5ElzRpL69+9vzsTExJgzx44dM2dc/oxcc5MmTTJnPvjgA3Omd+/e5ozLYExJ2rFjhzlz8OBBc2bUqFHmzC233GLOnD171pyRpC+++MKciYuLM2dcfhYFQWDOSFIoFHLKRQtXQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBtKCADgTbseYOo6yK+6utqcWbNmjTlTU1NjzowePdqc6dTJ7TQoLS01Z/bs2WPORCIRc+auu+4yZyTps88+M2dcBlaeOXPGnDlx4oQ5c+utt5ozkts57jI8NzMz05xxOd6uunTpYs5UVVWZMy6Dc+vq6syZlogrIQCAN5QQAMAbSggA4A0lBADwhhICAHhDCQEAvKGEAADeUEIAAG8oIQCAN5QQAMAbSggA4A0lBADwpl0PMK2vr2+2fZ0/f96c6dWrlznTr18/cyYxMdGckaSUlBRz5sYbbzRnkpKSzJk333zTnJGkadOmmTOux8/KZWCl6zn+s5/9zJw5evSoOfP++++bM2VlZebM6dOnzRnJbZDr2bNnnfZlFQRBs+wn2rgSAgB4QwkBALwxlVBOTo6GDRum+Ph4JSYmasKECdq3b1+jbaZOnapQKNToNmLEiCZdNACgbTCVUH5+vmbMmKGCggLl5uaqtrZWWVlZl/x/60MPPaTi4uKG24YNG5p00QCAtsH0woR33nmn0ddLly5VYmKitm/frnvuuafh/nA4rOTk5KZZIQCgzfpWzwmVl5dLkhISEhrdn5eXp8TERPXr10/Tpk276sdAV1dXq6KiotENANA+OJdQEASaM2eORo8e3ehz4rOzs7VixQpt2rRJr7zyirZt26b77rvvii91zMnJUSQSabilpaW5LgkA0Mo4v09o5syZ+vzzz/Xhhx82un/SpEkNv87MzNTQoUOVnp6u9evXa+LEiZf8PnPnztWcOXMavq6oqKCIAKCdcCqhWbNmad26ddqyZcs131CZkpKi9PR07d+//7KPh8NhhcNhl2UAAFo5UwkFQaBZs2ZpzZo1ysvLU0ZGxjUzZWVlKioqcnp3PQCgbTM9JzRjxgz97W9/08qVKxUfH6+SkhKVlJQ0jKmoqqrSc889p48++kiHDh1SXl6exo8frx49eujRRx+NyjcAAGi9TFdCS5YskSSNHTu20f1Lly7V1KlT1bFjR+3atUvLly/XqVOnlJKSonHjxmnVqlWKj49vskUDANoG83/HXU1cXJw2btz4rRYEAGg/2vUU7ebkMgG5srLSnNmxY4c507VrV3NGkrp162bOuEzRdplk7Do9eu3ateZMz549zZlOnex/9VxewOM60dnl/XonTpwwZ44fP27ONNexk9wmVZ87d86cac6J/i0NA0wBAN5QQgAAbyghAIA3lBAAwBtKCADgDSUEAPCGEgIAeEMJAQC8oYQAAN5QQgAAbyghAIA3lBAAwBsGmLZgJ0+eNGc6dLD/u8JlP5J07Ngxc8ZlUGNtba05EwqFzBlJio2NNWcOHDhgzsTExJgzrt+TC5fhuS5/ti6DfV24DjDt3r27OePyd7C5jkNLxJUQAMAbSggA4A0lBADwhhICAHhDCQEAvKGEAADeUEIAAG8oIQCAN5QQAMAbSggA4A0lBADwpsXNjguCQJJ0/vx5zytpnVzmVrlkpP//s7Jo6bPjXLgcP5fj0JzfU01NjTnTkmfHuR676upqc8bl2LW1n3cXv5/r+RkRClx+kkTR0aNHlZaW5nsZAIBvqaioSL169brqNi2uhOrr63Xs2DHFx8df8q+XiooKpaWlqaioSF27dvW0Qv84DhdwHC7gOFzAcbigJRyHIAhUWVmp1NTUa/5PQYv777gOHTpcszm7du3ark+yizgOF3AcLuA4XMBxuMD3cYhEIte1HS9MAAB4QwkBALxpVSUUDof14osvOn9KYlvBcbiA43ABx+ECjsMFre04tLgXJgAA2o9WdSUEAGhbKCEAgDeUEADAG0oIAOBNqyqhV199VRkZGbrhhhs0ZMgQffDBB76X1Kzmz5+vUCjU6JacnOx7WVG3ZcsWjR8/XqmpqQqFQlq7dm2jx4Mg0Pz585Wamqq4uDiNHTtWu3fv9rPYKLrWcZg6deol58eIESP8LDZKcnJyNGzYMMXHxysxMVETJkzQvn37Gm3THs6H6zkOreV8aDUltGrVKs2ePVvz5s3Tjh07NGbMGGVnZ+vIkSO+l9asBgwYoOLi4obbrl27fC8p6k6fPq1BgwZp8eLFl3385Zdf1sKFC7V48WJt27ZNycnJevDBB1VZWdnMK42uax0HSXrooYcanR8bNmxoxhVGX35+vmbMmKGCggLl5uaqtrZWWVlZOn36dMM27eF8uJ7jILWS8yFoJe66667g6aefbnTfrbfeGrzwwgueVtT8XnzxxWDQoEG+l+GVpGDNmjUNX9fX1wfJycnBSy+91HDfuXPngkgkEvzhD3/wsMLm8c3jEARBMGXKlOCHP/yhl/X4UlpaGkgK8vPzgyBov+fDN49DELSe86FVXAnV1NRo+/btysrKanR/VlaWtm7d6mlVfuzfv1+pqanKyMjQ5MmTdfDgQd9L8qqwsFAlJSWNzo1wOKx777233Z0bkpSXl6fExET169dP06ZNU2lpqe8lRVV5ebkkKSEhQVL7PR++eRwuag3nQ6sooRMnTqiurk5JSUmN7k9KSlJJSYmnVTW/4cOHa/ny5dq4caNee+01lZSUaOTIkSorK/O9NG8u/vm393NDkrKzs7VixQpt2rRJr7zyirZt26b77rvP6TNxWoMgCDRnzhyNHj1amZmZktrn+XC54yC1nvOhxU3RvppvfrRDEATN+kFfvmVnZzf8euDAgbr77rv13e9+V8uWLdOcOXM8rsy/9n5uSNKkSZMafp2ZmamhQ4cqPT1d69ev18SJEz2uLDpmzpypzz//XB9++OElj7Wn8+FKx6G1nA+t4kqoR48e6tix4yX/kiktLb3kXzztSZcuXTRw4EDt37/f91K8ufjqQM6NS6WkpCg9Pb1Nnh+zZs3SunXrtHnz5kYf/dLezocrHYfLaannQ6soodjYWA0ZMkS5ubmN7s/NzdXIkSM9rcq/6upq7dmzRykpKb6X4k1GRoaSk5MbnRs1NTXKz89v1+eGJJWVlamoqKhNnR9BEGjmzJlavXq1Nm3apIyMjEaPt5fz4VrH4XJa7Png8UURJm+88UYQExMTvP7668F///vfYPbs2UGXLl2CQ4cO+V5as3n22WeDvLy84ODBg0FBQUHwyCOPBPHx8W3+GFRWVgY7duwIduzYEUgKFi5cGOzYsSM4fPhwEARB8NJLLwWRSCRYvXp1sGvXruCJJ54IUlJSgoqKCs8rb1pXOw6VlZXBs88+G2zdujUoLCwMNm/eHNx9993BzTff3KaOwzPPPBNEIpEgLy8vKC4ubridOXOmYZv2cD5c6zi0pvOh1ZRQEATB73//+yA9PT2IjY0NBg8e3OjliO3BpEmTgpSUlCAmJiZITU0NJk6cGOzevdv3sqJu8+bNgaRLblOmTAmC4MLLcl988cUgOTk5CIfDwT333BPs2rXL76Kj4GrH4cyZM0FWVlbQs2fPICYmJujdu3cwZcqU4MiRI76X3aQu9/1LCpYuXdqwTXs4H651HFrT+cBHOQAAvGkVzwkBANomSggA4A0lBADwhhICAHhDCQEAvKGEAADeUEIAAG8oIQCAN5QQAMAbSggA4A0lBADwhhICAHjzfzIaJpZ8qd69AAAAAElFTkSuQmCC",
"text/plain": [
"![\"weather\"](\"images/gradient_descent.png\")
\n"
]
},
{
"cell_type": "markdown",
"id": "9f23d1c4",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"> ***Question***\n",
">\n",
"> - What is the problem if $\\lambda$ is too small? too big?\n",
"> - What happens if the cost function is a complicated function of $\\mathbf w$ with many local minima?\n",
"\n",
"In practice, the gradient descent method works well but is slow to converge.\n",
"\n",
"There are other methods that have better convergence properties for this iterative process: [Newton-Raphson](https://en.wikipedia.org/wiki/Newton%27s_method), [Conjugate gradient](https://en.wikipedia.org/wiki/Conjugate_gradient_method), etc (see turorial)"
]
},
{
"cell_type": "markdown",
"id": "1775ea7e",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"### Hidden layers"
]
},
{
"cell_type": "markdown",
"id": "b3d623ac",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"In the perceptron model, the complexity of the relationship between the input and the output is limited. \n",
"\n",
"To overcome this limitation, one way is to add one or more *hidden layers* of neurons between the input and output layers."
]
},
{
"cell_type": "markdown",
"id": "f3be17ee",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"The motivation to add these layers is to break down the problem into multiple small task, e.g. \"pick and edge\" and \"find a strait line\" in the case of the digit recognition.\n",
"\n",
"![\"weather\"](\"images/hidden_layer.png\")
"
]
},
{
"cell_type": "markdown",
"id": "57452641",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"The figure above is an illustration of the simplest possible architecture of a neural network: it comprises an input layer of $p$ neurons, one hidden layer, and one output layer.\n",
"\n",
"For historical reasons, neural networks are sometimes called *multi-layer perceptron (MLP)* even if the activation functions in the hidden layers do not constitute a perceptron."
]
},
{
"cell_type": "markdown",
"id": "30b66ecf",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"## Universal approximation theorem\n",
"\n",
"Why could adding more layers help? \n",
"\n",
"The universal approximation theorem states that a feedforward network with at least one hidden layer can approximate any continuous function with an (training) error that can be made as small as we want by adding neurons in the hidden layer.\n",
"\n",
"Of course, the function $\\boldsymbol x \\to \\boldsymbol y = f(\\boldsymbol x)$ should exist for this theorem to be applied.\n",
"\n",
"Hence, neural networks are \"universal\" tools (can be used every time we believe there exists a mapping: digit recognition, classification, uncover physical laws...)."
]
},
{
"cell_type": "markdown",
"id": "623bfcdf",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"One caveat however is that, if we are guaranteed the existence of a network, we have no guidance on how to find such network. We need to build a strategy to find the weights and biases to find that optimal solution.\n",
"\n",
"Plus, minimizing the training error could lead to overfitting ; it is the expected prediction error that should be minimized."
]
},
{
"cell_type": "markdown",
"id": "d52030dd",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"## Training a neural network: backpropagation\n",
"\n",
"Let us consider the case where we have more than one hidden layer.\n",
"\n",
"We write with a subscript $\\boldsymbol a^l$ the activation in the $l^{th}$ layer. So we have,\n",
"\n",
"\\begin{equation}\n",
"\\boldsymbol a^l = \\sigma \\left( \\boldsymbol W^l \\boldsymbol a^{l-1} \\right)\n",
"\\end{equation}\n",
"\n",
"with $\\boldsymbol W^l$ the matrix of weights (including the bias) to compute the activation in layer $l$ and for some activation function $\\sigma$.\n",
"\n",
"The activation in the first layer is fixed to the input, $\\boldsymbol x = \\boldsymbol a^0$, while the activation in the last layer gives the output, $\\boldsymbol y = \\boldsymbol a^L$, resulting in $L + 1$ layers."
]
},
{
"cell_type": "markdown",
"id": "97fc5146",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"We note,\n",
"\n",
"\\begin{equation}\n",
"\\boldsymbol z^l := \\boldsymbol W^l \\boldsymbol a^{l-1},\n",
"\\end{equation}\n",
"\n",
"the weighted sum in layer $l$."
]
},
{
"cell_type": "markdown",
"id": "793f3628",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"> ***Question***\n",
">\n",
"> - If there are $p$ neurons in layer $l-1$ and $q$ neurons in layer $l$, what is the dimension of each of the terms in the equation above? (Hint: don't forget about the bias.)"
]
},
{
"cell_type": "markdown",
"id": "f41f8e8b",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"The cost function is\n",
"\n",
"\\begin{equation}\n",
"C = \\frac{1}{N}\\sum_{i = 1}^N \\| \\boldsymbol y_i - \\boldsymbol a^L_i \\|^2\\, .\n",
"\\end{equation}\n",
"\n",
"We want to compute the derivative of this cost function with respect to **all** weights and biases in the network in order to use the gradient descent algorithm:\n",
"\n",
"\\begin{equation}\n",
"\\frac{\\partial C}{\\partial \\boldsymbol W^l} \\, ,\n",
"\\end{equation}\n",
"for all $l$."
]
},
{
"cell_type": "markdown",
"id": "cc4ac682",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"Observe that the derivatives in one layer is related to the derivatives in the previous layers by the chain rule:\n",
"\n",
"\\begin{equation}\n",
"\\frac{\\partial C}{\\partial \\boldsymbol W^l} = \\frac{\\partial C}{\\partial \\boldsymbol z^l} \\frac{\\partial \\boldsymbol z^l}{\\partial \\boldsymbol W^l}.\n",
"\\end{equation}\n",
"\n",
"Let us introduce the notation\n",
"\n",
"\\begin{equation}\n",
"\\boldsymbol \\delta^l = \\frac{\\partial C}{\\partial \\boldsymbol z^l},\n",
"\\end{equation}\n",
"\n",
"which is a row vector according to our numerator layout convention."
]
},
{
"cell_type": "markdown",
"id": "bbcebe09",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"Let's first derive $\\mathbf \\delta^L$ in the last layer. For a single point,\n",
"\n",
"\\begin{equation}\n",
"C = (\\boldsymbol y - \\sigma(\\boldsymbol z^L))^\\top ( \\boldsymbol y - \\sigma (\\boldsymbol z^L))\\, ,\n",
"\\end{equation}\n",
"\n",
"so,\n",
"\n",
"\\begin{equation}\n",
"\\boldsymbol \\delta^L = -2( \\boldsymbol y - \\sigma(\\boldsymbol z^L))^\\top \\Sigma' (\\boldsymbol z^L))\\, ,\n",
"\\end{equation}\n",
"\n",
"where $\\Sigma'$ is a matrix with diagonal entries which are $\\sigma'(\\boldsymbol z^l)$."
]
},
{
"cell_type": "markdown",
"id": "4a1c41dc",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"The derivative of a vector by a matrix is given by\n",
"\n",
"\\begin{equation}\n",
"\\frac{\\partial \\boldsymbol z^l}{\\partial \\boldsymbol W^l} = \\frac{\\partial }{\\partial \\boldsymbol W^l} \\boldsymbol W^l \\boldsymbol a^{l-1},\n",
"\\end{equation}\n",
"\n",
"which is a tensor of order 3 with a lot of zeros in it (sparsity).\n",
"\n",
"The above equation simplifies in\n",
"\n",
"\\begin{equation}\n",
"\\frac{\\partial C}{\\partial \\boldsymbol W^l} = \\boldsymbol a^{l-1} \\delta^l.\n",
"\\end{equation}"
]
},
{
"cell_type": "markdown",
"id": "aa59b3f2",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"> ***Question***\n",
">\n",
"> - What is the physical interpretation of this derivative?\n",
"> - *(Optional)* Verify that $\\frac{\\partial C}{\\partial \\boldsymbol W^l}$ is the same dimension as ${\\boldsymbol W^l}^\\top$"
]
},
{
"cell_type": "markdown",
"id": "3bd1d19e",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"In addition, the chain rule gives us the following recurrence relation,\n",
"\n",
"\\begin{align}\n",
"\\boldsymbol \\delta^l &= \\frac{\\partial C}{\\partial \\boldsymbol z^{l+1}} \\frac{\\partial \\boldsymbol z^{l+1}}{\\partial \\mathbf z^{l}}\\\\\n",
"&= \\boldsymbol \\delta^{l+1} \\frac{\\partial \\boldsymbol z^{l+1}}{\\partial \\boldsymbol z^{l}}.\n",
"\\end{align}"
]
},
{
"cell_type": "markdown",
"id": "a3a5e0c0",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"Moreover we have\n",
"\n",
"\\begin{equation}\n",
"\\boldsymbol z^{l+1} = \\boldsymbol W^{l+1} \\boldsymbol a^l = \\boldsymbol W^{l+1} \\sigma(\\boldsymbol z^l)\n",
"\\end{equation}\n",
"\n",
"so \n",
"\n",
"\\begin{equation}\n",
"\\frac{\\partial \\boldsymbol z^{l+1}}{\\partial \\boldsymbol z^{l}} = \\boldsymbol W^{l+1} \\Sigma'(\\boldsymbol z^l).\n",
"\\end{equation}"
]
},
{
"cell_type": "markdown",
"id": "243a688f",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"So we have\n",
"\n",
"\\begin{equation}\n",
"\\boldsymbol \\delta^l = \\boldsymbol \\delta^{l+1} \\boldsymbol W^{l+1} \\Sigma'(\\boldsymbol z^l)\n",
"\\end{equation}"
]
},
{
"cell_type": "markdown",
"id": "99b3f4d8",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"> ***Question***\n",
"> \n",
"> - Suppose that there are $p$ neurons in layer $l$ and $q$ neurons in layer $l+1$. What is the dimension of all the terms in the equation above?\n",
"> - Since $\\Sigma'$ is a diagonal matrix, how are you going to compute the product of these 3 elements in a computer program?"
]
},
{
"cell_type": "markdown",
"id": "d243336d",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"\n",
" Main equations\n",
"\n",
"- \\begin{equation}\n",
"\\frac{\\partial C}{\\partial \\boldsymbol W^l} = \\boldsymbol a^{l-1} \\delta^l\n",
"\\end{equation}\n",
" \n",
"- \\begin{equation}\n",
"\\boldsymbol \\delta^l = \\boldsymbol \\delta^{l+1} \\boldsymbol W^{l+1} \\Sigma'(\\boldsymbol z^l)\n",
"\\end{equation}\n",
" \n",
" \n",
"- \\begin{equation}\n",
"\\boldsymbol \\delta^L = -2( \\boldsymbol y - \\sigma(\\boldsymbol z^L))^\\top \\Sigma' (\\boldsymbol z^L))\\, ,\n",
"\\end{equation}\n",
"
"
]
},
{
"cell_type": "markdown",
"id": "45f3da11",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"Let us implement these formula in the [Tutorial: Universal approximation theorem](10_tutorial_neural_networks.ipynb) and see if we can make sense of it."
]
},
{
"cell_type": "markdown",
"id": "631f9b78",
"metadata": {
"slideshow": {
"slide_type": "subslide"
}
},
"source": [
"## References\n",
"\n",
"- [Chap. 6 in Goodfellow, I., Bengio, Y., Courville, A., 2016. *Deep Learning*. MIT Press, Cambridge](https://www.deeplearningbook.org/contents/mlp.html).\n"
]
},
{
"cell_type": "markdown",
"id": "5e186998",
"metadata": {
"slideshow": {
"slide_type": "slide"
}
},
"source": [
"***\n",
"## Credit\n",
"\n",
"[//]: # \"This notebook is part of [E4C Interdisciplinary Center - Education](https://gitlab.in2p3.fr/energy4climate/public/education).\"\n",
"Contributors include Bruno Deremble and Alexis Tantet.\n",
"\n",
"\n", "\n", "
\n",
" \n",
"![\"Logo](\"images/logos/logo_lmd.jpg\")
\n",
"\n",
"![\"Logo](\"images/logos/logo_ipsl.png\")
\n",
"\n",
"![\"Logo](\"images/logos/logo_e4c_final.png\")
\n",
"\n",
"![\"Logo](\"images/logos/logo_ep.png\")
\n",
"\n",
"![\"Logo](\"images/logos/logo_su.png\")
\n",
"\n",
"![\"Logo](\"images/logos/logo_ens.jpg\")
\n",
"\n",
"![\"Logo](\"images/logos/logo_cnrs.png\")
\n",
" \n",
"
\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"\n",
" \n",
"\n", "\n", "
\n",
" ![\"Creative](\"https://i.creativecommons.org/l/by-sa/4.0/88x31.png\")
\n",
"
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License.\n", "
"
]
},
{
"cell_type": "markdown",
"id": "2aa72f62",
"metadata": {},
"source": [
"# "
]
}
],
"metadata": {
"celltoolbar": "Slideshow",
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
"autoclose": true,
"autocomplete": false,
"bibliofile": "biblio.bib",
"cite_by": "apalike",
"current_citInitial": 1,
"eqLabelWithNumbers": true,
"eqNumInitial": 1,
"hotkeys": {
"equation": "Ctrl-E",
"itemize": "Ctrl-I"
},
"labels_anchors": false,
"latex_user_defs": false,
"report_style_numbering": false,
"user_envs_cfg": false
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 5
}
\n",
" This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License.\n", "