{ "nbformat": 4, "nbformat_minor": 2, "metadata": { "kernelspec": { "name": "python3", "display_name": "Python 3.8.10 64-bit ('tf': conda)" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" }, "colab": { "name": "Gradients_in_TF2.ipynb", "provenance": [], "collapsed_sections": [] }, "interpreter": { "hash": "859cadc33cf9c93d9354f622d0bfdc75f7d75fc23a9b22a18dfc646072acf83e" } }, "cells": [ { "cell_type": "code", "execution_count": 1, "source": [ "import numpy as np\n", "import tensorflow as tf\n", "from tensorflow import keras\n", "%matplotlib inline \n", "import matplotlib.pyplot as plt" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Init Plugin\n", "Init Graph Optimizer\n", "Init Kernel\n" ] } ], "metadata": { "id": "hMeZ2inBVTdF" } }, { "cell_type": "markdown", "source": [ "# Tensors (https://www.tensorflow.org/guide/tensor):\n", "### Tensors are multi-dimensional arrays with a uniform type, similar to numpy arrays\n", "\n" ], "metadata": { "id": "8O8igM3SydJD" } }, { "cell_type": "code", "execution_count": 2, "source": [ "tensor1 = tf.constant(4)\n", "print('\\ntensor1: ',tensor1)\n", "\n", "tensor2 = tf.constant([1.0,2.0,3.0])\n", "print('\\ntensor2: ',tensor2)\n", "\n", "tensor3 = tf.constant([[1.0,2.0,3.0],[3.4,1.0,1.0]])\n", "print('\\ntensor3: ',tensor3)\n", "\n", "tensor4 = tf.linspace(0,1,10)\n", "print('\\ntensor4: ',tensor4)" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "tensor1: Metal device set to: Apple M1\n", " tf.Tensor(4, shape=(), dtype=int32)\n", "\n", "systemMemory: 16.00 GB\n", "maxCacheSize: 5.33 GB\n", "\n", "\n", "tensor2: tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32)\n", "\n", "tensor3: tf.Tensor(\n", "[[1. 2. 3. ]\n", " [3.4 1. 1. ]], shape=(2, 3), dtype=float32)\n", "\n", "tensor4: tf.Tensor(\n", "[0. 0.11111111 0.22222222 0.33333333 0.44444444 0.55555556\n", " 0.66666667 0.77777778 0.88888889 1. ], shape=(10,), dtype=float64)\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "2021-09-29 08:48:08.666806: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n", "2021-09-29 08:48:08.667032: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: )\n" ] } ], "metadata": { "id": "1vBO7fdAygTL", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "ed0f1e07-6912-4df8-b685-63a30682f9c5" } }, { "cell_type": "markdown", "source": [ "### Tensors can be converted to numpy arrays" ], "metadata": { "id": "v8V8W9Uf0KKR" } }, { "cell_type": "code", "execution_count": 3, "source": [ "tensor4_np = tensor4.numpy()\n", "print('\\ntensor4 converted numpy array: ',tensor4_np)\n", "print('\\ntype: ',type(tensor4_np))" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "tensor4 converted numpy array: [0. 0.11111111 0.22222222 0.33333333 0.44444444 0.55555556\n", " 0.66666667 0.77777778 0.88888889 1. ]\n", "\n", "type: \n" ] } ], "metadata": { "id": "fl5ut3D00Qy-", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "53ed13b6-34f0-4b15-c1e4-50fcde631f4b" } }, { "cell_type": "markdown", "source": [ "\n", "### All tensors are immutable like Python numbers and strings: you can never update the contents of a tensor, only create a new one.\n", "\n" ], "metadata": { "id": "6RVCxdcx0AaP" } }, { "cell_type": "code", "execution_count": 4, "source": [ "print('6th element of tensor4: ',tensor4[5].numpy())\n", "tensor4[5] = 100.0" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "6th element of tensor4: 0.5555555555555556\n" ] }, { "output_type": "error", "ename": "TypeError", "evalue": "'tensorflow.python.framework.ops.EagerTensor' object does not support item assignment", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/var/folders/6h/mmvp5df90fb3fsqtc5d9mxt40000gn/T/ipykernel_1503/2405235518.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'6th element of tensor4: '\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtensor4\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtensor4\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;31mTypeError\u001b[0m: 'tensorflow.python.framework.ops.EagerTensor' object does not support item assignment" ] } ], "metadata": { "id": "fA8stmW10za6", "colab": { "base_uri": "https://localhost:8080/", "height": 201 }, "outputId": "d4330068-03b2-4a67-8034-d2eae445623c" } }, { "cell_type": "markdown", "source": [ "# Variables (https://www.tensorflow.org/guide/variable):\n", "\n", "### A variable is a tensor whose value can be changed by running operations on it" ], "metadata": { "id": "cZ5MAgDn1kMQ" } }, { "cell_type": "code", "execution_count": null, "source": [ "var1 = tf.Variable(4)\n", "print('\\nvar1: ',var1)\n", "\n", "var2 = tf.Variable(tensor2)\n", "print('\\nvar2: ',var2)\n", "\n", "var3 = tf.Variable(tf.linspace(0,1,10))\n", "print('\\nvar3: ',var3)" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "var1: \n", "\n", "var2: \n", "\n", "var3: \n" ] } ], "metadata": { "id": "CRh-la_X131p", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "57fdee38-987f-4ebf-a427-8a0f6b30aba3" } }, { "cell_type": "markdown", "source": [ "### After construction, the type and shape of the variable are fixed. The value can be changed using one of the assign methods." ], "metadata": { "id": "KV6ZxudhAOZe" } }, { "cell_type": "code", "execution_count": 5, "source": [ "var4 = tf.Variable([1.0,2.0])\n", "print('\\nvar4: ', var4.numpy())\n", "\n", "var4.assign([3.0,4.0])\n", "print('\\nmodified var4: ',var4.numpy())\n", "\n", "var4.assign_add([1.0,2.0])\n", "print('\\nmodified var4: ',var4.numpy())\n", "\n", "var4.assign_sub([1.0,2.0])\n", "print('\\nmodified var4: ',var4.numpy())\n", "\n", "print('\\nrecall that the shape is fixed')\n", "var4.assign([3.0,4.0,4.0])" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "var4: [1. 2.]\n", "\n", "modified var4: [3. 4.]\n", "\n", "modified var4: [4. 6.]\n", "\n", "modified var4: [3. 4.]\n", "\n", "recall that the shape is fixed\n" ] }, { "output_type": "error", "ename": "ValueError", "evalue": "Cannot assign to variable Variable:0 due to variable shape (2,) and value shape (3,) are incompatible", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m/var/folders/6h/mmvp5df90fb3fsqtc5d9mxt40000gn/T/ipykernel_1503/3591645959.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\nrecall that the shape is fixed'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mvar4\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massign\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m4.0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m4.0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/miniforge3/envs/tf/lib/python3.8/site-packages/tensorflow/python/ops/resource_variable_ops.py\u001b[0m in \u001b[0;36massign\u001b[0;34m(self, value, use_locking, name, read_value)\u001b[0m\n\u001b[1;32m 896\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0mtensor_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\" \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 898\u001b[0;31m raise ValueError(\n\u001b[0m\u001b[1;32m 899\u001b[0m (\"Cannot assign to variable%s due to variable shape %s and value \"\n\u001b[1;32m 900\u001b[0m \"shape %s are incompatible\") %\n", "\u001b[0;31mValueError\u001b[0m: Cannot assign to variable Variable:0 due to variable shape (2,) and value shape (3,) are incompatible" ] } ], "metadata": { "id": "2efeQVVBANxl", "colab": { "base_uri": "https://localhost:8080/", "height": 496 }, "outputId": "0965655a-8cb5-436c-bfe1-e8d973c13f20" } }, { "cell_type": "markdown", "source": [ "# Gradients in Tensorflow:\n", "\n", "### To differentiate automatically, TensorFlow needs to remember what operations happen in what order during the forward pass. Then, during the backward pass, TensorFlow traverses this list of operations in reverse order to compute gradients.\n", "\n", "### We use the tf.GradientTape api for this\n", "\n", "#### (see detailed guide here https://www.tensorflow.org/guide/autodiff)" ], "metadata": { "id": "N_YfrXkoVTdG" } }, { "cell_type": "markdown", "source": [ "### Differentiating wrt a scalar variable" ], "metadata": { "id": "gkK-xns8VTdI" } }, { "cell_type": "code", "execution_count": 6, "source": [ "x = tf.Variable(4.0)\n", "print(' x:',x)\n", "\n", "with tf.GradientTape() as tape:\n", " y = x**2\n", " z = 2*y\n", "\n", "print(f'\\n y:', y) # NOTE THE OBJECT TYPE\n", "print(f'\\n z:', z)\n", "\n", "dz_dx = tape.gradient(z, x) # Evaluate gradient\n", "print(f'\\n dz_dx:',dz_dx.numpy())" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " x: \n", "\n", " y: tf.Tensor(16.0, shape=(), dtype=float32)\n", "\n", " z: tf.Tensor(32.0, shape=(), dtype=float32)\n", "\n", " dz_dx: 16.0\n" ] } ], "metadata": { "id": "eGdt58rgVTdJ", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "b548f287-5bd6-446f-bf69-a644f70b0f3d" } }, { "cell_type": "markdown", "source": [ "### Differentiating wrt a scalar tensor (not a variable)" ], "metadata": { "id": "LIUbIw9nVTdM" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.constant(4.0)\n", "print(' x:',x)\n", "\n", "with tf.GradientTape() as tape:\n", " y = x**2\n", " z = 2*y\n", "\n", "dz_dx = tape.gradient(z, x) # Evaluate gradient\n", "print(f'\\n dz_dx:',dz_dx)" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " x: tf.Tensor(4.0, shape=(), dtype=float32)\n", "\n", " dz_dx: None\n" ] } ], "metadata": { "id": "jNHRZ0zoVTdM", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "234764d3-ea59-4a35-f547-03c31b541a7d" } }, { "cell_type": "markdown", "source": [ "### To differentiate wrt a tensor x, it needs to be watched by the tape" ], "metadata": { "id": "238giV5zVTdN" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.constant(4.0)\n", "print(' x:',x)\n", "\n", "with tf.GradientTape() as tape:\n", " tape.watch(x)\n", " y = x**2\n", " z = 2*y\n", "\n", "dz_dx = tape.gradient(z, x) # Evaluate gradient\n", "print(f'\\n dz_dx:',dz_dx)" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " x: tf.Tensor(4.0, shape=(), dtype=float32)\n", "\n", " dz_dx: tf.Tensor(16.0, shape=(), dtype=float32)\n" ] } ], "metadata": { "id": "_qvM4Mn8VTdO", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "0d4ecf0d-da85-4348-ee2d-2d6d028c3dc0" } }, { "cell_type": "markdown", "source": [ "## Non-persistant vs persistent tapes\n", "### By default, the tape is deleted after a single call to tape.gradient()" ], "metadata": { "id": "cLCEMwfvVTdO" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.Variable(4.0)\n", "with tf.GradientTape() as tape:\n", " y = x**2\n", " z = 2*y\n", "\n", "dz_dx = tape.gradient(z, x) #(4 * x)\n", "print(f'\\n dz_dx:',dz_dx) \n", "dy_dx = tape.gradient(y, x) #(2 * x)" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " dz_dx: tf.Tensor(16.0, shape=(), dtype=float32)\n" ] }, { "output_type": "error", "ename": "RuntimeError", "evalue": "ignored", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mdz_dx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#(4 * x)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'\\n dz_dx:'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdz_dx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mdy_dx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#(2 * x)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/backprop.py\u001b[0m in \u001b[0;36mgradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m 1030\u001b[0m \"\"\"\n\u001b[1;32m 1031\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1032\u001b[0;31m raise RuntimeError(\"A non-persistent GradientTape can only be used to \"\n\u001b[0m\u001b[1;32m 1033\u001b[0m \"compute one set of gradients (or jacobians)\")\n\u001b[1;32m 1034\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_recording\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mRuntimeError\u001b[0m: A non-persistent GradientTape can only be used to compute one set of gradients (or jacobians)" ] } ], "metadata": { "id": "ly2VCSo_VTdP", "colab": { "base_uri": "https://localhost:8080/", "height": 357 }, "outputId": "0dbebf92-5388-4cfd-c5c3-15a0734dd180" } }, { "cell_type": "markdown", "source": [ "### Option 1: Use a persistant tape" ], "metadata": { "id": "AZ4JXqulVTdP" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.Variable(4.0)\n", "with tf.GradientTape(persistent=True) as tape:\n", " y = x**2\n", " z = 2*y\n", "\n", "dz_dx = tape.gradient(z, x) #(4 * x)\n", "print(f'\\n dz_dx:',dz_dx) \n", "dy_dx = tape.gradient(y, x) #(2 * x)\n", "print(f'\\n dy_dx:',dy_dx)\n", "\n", "del tape # Clearing the tape, since it doesn't happen automatically now" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " dz_dx: tf.Tensor(16.0, shape=(), dtype=float32)\n", "\n", " dy_dx: tf.Tensor(8.0, shape=(), dtype=float32)\n" ] } ], "metadata": { "id": "6JBMVnc_VTdQ", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "f2f6528f-412e-40eb-d48e-12817c2b03c8" } }, { "cell_type": "markdown", "source": [ "### Option 2: use nested tapes" ], "metadata": { "id": "bfdzCRotVTdQ" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.Variable(4.0)\n", "with tf.GradientTape() as tape1:\n", " with tf.GradientTape() as tape2:\n", " y = x**2\n", " z = 2*y\n", " dz_dx = tape2.gradient(z, x) # Tape 2 gets deleted after this call\n", "dy_dx = tape1.gradient(y, x) # Tape 1 gets deleted after this call\n", "print(f'\\n dz_dx:',dz_dx) \n", "print(f'\\n dy_dx:',dy_dx) " ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " dz_dx: tf.Tensor(16.0, shape=(), dtype=float32)\n", "\n", " dy_dx: tf.Tensor(8.0, shape=(), dtype=float32)\n" ] } ], "metadata": { "id": "2laqDGLoVTdQ", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "f885409e-dcf8-4bca-8d98-8d612ac58cdb" } }, { "cell_type": "markdown", "source": [ "### We can use this strategy to evalaute higher-order derivates" ], "metadata": { "id": "dRBnzEcfM_yM" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.Variable(2.0)\n", "with tf.GradientTape() as tape1:\n", " with tf.GradientTape() as tape2:\n", " y = x**4\n", " dy_dx = tape2.gradient(y, x) # 4*x**3\n", "d2y_dx2 = tape1.gradient(dy_dx, x) # 12*x**2\n", "print(f'\\n dy_dx:',dy_dx) \n", "print(f'\\n d2y_dx2:',d2y_dx2) " ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " dy_dx: tf.Tensor(32.0, shape=(), dtype=float32)\n", "\n", " d2y_dx2: tf.Tensor(48.0, shape=(), dtype=float32)\n" ] } ], "metadata": { "id": "-9HOouSzNKeG", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "d384e798-744c-4299-8b99-af229350e2d4" } }, { "cell_type": "markdown", "source": [ "## Differentiating a tensor wrt to a tensor\n", "### Say $ x \\in \\mathbb{R}^d$ and $y \\in \\mathbb{R}^D$, the $g := grad(y,x) \\in \\mathbb{R}^d$ with $g_i = \\sum_{j=1}^D \\frac{\\partial y_j}{\\partial x_i}$\n", "\n", "### In the following example\n", "### $$x = \\begin{bmatrix}x_1\\\\x_2\\\\x_3\\end{bmatrix}= \\begin{bmatrix}2.0\\\\3.0\\\\4.0\\end{bmatrix}, \\quad A = \\begin{bmatrix} 1.0 & 2.0 & 0.0 \\\\ 1.0 & 1.0 & -1.0\\end{bmatrix}, \\quad y = Ax = \\begin{bmatrix}x_1 + 2x_2\\\\x_1 + x_2-x_3\\end{bmatrix} = \\begin{bmatrix}8\\\\1\\end{bmatrix}$$\n", "### $$\\frac{d y}{d x} = \\begin{bmatrix}1 & 1\\\\2 & 1 \\\\0 & -1\\end{bmatrix}^\\top \\quad \\implies grad(y,x) = \\begin{bmatrix}2\\\\3 \\\\-1\\end{bmatrix}$$" ], "metadata": { "id": "HEsF9mc5VTdT" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.Variable([[2.0],[3.0],[4.0]])\n", "A = tf.constant([[1.0,2.0,0.0],[1.0,1.0,-1.0]])\n", "\n", "with tf.GradientTape() as tape:\n", " y = tf.matmul(A,x)\n", "\n", "print('\\nx: ',x.numpy())\n", "print('\\ny: ',y.numpy())\n", "\n", "grad = tape.gradient(y, x)\n", "print('\\ngrad(y,x): ',grad.numpy())" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "x: [[2.]\n", " [3.]\n", " [4.]]\n", "\n", "y: [[8.]\n", " [1.]]\n", "\n", "dydx: [[ 2.]\n", " [ 3.]\n", " [-1.]]\n" ] } ], "metadata": { "id": "u8syTA6HpzZk", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "cd82312b-75cb-44c7-b8ba-8bd95eadcdf4" } }, { "cell_type": "markdown", "source": [ "### To revaluate the full Jacobian, we need to use tape.jacobian (https://www.tensorflow.org/guide/advanced_autodiff#jacobians)\n", "### Note that since x in the example below is a tensor of size 3x1 and y is a tensor of size 2x1, the Jacobian is a tensor of size 2x1x3x1. " ], "metadata": { "id": "ePTiZ6RJvOnN" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.Variable([[2.0],[3.0],[4.0]])\n", "A = tf.constant([[1.0,2.0,0.0],[1.0,1.0,-1.0]])\n", "\n", "with tf.GradientTape() as tape:\n", " y = tf.matmul(A,x)\n", "\n", "print('\\nx: ',x.numpy())\n", "print('\\ny: ',y.numpy())\n", "\n", "jac = tape.jacobian(y, x)\n", "print('\\ndydx: ',jac.numpy())\n", "print('\\ndydx is a tensor of shape: ',jac.shape)\n", "print('\\nsqueezed dydx: ',tf.squeeze(jac).numpy())" ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "x: [[2.]\n", " [3.]\n", " [4.]]\n", "\n", "y: [[8.]\n", " [1.]]\n", "\n", "dydx: [[[[ 1.]\n", " [ 2.]\n", " [ 0.]]]\n", "\n", "\n", " [[[ 1.]\n", " [ 1.]\n", " [-1.]]]]\n", "\n", "dydx is a tensor of shape: (2, 1, 3, 1)\n", "\n", "squeezed dydx: [[ 1. 2. 0.]\n", " [ 1. 1. -1.]]\n" ] } ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "lDvKdUMFz5jd", "outputId": "15610d34-44c3-4f70-c951-3ba030b604d4" } }, { "cell_type": "markdown", "source": [ "### For an element-wise calculation, the gradient of the sum gives the derivative of each element with respect to its input-element, since each element is independent. Another way to say this is that the Jacobian is a diagonal matrix." ], "metadata": { "id": "-iJa83Yh2U7C" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.linspace(0.0, 4*np.pi, 100)\n", "\n", "with tf.GradientTape() as tape:\n", " tape.watch(x)\n", " y = tf.math.sin(x)\n", "\n", "dy_dx = tape.gradient(y,x)\n", "\n", "plt.plot(x, y, label='y')\n", "plt.plot(x, dy_dx, label='dy/dx')\n", "plt.legend(loc='lower left',fontsize=15);\n", "plt.xlabel('x',fontsize=15);" ], "outputs": [ { "output_type": "display_data", "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAELCAYAAADURYGZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9Z3Rc13mo/Wz03gaNAEl0gr2IIEES7KIoUV2uarFsSZET23FW/N3c2Elu7LjkOtW+cYvlJrqq2JJFS5RIUewEu0iKHR0EQPTegZnZ3489QwEQQJQpZw9wnrVmDXDmlJfgOfPutwspJSYmJiYmMxc/owUwMTExMTEWUxGYmJiYzHBMRWBiYmIywzEVgYmJickMx1QEJiYmJjOcAKMFmArx8fEyPT3daDFMTExMfIqzZ882SSkTRm73SUWQnp7OmTNnjBbDxMTExKcQQlSOtt10DZmYmJjMcExFYGJiYjLDMRWBiYmJyQzHVAQmJiYmMxxTEZiYmJjMcNyiCIQQPxdCNAghLo3xuRBC/LcQokQI8b4Q4o4hnz0lhCh2vJ5yhzwmJiYmJhPHXRbBC8A9t/l8B5DjeD0H/AhACBEHfBXIB1YDXxVCxLpJJhMTExOTCeCWOgIp5WEhRPptdnkI+KVUPa9PCCFihBCzgM3AO1LKFgAhxDsohfI7d8j1IS68CP2dkLEJ4nNACI9cZqJIKTle1kxdex9RIYFEhwWSkxhBTFiQoXKZTBIp4cYJ6G1xbBAwdw2ExRkqFkBDZx+Xb3ZQUt/FwpQo8jPiCPA3PcI+R187VBZC+RHY9lUICHbr6b1VUJYKVA35vdqxbaztH0II8RzKmmDu3LlTk+LSq1C8R/0ckQyLPwJ3/hMEhk7tfFNkwGpn14Wb/PRIGdfqOod9FhkcwBfvzOGpdekEBZgPrNZICaX7Yf834eZ7wz8LioS1n4O1n4eQaK+L9s6Vev75T5epbu0dtj0mLJC7FiTxuS3ZZMSHe10uk0lSsg8O/F91f0k7BITAskdh1lK3XsZnKoullM8DzwPk5eVNbZrO4y9BazmUH1YP8IkfQsUR+MQvIS7TneKOSU1bL3/2s5OUNXaTmxTJv39sKSvTYunss9LSM8AvCyv41u6r/O7UDb7x8GIKsuO9IpfJJOlqhFc+DZVHIXoOPPDfkLJcfTbQDSf/Bw79K5z8Mdz7H7D0414Rq2fAyjffvMpvT95gwawo/s/9GSxOiSIzIYKzla28famW3Rdr2XO5jh88cQcbcj7UbcBEB+w2OPRv6h6yZMGG/wUZG2H2KggMcfvlhLsmlDlcQ29IKReP8tmPgYNSyt85fr+OcgttBjZLKT872n5jkZeXJ93SYqJoD7z6nFrZfeTHkLvD9XPehsrmbh7/yUk6+gb5zieWc+eCRMQo7qkD1xr4xhtXqGrt4adPrWLTPPNh1YqeFnjhfmgpg+3fgDs+NbqpXnsBdv9vqDoJH3keln7Co2JVt/bwqZ+forypm+c2ZvL/3ZU7qlVZ1dLDszvPUNLYxdceWMifrU33qFwmk6SnBf7wLJS+C8seg/v+C4LC3HJqIcRZKWXeyO3e8j3sAj7lyB5aA7RLKWuBPcB2IUSsI0i83bHNO8y7Gz57GOIy4KUnofK4xy5V0tDJx//nOD0DVn7352vYtjBpVCUAsGV+In/8QgHZiZH8xa/O8t6NVo/JZTJJelvhlw9BSyk8/iKs/vOx/bWzlsGfvQbp6+G1z8Ll1zwmVne/lWd3nqGxs5/fPJvPV3YsGNO1OCcujD98bh2b5yXwf16/zPfeLfaYXCaTxDaovosqjsD934WHf+Q2JXA73JU++jvgOJArhKgWQjwjhPgLIcRfOHbZDZQBJcBPgM8BOILE3wBOO15fdwaOvUZsGjy1C2LS4JWnoLPO7Zdo7urnsZ+cxC7hxefWsjh1fJ9xVEggv3x6NUlRwTz9wmmK6jvHPcbEw/R3wa8+Ao3X4JO/gczN4x8TFAaPvQhz8uH3z8D1t9wult0u+ZuXzlNU38n3H7+DdVnjuxMjggN4/lN5PLw8hf/aV8Sxkia3y2UyBd75KlQeg4d+AHmf8VpCi9tcQ97Eba6hodRfgZ/eCclL4ak/QYB7MneklPzFr89y4Fojf/x8AQtToiZ1fFVLDx/9USGB/n7s+ZuNRAT7TFhn+rH7b+HUT+DR38L8eyd3bH+ncie1VsDnT0FkktvE+s+91/ne/hL+z/0LeWZ9xqSO7Rmw8uD3j9HWM8juv15PYqT7/c8mE+Ti7+EPz8Dqz8K9/+aRSxjtGtKfpIXw0Peh6gTs/Qe3nfa1czXsuVzPl7bPm7QSAGXG/+jJldxs7+U/9lx3m1wmk6TiGJx6HvI/O3klABAcCR/5CQz2wptfUnEpN3DgegPf21/CJ/Pm8HRB+qSPDwsK4IdP3EFX/yB//bvz2Oy+tzCcFtRfgV1/BXPWwPZvev3ypiIYyuKPwprPqQf+xgmXT3ezrZevvn6ZvLRY/nzD1LOSVqbF8qk1aew8XsE5M17gfQZ64PXPQ2y6SjeeKgnzYMvfw7U34PKrLovVN2jja7suk5UQztcfXjRmzGk85iVF8o2HFnO8rJkfHSxxWS6TSSKlUgJB4fCJnW7zRkwGUxGMZOs/QmQKvPV3YLdP+TRSSv7379/HJiX/+Yll+Pu55uv7X3fnkhQZwldevcigbepymUyB/d9UaccPfl89rK6w9guQcodyM3U1unSqnx0tp7K5h689uIjgAH+XzvXxvDnsWJzMDw6UUt/R59K5TCbJxVeg5gxs+2eITDZEBFMRjCQoHLZ9DWrPw/svTvk0+642cLSkia/smE+axfXCnciQQL7+0CKu1XXy/OEyl89nMkFunlP1JnnPQMYG18/nHwAP/1DFDPb8/dTFauvl+/tLuHtRkttqAb6yYwFWu53/2lvklvOZTICBbhUgnrVMpYoahKkIRmPJxyF1Jez7Z5UpMknsdsl/7r1OuiWMx1ZPsQp6FLYvSmbH4mT++91iGjrNVZtX2P8tCI1ViwN3kbhAVRxffAUark7pFP+y+yp2KfnH+xa6Tay5ljA+tTadV85Wcb3OzFLzCoXfg86bcM+3wc+4r2NTEYyGn5/6j+mqg2PfnfThb1ys5VpdJ39z1zy393X5u3vmM2iz8xPTKvA8Vaeh5B0o+CKETD7Qf1vWfRGCIuDgtyd96KnyFt54v5bPbspiTpx7c8z/ams2EcEB/N+3pqagTCZBew0c/S4sfBjS1hkqiqkIxmLOalj8MYfGnnhtgdVm57vvFJGbFMkDS1PcLlZ6fDgPLU/l1ydu0NzV7/bzmwzh4L9AmAVW/bn7zx0WpzKQrvwR6i9P6tDv7S8mPiKYv9yU5XaxYsKC+MLWbA5eb+RosVlb4FEO/ovqH3TX142WxFQEt2XL34O1X2URTZBX36uhrKmbL22fh5+LAeKx+PyWbPqsNn56tNwj5zcBbpxU/agK/hqCIzxzjbWfh+CoSVkFl2raOVLcxNPr0wkNci1APBafWpvO7NhQ/mOvma7sMTpq4cJLqj1JbJrR0piK4LZYsmDB/XD6ZxOKFQxY7fy/d4tZNjua7QvdVzA0kuzECO5bMotfFlbQ1jPgsevMaA7+C4QnwKpnPXeNsDhY85dwdRfUXZzQIf9zqJSI4ACeyPfcl0dIoD9/viGT81VtnK0005U9wqkfg7SpDrUaYCqC8Vj3Rehrg/O/GXfXty7VUtPWy19vy5lyTvdE+autOXQP2Pi5aRW4nxsnoOygsgZcTRcdjzWfg+Bo1WVyHCqbu9l9sZYn8ucSHRroUbE+tnI2USEB5v3lCfo74czPYcEDXut6PB6mIhiPOatVn5jj3web9ba77iysICM+nM3zEj0uVm5yJPcsSuYXhRV09d9eLpNJcvJ/1AyBvKc9f63QGFj1DFx7E9pu3HbXnxwpI8DPj6cn2UZiKoQHB/BY/lzeulRLVUuPx683ozj3azVoZt0XjZbkFqYimAjrvqge0qu7xtzlYnU7791o48/WpHksNjCS5zZl0tln5fXzNV653oygsx6u/gmWP+l5a8BJ3mfU+9kXxtylqaufV85U88iKVJKivNMP6Km16Qgh2FlY4ZXrzQhsVjj+Q5i7DmZ/qOWPYZiKYCLk7oC4LJVBNEaPmF8eryAsyJ+PrpztNbFWzIlhwawofnvyBr7YPFBL3vsl2K3esQacxMyFefeoa1tHj/n85sQNBmx2ntvkPVdCSkwo9y6ZxUunq0yr011cfR3ab8C6vzJakmGYimAi+PmroM7N96D69Ic+bu0e4PULN3lkRarHfbdDEULweP5cLt/s4EJ1u9euO22xWeHsLyBzC8Rne/faq56B7sZRrU6bXfLymSrWZ8eTleChDKYxeGZ9Bp39Vl4+XTX+zibjc/LHYMlWil8jTEUwUZZ+EgLDlH9vBC+dqWLAaudTBkx6enh5CmFB/vzmRKXXrz3tKHobOmo8myk0FplbITZDZaiN4FhJEzVtvXxy1Ryvi7V8Tgwr02L51YlK0+p0lcYiNa3ujqcMrSIeDb2k0ZngSFUBeOlV1R/Egc0u+dXxStZmWshNjvS6WJEhgTy0PJU/vX+T9t5Br19/WnH6pxCVasxqzc9PuaNuFH6owOyl01XEhgVylwdTkm/Ho6vmUN7UbU7Kc5Xzvwbhr4bPa4a7JpTdI4S4LoQoEUJ8eZTPvyOEOO94FQkh2oZ8Zhvy2djRWB1Y8SQMdKpgooMjxY1qIP1a44pCnsifS9+gndfeqzZMBp+nuRTKDsDKz6jGcEaw4knwDx5mFbR0D7D3Sh2PrJjtcofRqXLvklmEBfnz+7Pm/TVlbINw/ndqkRHh+azCyeKyIhBC+AM/AHYAC4HHhBDDOmFJKf9GSrlcSrkc+B4wtBl7r/MzKeWDrsrjUdLWKfN9iHvoj+dqiA4N5M4Fxv3nLk6NZtnsaH5jBo2nzrlfqdXaHZ8yToawOFj8EXj/ZTXABnj1vWoGbdIQt5CT8OAAdiyexRsXaukbtBkmh09Tsg+6G2DFE0ZLMirusAhWAyVSyjIp5QDwIvDQbfZ/DPidG67rfYRQ/5EVR6ClnO5+K3su13P/0lmGrdacPJ4/l+KGLs5VtY2/s8lw7HZ4/xXIvtOtIySnxLJHldVZ9DZSqiDx8jkxhrgdh/LRlal09lvZc9n9M71nBOd+rSrVc7YbLcmouEMRpAJDUwqqHds+hBAiDcgA9g/ZHCKEOCOEOCGEeHisiwghnnPsd6ax0bWBHi6x7DFAwPnf8valOnoHbTyyYtR/rlfZsWQWQQF+7Dp/02hRfI8bx6GjWrUfN5r0DRCRDO+/wrmqNorquwy1BpysybCQGhNquoemQlejSkRY+knw915W4WTwdrD4UeD3Usqh9mWaY5jy48B3hRCjtlSUUj4vpcyTUuYlJLhnEMeUiJ4NWVvh/G95/dwN5saFsTIt1jh5HESFBLI1N5E33q81585Olosvq4yw3CnMInY3fv5qZGrxXnafvEJooD/3L51ltFT4+Qk+unI2R0uaqG3vNVoc3+L9l1RtyoonjZZkTNyhCGqAoUuW2Y5to/EoI9xCUsoax3sZcBBY4QaZPMuKJ6CjGnvZER5ekerxvkIT5cHlKTR19XO8tNloUXwH6wBc/iPMv89zXUYny9KPg30QrrzOtoVJRIbosYr86B2pSKk67JpMggu/U4OuEhcYLcmYuEMRnAZyhBAZQogg1Jf9h7J/hBDzgVjg+JBtsUKIYMfP8UABcMUNMnmWeTsY9A/lXr8TWriFnGydn0hEcAC7LpgP6oQp2aeaCi75hNGSfMCs5fREZXKX7RD3LTHeGnCSZglndUYcr75XbSYlTJSmYqi/pIfb8Ta4rAiklFbgC8Ae4CrwspTyshDi60KIoVlAjwIvyuF30ALgjBDiAnAA+LaUUn9FEBRGoV8e9wWeISM22GhpbhES6M/2RUm8damOfquZ3TEhLr6shs9kbTFakg8QgqMhW8j3u8bmZL2GDz24LIXSxm6K6ic/wnVGcvmP6n2B3gmRbokRSCl3SynnSSmzpJTfcmz7JynlriH7fE1K+eURxxVKKZdIKZc53j9cVqkh1+s6+W33SqJlB1QeNVqcYTy0PJXOPisHrxsYUPcV+jrg+luw6CNaBfEGrHb+X8NyAEKuvTrO3t7l7kXJCAG7L9YaLYpvcOWPMGcNROvjORgNs7J4Crx1qZZDcjkyMAwuv2a0OMMoyLJgCQ8ys4cmwrU3wNoHSzVyC6FaSlzus9BuWa7SWjUiITKY1elxvHXJVATj4nQLLRozGVIbTEUwBd6+VMfS9GRE7g5VZTzOnAJvEuDvx31LZ7Hvar3ZMXI8rrwO0XNg9iqjJRnGG+/XEhkSQPjKT0LDZVX1rBH3LplFUX0XJQ2dRouiN0630MLblVXpgakIJklFUzfX6jq5Z1Gy6j3U06wKzDTigWUp9FvtHLjWYLQo+tLfCaUHYP79qlBQE/qtNvZeqePuRckELHxAbRzS0kQH7lnsdA+ZxWW35fJryi0UlWK0JONiKoJJ4qysvHtxMuTcBYHhyg+oEXfMjSU+Ioi9V+qNFkVfSvaBrV/NpNaIw0VNdPZZVe1AzByYtVy5sDQiKSqEvLRYM05wO5qKlTW36BGjJZkQpiKYJG9frmPp7GhSY0IhMBRy79HOPeTvJ9i2IIkD1xrM7KGxuPqGyhaau9ZoSYax+2ItMWGBFGTHqw3z71czMDr0+tLdsXgW1+o6KWs0s4dG5ZZbSO9sISemIpgEde19nLvRxt2Lkj/Y6HQPaZY9dNfCJLr6rZwoazFaFP2wDkDxXjV5zs/YHlFDsdrs7L/WwNb5iQT6Ox5Np8Vy/U3jBBuFHUvUM/DWJdM9NCrObCEfcAuBqQgmxd4rDrfQUEWQvQ0CQlQaokYUZMcTFuTPO1fMB/VDlB+G/g6Y/4DRkgzjbGUr7b2D3LVgSOO7hPlqTOpVvdxDs6JDuWNujOkeGo22GypbSDO34+0wFcEkePtSHdmJEWQnDmlFEBQGmZuVItCo2jIk0J9N8xJ450o9drP30HCuvQFBEer/TSP2Xa0nyN+PDfOG9NISQn2hVByBXr06y969KJnLNzu42Wb2HhrG9bfVuw69qyaIqQgmSEv3ACfLW1S20Ejm3QNtldBw1fuC3Ybti5Ko7+jn/RpznvEt7Ha4vltZcoEhRktzCykl71ypZ22WhYjgEYNx5j+gmpYV7zVGuDG402G57Dez04ZzfTdYcsAyav9MLTEVwQTZf60Bm10Odws5cY42LNLLPbQ1Nwl/P8Fes4f8B1Sfhq56FYTViNLGbiqae9g22jjK1JWqNbVmaaRZCeGkWcJ496qZnXaLvg6oOKqSSHwIUxFMkAPXGkiKCmZxatSHP4yapdL8nCahJkSHBbImM85MIx3KtTfALxDm6TUgZJ/jy3TbaJPu/PxUd9SSfTDY52XJxkYIwdb5iRwrbaZnQJ+sOUMp3a86x87bYbQkk8JUBBNg0GbncFEjW3ITx245nbvDsdrUq8fP9oXJlDR0UWqm+SmK90J6AYREGy3JMPZdqWdxahSzokNH3yH3Xhjs0S477c75SQxY7RSWmK3PATWAJiQG5uQbLcmkMBXBBDhT0Upnv5XNubeZSzzvHkBq6MdVMptVxkBrJTRe025cYHNXP2dvtLJtwW3GZKYXQEAoFL/jPcEmwOqMOMKD/HnXvL/AblPPf8528A8Yf3+NMBXBBDh4vYFAf8H6nPixd5q1DCJTtIsTzI4NIycxwuxGClDi+BLNvstYOUaw/1oDUnJ7RRAYChkbtFtoBAX4sXFeAvuv1ZszCqpPq5oiH4sPgKkIJsT+aw2szoj7cDbHUISAeXer/jVWvXrIb85N4FR5C90zvQld8T6ISYP4HKMlGca+q/XMig5hUcoo8aeh5GyHljLtmtBtnZ9IfUc/l292GC2KsVx/C/wCVEaaj2EqgnGoaumhuKGLLbdzCznJ3QEDXdo1oducm8iAzT6zR1gO9kH5IfVlqlGTuUGbnWMlzWzOTRh/5KnzC0Yz99CW+YkIAe9eneHuoaK3IW2ddvGnieAWRSCEuEcIcV0IUSKE+PIon39aCNEohDjveD075LOnhBDFjtdT7pDHnRy4rm7urfMnoAgyNqoq4+J9HpZqcuSlxxIW5M/Bohn8oFYeU8FWzeID71W20tVvZdPQIrKxiMuA+HnauYfiI4JZPieG/ddmcHZaW5Uj/nS30ZJMCZcVgRDCH/gBsANYCDwmhFg4yq4vSSmXO14/dRwbB3wVyAdWA18VQsS6KpM7OXCtgTRLGBnx4ePvHBgKaQVQ+q7nBZsEwQH+rMuK5+D1xpnrxy1+Rynp9PVGSzKMQ0WN+PsJ1mXfJv40lJztKk99oNuzgk2SrbmJXKhup7FTL7eo13A+8z7oFgL3WASrgRIpZZmUcgB4EZjoJIa7gXeklC1SylbgHUCbSEvvgI3C0ubbp42OJPtOaCpS/UY0YnNuAtWtvTM3jbR4L6RvUC1BNOJwcSMr58YSFTLBUZk5d6n22eV6uR83OiyaYyVNBktiECX7ICoVEnKNlmRKuEMRpAJVQ36vdmwbyUeFEO8LIX4vhJgzyWMRQjwnhDgjhDjT2OidDJjjZU30W+1smYhbyEnWneq9RC+rYHOuelBnZPZQcym0lGrnFmrs7OdSTQebcifgFnIyd63qk6SZe2hxajSxYYEcLpqB95fNCmWHIWurVvGnyeCtYPGfgHQp5VLUqn/nZE8gpXxeSpknpcxLSJjEg+MCh4uaCAn0Iz8jbuIHJeSqlYFm7qEZnUbqDK7m6JU2eqRY/V9MKD7gJCBYNcsrfkerJof+foL1OQkcLm6aee7HmjPQ3668AT6KOxRBDTBnyO+zHdtuIaVsllI6nYc/BVZO9FgjOVLcyOoMCyGBk+hZL4S6IcoOgW3Qc8JNgRmbRlqyDyzZKtiqEYeKGrGEB7Fw1jhpoyPJ3gbtN9QULI3YmBNPU1c/V2tn2CzjkndB+GnXzXYyuEMRnAZyhBAZQogg4FFg19AdhBCzhvz6IOBs07kH2C6EiHUEibc7thnOzbZeShu72Xi7IrKxyLpT9buvPuN+wVxgRqaRWvtVxlDWVqMlGYbNLjlc1MjGeQn4+U3SneD8t5QdcL9gLrAhR1k2TktnxlCyD1LzIFSrPJdJ4bIikFJagS+gvsCvAi9LKS8LIb4uhHDOafuiEOKyEOIC8EXg045jW4BvoJTJaeDrjm2Gc7RYBb1uW008Fpmb1ApBM/dQXnosoYH+HJ1JAb2qkyptNHOL0ZIM41JNO609g5NzCzmJTYO4TNXgTCOSo0PITYrk8ExSBN3NcPOcT7uFANzSEENKuRvYPWLbPw35+SvAV8Y49ufAz90hhzs5XNxIQmQwuUmRkz84NFatEEreha3/6H7hpkhwgD+rM+JmliIoPQDCX7u00cNFjQgBG6ay0ACl2N5/Sbkf/SeYceQFNs6LZ2dhJb0DNkKD9BkD6jHKDgDygyQRH8WsLB4Fu11SWNrMhuz4iaeNjiT7TrVS6NbLDbM+O56Shi5q22fIVKmyAzB7FYRM0g/vYQ4VNbI4JRpLRPDUTpC1RVWxV592r2AusiEngQGbnRPlet33HqN0v+o2mnqH0ZK4hKkIRuFKbQct3QNsmDfF1Ro4Ckukdn7cAkfhktP1Na3paYGb59WXpkZ09Vs5X9U2dWsAVE2E8FMWj0aszogjOMCPI0Uz4P6SUln9WVvAz7etH1MRjILTx1kw0WrP0UhZoXqOaKYI5idHEh8RNDMKf8oPAVK7+MCp8masduna/RUaoyaXaXZ/hQT6k59pmRlxgsZr0FWn3f01FUxFMApHi5uYnxxJYqQLM239/NWqreywVvnefn6CdVnxHC1pnv753qUHIDhKfWFqxNHiZoID/FiZ5mKWSeYWqDmr3VD7jTkzxP1Ydki9Z242Ugq3YCqCEfQO2DhT0eqa2e4kc7PK924td/1cbmS9I9/7ev00zveWDrdc+gbthoQcK2liVXrc5OpTRiNrC0i7dt1unZbOsek+taz8EMSmqywuH8dUBCM4Wd7MgM1+KyfaJTI3q3fnykET1s+EOEFLmer3pFl8oKGzj+v1nazLtrh+stmrVLsJzeIEuUmRxIUHUVg6je8vm1U1/8vYZLQkbsFUBCM4VtJEUIAfqyfTVmIsLNlqalnZQdfP5UZSYkLJTAif3mmkTt+5Zv5bZzHfelfiA078A1VarGZxAj8/wdosC4XT2f1485wqGs00FcG0pLC0mZVzY10320G1m8jcBOWHwW53/XxuZH12PCfLWui32owWxTOUHYToOWDJMlqSYRwtbiI6NJBFKW4aXpK5RVk/rZXuOZ+bKMiKp66jj7Imvdplu43yg+rdtAimH63dA1yp7WBdlhvMdicZm6C3Beovue+cbmB9djy9gzbO3dAr0OgW7HbVpjlzk1bdIKWUHCtpYm2mBf/JtpUYC+eKVLs4gXqGCqer1Vl2CJKWQLgbLDsNMBXBEE6UNSMl7vHfOnE+qOV6xQnWZFnwE9O0f3z9Rehr0261VtHcw832PgrckYjgJGE+hCcoq1Mj5saFkRoTOj0DxoO9UHVq2riFwFQEwygsbSYsyJ+ls2Pcd9KoFDVeULOAcVRIIEtmx0zPBnTOL8X0DcbKMQJnTMYt8QEnQqgRqeV6pSkLIViXZeF4WTM2uz5yuYUbJ9RwIM0WGq5gKoIhFJY2sTojjkB/N/9ZMjZBZSFYB9x7XhdZm2nhfFUbPQPTrC11+WGw5EDUrPH39SKFJU2kRIeQbnHzlLT0DdBZC80l7j2vixRkx9PeO8jV2g6jRXEv5YfAL0ANqp8mmIrAQX1HH6WN3e6NDzjJ3ASD3ar4RyPWZVmw2iWnK1qNFsV92AaV0s3YaLQkw7DbJcfLmlnnSv+qsXD+WzVzPzqfpWnnfiw7pJpKBkcYLYnbMBWBA6eLZF2WB4I/6etVXxjN0kjz0mMJ9BfTyz1087xqxqaZIrha10FbzyBrMz2w0IjLhKjZ2s0xTowKIScxgmPT6f7qbYPa89MqPgCmIrhFYalK64oeT2oAACAASURBVJv0tKiJEBoLyUtVAYpGhAUFsHxODMenU+GPc1WsWXzgRJkas7HWExanEJCxQWUOaZamvC7LwunyFgasesk1ZSoLVTW3ZgsNV3GLIhBC3COEuC6EKBFCfHmUz78khLjiGF7/rhAibchnNiHEecdr18hjvUVhaTNrMy2TnxY1UTI2QPUplXGgEWszLVysaaejT6+xmlOm4ggkLYZwD3zhusDx0mbSLGGkxIR65gIZG6GnGRqueOb8U2SdI035fNU0SVOuOAIBIaqqexrhsiIQQvgDPwB2AAuBx4QQC0fsdg7Icwyv/z3wb0M+65VSLne8HsQAqlp6qG7tdW/a6EjSN4BtQLv+8WuyLNglnC7XYjCca1j7VUaHZqs1m11ysrzZM24hJ04LSLM00vyMOIRQqdnTgoojMGc1BExxjoSmuMMiWA2USCnLpJQDwIvAQ0N3kFIekFL2OH49gRpSrw3OnigeCRQ7mbtWxQk08+PeMTeWoAA/CqeDH7f6NFj7tHMLXbnZQWef1TNuIScxcyA2QztFEBMWxILkqOmhCHpaoO6SdveXO3CHIkgFqob8Xu3YNhbPAG8N+T1ECHFGCHFCCPHwWAcJIZ5z7HemsdG9vc6PlzaTEBlMVoIHswBComDWcu0qQEMC/Vk5N3Z6BIzLDytlq1la3/EytdDwqEUAyhKqPKYaomnEmkwLZytbfb+dSeUxQJqKwFWEEE8CecC/D9mcJqXMAx4HviuEGLU5jJTyeSllnpQyLyHBDZ1BPzgvJ8paWJNpcX9a30gyNkD1GRjoGX9fL7Iuy8KV2g5au/Wqc5g05Udg1jI1tEUjTpS1kJkQTmKUC/MtJkLGRtUIrfaCZ68zSdZmWei32jnv6+1MKo5CQKjPj6UcDXcoghpgzpDfZzu2DUMIsQ34B+BBKWW/c7uUssbxXgYcBFa4QaYJU9ncQ11HH2sy3dBtdDzSN4B9EKpOev5ak8Dpsjjpy3NmB3qg5ox2qzWrzc6pcrXQ8Djp69V7pV7ZaavTnXECH49DlR+BufnTLj4A7lEEp4EcIUSGECIIeBQYlv0jhFgB/BilBBqGbI8VQgQ7fo4HCgCvpj04fZdeeVDnrgHhr517aOnsGMKC/H3bPVR9WgXjNVMEl2520NVv9bxbCCAyWVVUa5amHB2m0rJ9Ok7Q3QQNl7W7v9yFy4pASmkFvgDsAa4CL0spLwshvi6EcGYB/TsQAbwyIk10AXBGCHEBOAB8W0rpdUWQEBlMZny45y8WHKnMSs0e1CDH2MSTvpw5VHFUxQfmrjFakmE4latXFhqgrILK49rFCdZmWjh7o5W+QR+NE1QeU++mIhgbKeVuKeU8KWWWlPJbjm3/JKXc5fh5m5QyaWSaqJSyUEq5REq5zPH+M3fIMwm5vRcfcJK+XrWa6O/yzvUmyJpMC9fqOmnx1ThBxVEVjA/xQEGgCxwvayYnMYKESC+5E9LXw0An1L3vnetNkDWZFgasdt+tJyg/AoFh0zI+ADO8stir8QEn6RvAboWqE9675gRw/g1O+aJVMNjriA8UGC3JMAZtds5UtHg2bXQkzjiBZlbnqow4/Hy5nqDiiLI2/QONlsQjzGhF4NX4gJM5+apzoWb1BEtSYwgJ9PPNB1XX+EBNOz0DNvIzvHh/RSarEamaKQLnVDafjEN1NULjNe3uL3cy4xVBfISX4gNOgiMg5Y4PfI6aEBTgR15anG/GCTSNDzj/lvnetDhBWQU3joNdL3/8msw4zlW1+V6cwJmFZSqC6ccH8YE478UHnKQXqOHXA3rNc83PiONaXQdtPT4WJ6g4quoHQtw0B9hNnChrJjsxgvgIL6cbpm9Q9QSaxQnWZqk4gc+NR604BoHhkLLcaEk8xoxVBDdanPEBA5qTpa13xAn0qidYk2VBSh+LEwz2KtdQml7xAavNzpmKVvIzvGwNwAd/C83cQ3npKk7gc/UqlcdUf6FpGh+AGawIDIkPOJmb76gn0Ms9tHR2NMEBfr5V+FN9Rsv4wJVaVT+Qb8T9FTUL4rK0UwRRIYEsTPGxeoJuR0dXZxB+mjKDFUEL8RHBZCV4MT7gJDhSmZmaxQmCA/wd9QQ+9KBqGh+4tdAwwiKAD+oJNIsT5GdYOHejzXf6Dt0oVO+mIph+qPhAM/lGxAecpBVo2XdoTabqO9Te4yPzCSqOqqE/mvUXOlnWQma8F/oLjUX6Buhv1y5OkJ8RR7/VzoWqdqNFmRjO/kIp07N+wMmMVATVrb3UtvcZt1oDtcKwD2o3nyA/Iw4p4XSFD7iHBvvU30+z1ZrNLjlV0eL9bKGhOGsqNHM/rnbMJzjpK+6himMwZxUEBBktiUeZkYrAabYb4r91MneNcmlo5h5aPjfGESfwgQe15izY+rULFF+tVfMHvFo/MJKoFDWfoLLQOBlGISYsiNykSN9IU+5thfpLKrljmjMjFcHJ8hbiwoPISfTg/IHxCImG5CXardiCA/xZMTfGNx7UymOAgLS1RksyjA8WGgZaBKCsghuF2s0xds4nGLTpJdeHqDyOmj+g10LDE8xQRdDMqvRY4+IDTtI3KNfGYJ+xcoxgdYaFyzd9YI5xxVFIWgShsUZLMoyT5S2kWcKYFe2h+cQTJa1ArWo1m2OcnxFH76CN96s1jxNUHgP/YEjNM1oSjzPjFMHNtl6qWnqNNdudpBUo10bNWaMlGcaajDjsEs5WtBotythYB6DqlHZuIbtdcrqixZj6gZE4/zaauR9XO/422menVRxVQ+oDDQr4e5EZpwicN5/hZjs4XBpCu3zvFXNjCfQXnND5Qb15Dqy92pnt1+s7aesZ1GOhEZsG0XO0u78sEcHkJEboXa/S58i40uz+8hQzTxGUtRAVEsD8ZA3aFYfGQtJi7SZKhQb5s2x2DCd1flCdq1zNLIKTusQHnKQVqICxlEZLMoz8zDjOVrRg1TVOcOMESLt295enmHmKoLyF1Rlx+PsZHB9wkl4AVaeVq0Mj8jPjuFjTTne/XgNOblF5DBLmQ3i80ZIM42R5C6kxocyODTNaFEV6AfQ0QVOR0ZIMIz/DQveAjUs3O4wWZXQqj4FfoHINzQDcogiEEPcIIa4LIUqEEF8e5fNgIcRLjs9PCiHSh3z2Fcf260KIu90hz1g0dPRR3tSth9nuJK1AuThunjNakmGszrBgs0veu6FhnMBmVSs2zVZrUkpOlWsSH3Ciad8hp8WkbT1BxTFIXQlBmih0D+OyIhBC+AM/AHYAC4HHhBALR+z2DNAqpcwGvgP8q+PYhagZx4uAe4AfOs7nEU44UiJXa/WgrlPvmrmHVqbF4u8n9HQP1V2AgS7t/LeljV00dw/o4xYCiMuEyFnaBYwTI0PIjA/XM025v0stzDS7vzyJOyyC1UCJlLJMSjkAvAg8NGKfh4Cdjp9/D9wpVO7mQ8CLUsp+KWU5UOI4n0c4Vd5MRHAAi1I0iA84CY9XLg7N6gkiggNYnBqtZ2ZHhZ7xAWfwUyuLUwi12Kg4pmWc4HRFCza7XnJRdRKk7YNFmiZcrG7nc785y41m97elcYciSAWqhvxe7dg26j6OYfftgGWCxwIghHhOCHFGCHGmsbFxSoJKCRvnxRPgr1loJK1A3XyaDRxfkxHHhap2/QaJVB5T3TUjk42WZBgny1tIjAwmzaKZOyGtALrqoKXMaEmGkZ9hobPPytVazeIElYWqO/CcfKMlGcaRkkZ2X6wjIiTA7efW7BtxbKSUz0sp86SUeQkJCVM6x7ceWcIPn1jpZsncQHqBcnXUXTBakmHkZ8YxYLPrFSew29T0Lc3MdhUfaCY/02J8oeJINJ1j/EE9gWbuocpjqjtwcKTRkgzjZFkL85IiiAt3f98jdyiCGmDOkN9nO7aNuo8QIgCIBponeOz0J03PBmEr05wNwjR6UOsvqxxvzfq/VDb3UN/Rr1eg2En8PAhP0K7vUEpMKHPiQvUKGA/2qgJPzdyOatBRi8fcju5QBKeBHCFEhhAiCBX83TVin13AU46fPwbsl1JKx/ZHHVlFGUAOcMoNMvkWkcnK1aFZQC86NJCFs6L0ihM4/0aaWQTOv9EanQLFTpxxAs3uL1DuoVMVLdh1iRNUn1aDjjRTBJdvdtA9YPNYoovLisDh8/8CsAe4CrwspbwshPi6EOJBx24/AyxCiBLgS8CXHcdeBl4GrgBvA5+XUmrmkPYS6QXmIJGJUHEUYtIgerbRkgzjZFkLlvAgshIMbGR4O9LWQ3sVtFYaLckw8jPiaOsZpLihy2hRFBWORoaaDTrydEcEt8QIpJS7pZTzpJRZUspvObb9k5Ryl+PnPinlx6WU2VLK1VLKsiHHfstxXK6U8i13yOOTpK1Xg0TqLxstyTDyMzUaJGK3K/eGZvMH4INCRe3iA07S9ew75BwVq43VWXlMdQXWddBRpGf6HvlMsHjao+mDujpdo8KfxmvQ26Kd2V7d2kNNW6+e8QEnCQtUSxPN4lCzY0NJiQ7RIw5l7Z+xg45MRaAL0bOVy0OzzI7Y8CDmJ2sySETT+MCt+gEjBx2Nh5+fo++QXveXEILVGXGcLG9GGl3nUPMeWPu0W2h4Y9CRqQh0In29cn1oNkgkPyNOj0EiFUchyqEwNeJkWTMxYYHkJumVbvgh0gqgtQLa9UrMy8+00NQ1QGljt7GCOJWkZoVkp8qdCw3TIpgZpBUo10fjVaMlGcaaTIvxg0SkVBZBeoHKgtGIk+UtrE6Pw0+XRoZjoan7MV+X+QQVRyFxEYTp5eI7Wd7M3DjPDjoyFYFOaDxwHAx+UJuKobtRO7P9ZlsvN1p69HYLOUlaDMHR2rkfM+LDSYwMNjZOYBtUg440czva7d5pZGgqAp2IcQwS0cyP6xwkYuiD6vybaBbI07p+YCR+/moYkmYWgRCC/EyLsXGCm+dgsEe7+6u4oYvWnkGPLzRMRaATQqgVr6YNws4YOUik4hhEJKtumhqh1aCjiZBWAM0l0FlntCTDWJMZR31HPxUeaKg2ISqOqHfNLM5b9QOmRTDDSF+vBok0XjdakmE4B4lcNmKQiO7xAZ0GHY2HtnECRz2BUWnKFcdUiq1mg45OlDWTGhPKnDjPNjI0FYFu3IoTHDFWjhHcGiRiRJygpQw6a7VbrdU7Bh2t8YX4gJPkZRAUqV0cKishnPiIYE4YoQhsg2rQkWbxASklJ8pavHJ/mYpAN2IzIDJFuxXbrUEiRsQJnEoxfYP3r30bnF9aWs0fGA//ANU+QbOFhooTxHGyvMX7cYLaCzDYrd1Co7ihixYvDToyFYFuCKHcQ1rGCVSDMK8PEqk4ChFJEJ/j3euOw8nyFiKDA1io06CjiZCxQc0w7qw3WpJhrMm0UNvex40WL8cJKvRMRHAuNNaaFsEMJb0AuhtUyqRGrMmMo7PPyhVvxgmkVA9q+nrt4gMnyppZ5UvxASfOLzzNstPWONOUvW11VhxVrbojEr173XE4WdZCakwos2M9Vz/gxFQEOuJ0gWj2oDpXJl714zrjA5qt1ho6+yhr7Na7v9BY3IoT6HV/ZSdGYAkP8u79ZbM64gN63V8qPtBMfqZ3GhmaikBH4jJVqqRmD2piVAiZCeEc9+aDqm18QK1afSpQ7MQ/QNUTaHZ/GRInqLsAA53axQdKGrpo7h7w2v1lKgIduRUnOKpdnGBNpoXT5V6sJ3DGByzZ3rneBDle2kxkcACLfC0+4CR9vbZxgpq2Xqpbe71zQTM+AJiKQF8yNkBXvXZxgrWZFjr7rd6pJ5ASyo9oGR84WdbM6ow4Avx99BG6NcdYr+wh5wrYa1Zn+REVH4hM9s71JsgJL8YHwEVFIISIE0K8I4QodrzHjrLPciHEcSHEZSHE+0KITw757AUhRLkQ4rzjtdwVeaYVTldIxWFj5RiBM5XNK37c5lLoqtNutVbf0UdZUzdrs3zQLeQkeRkER2nnHsrxZpzANgg3jmvndvR2fABctwi+DLwrpcwB3nX8PpIe4FNSykXAPcB3hRBDx//8rZRyueN13kV5pg9xmRCVqlYsGpEYGUJWQrh3HlRN4wPHS539hXxYEfgHwFw94wRrMi0cL/VC36Gb52GgS1nfGuHt+AC4rggeAnY6ft4JPDxyByllkZSy2PHzTaABSHDxutMfIdQXoK5xgopWz8cJNI4PRIUEsGCWj8YHnKSvh+Zi7foOrc1S9QSVnu475LS2NVtoeDs+AK4rgiQpZa3j5zog6XY7CyFWA0FA6ZDN33K4jL4jhAi+zbHPCSHOCCHONDY2uii2j5CxQfUdatBrPsHaLAtd/VYueTJOoHH9wPGyZvIzLb5XPzCSW3ECvawCp8utsNTDVmf5EUhcqF1/ocLSZlKiQ7wWH4AJKAIhxD4hxKVRXg8N3U8qO27MpasQYhbwK+AzUkrnUvIrwHxgFRAH/N1Yx0spn5dS5kkp8xISZohBcStOoJd7yNlSwaPuoeYSLeMDNY75A95crXmM5KUqTlCuVxwq0zGfwKMBY+sAVJ3Uzhqw2yXHy5pZlx3vtfgATEARSCm3SSkXj/J6Hah3fME7v+gbRjuHECIKeBP4BynliSHnrpWKfuAXwGp3/KOmDbFpakaBZg9qQmQw2YkRt3zlHqH8kHrP2OS5a0wB57/ZpwPFTvwDHG3P9VpoCCFYl+XhOEHNWTV/IGOjZ84/Ra7WddDWM8g6L99frrqGdgFPOX5+Cnh95A5CiCDgNeCXUsrfj/jMqUQEKr5wyUV5ph8ZjjiBZnOM12VZOF3RwoDVQ3KVHVLziTWbP3CirJlYX5hPPFEyN6nq7bYqoyUZxtosC01d/ZQ0dHnmAhVHAKFdx1GjFhquKoJvA3cJIYqBbY7fEULkCSF+6tjnE8BG4NOjpIn+RghxEbgIxAPfdFGe6Uf6Ruhrg/qLRksyjHVZFnoGbFyobnP/ye129aBmbNQvPlDaTH6GRf/5xBPFuSLWzOpcl6X89h6LE5QfhuQlEPqhjHdDKSxtJjM+3KPziUfDJUUgpWyWUt4ppcxxuJBaHNvPSCmfdfz8ayll4JAU0VtpolLKrVLKJQ5X05NSSg+pfx/GmdqmWRrpmkwLQsCxkib3n7z+IvS2qtWqRlS19FDT1js93EJOEhdCWPwHrjhNmBMXRmpMqGfcj4N9aj6xZm6hQZudk2XNhtxfPloWOYOISoG4LO38uDFhQSxOiaawxAMPqnN1qtmD6lR63vbfehQh1N+5/LB2acprsyycKG/G7u6259WnwNavXaD4Yk073QO2W9aQNzEVgS+QsREqC1WnRI1Yl23hXFUrPQNulqv8MFhylBLUiKMlTSQ6AuXTioyNqsNrc4nRkgxjXZaFtp5Brta5OU25/AgIf9V4TyMKHQuNNV4YRDMSUxH4Apmbob8Dbp4zWpJhFGTFM2iTnCp3Y/9426BSeppZA3a75HhpM+u9nNbnFZwuuLKDhooxEqeLxO3uobKDkLoSQqLde14XKSxtZn5yJJaIMcupPIapCHyBjI2A0O5BzUuPJdBfuPdBrXlPlf1rFh+4VtdJc/cA67L1Kj5yC7EZED1Hu4DxrOhQMuPD3RuH6mtXqaOZm913TjfQN2jjTGWrIW4hMBWBbxAWB7OWQdkBoyUZRlhQACvmxnKs1I0PavlhVFqfXv7bQse/sSB7GsUHnDjjBBVHtEtTLsiO52S5G9OUK46CtGmnCN670cqA1W5Y/MlUBL5C5maV6dCvV2JVQVY8l2920NYz4J4Tlh9SaX1hek3+OlrSRGaC99P6vEbGJpWppVma8vqceHoGbJy70eqeE5YdhMAwmL3KPedzE8dLm/ETsNqA+ACYisB3yNwMdkfbXI0oyLYgpZv8uIO9quxfs/jAgNXOqfIW1k9Ht5ATTesJ1mZZ8BNKEbuFsoOqmjogyD3ncxNHiptYOjuGqJBAQ65vKgJfYe4a8A/WLk6wbE4M4UH+7nEP3TgOtgHtzPbzVW30DNgomM6KIGqWGtBSqpf7MSokkGVzYtyjCNpr1FS2zM2un8uNtPcM8n51GxtzjLu/TEXgKwSGKmWgmSII9PdjdUace+oJSveDfxCkrXP9XG7kaEkTfsLH5w9MhKytKmNrsM9oSYaxITueC1VttPcOunYiZ9Fc5mZXRXIrhaVN2CVsmGdcM01TEfgSmZuh/hJ0jdrbzzDW5yRQ1tRNdauL/eNLDyhlFxTuHsHcRGFJE0tmxxAdaozZ7jWytoK1F6pOjL+vFynIjscu3dDttuwghCeoamqNOFzcRERwAMvnxIy/s4cwFYEvkblZvWvmx3WatIeLXDDfO+uUksva6iap3ENXv5XzVW2sn47ZQiNJKwC/QGWZacSKubGEBflztNiF+0tKpQgyNoGfPl97UkqOFDeyJtNCoIHzr/X5i5iMz6xlEBKjXRppdmIEs6JDOFLswsAgp8tLM0VwsqwZq11SYFB+t1cJjlAWmWaKICjAj/yMONfqCRqvQVe9dm6hyuYeqlt72TjP2PvLVAS+hJ+/yu4oPahVXxghBBtzEjha0jT18ZWl+1Xzs6Ql7hXORQ4VNRIa6M/KdL26VHqMrC1Qd1Fb92NNW+/UTuAMgmdudpdIbuGIQ7kZnZFmKgJfI2srdFSr7AeN2DAvns4+Kxeq2yd/sN2uHtSsLVqZ7aAUwbosC8EB/kaL4h2cFplmSQnOL8pjU3UPlb6rmjfGzHGjVK5zpKiR1JhQMuKNjYvp9dSZjE/2NvVess9YOUawPjsePwGHi6bgHmq4DN0N2rmFKpq6qWzuYVPuDBmNCpC8DELjtHMPzUuKIDEymENTcT8O9qqK4py73C+YC1htdo6XNrMhx/j+VS4pAiFEnBDiHSFEseN9VPtZCGEbMpRm15DtGUKIk0KIEiHES45pZia3I2YOJMyH4neMlmQYMWFBLJ0dw+GpPKjOL53MLe4VykWc/5ZNBqb1eR0/P2WZle7Xzv24aV4CR4oaJ+9+rDgG1j7I1ksRXKhuo7PfyoYc4+8vVy2CLwPvSilzgHcdv49G75ChNA8O2f6vwHeklNlAK/CMi/LMDLK3QeUxGOg2WpJhbMxx5Hv3TDLfu3Q/JC5SRU0aceh6I+mWMNIseqWzepysO1VgteGK0ZIMY3NuIh19KotrUpTsg4AQ7cZSHiluQgg95lu4qggeAnY6ft6Jmjs8IRxzircCzjnGkzp+RpO9TVXgVhw1WpJhbJyXgF0yuSrjgR6oPK5WoRrRb7VRWNrMxplkDThx/l9o5h5anxOPv5/g4PVJWp0l70D6elWUqREHrzeyNDWa2HDjHSGuKoIkKWWt4+c6IGmM/UKEEGeEECeEEM4vewvQJqV0TjWpBlJdlGdmkLZONc7SLE6wfE4MkcEBk0sjrTympkVppgjOVLTSO2ibWW4hJ1EpkLBAu/srOjSQO+bGcLBoEhlNLeVq4I4ztqYJzV39XKhuY8v8RKNFASagCIQQ+4QQl0Z5PTR0PymlBMZyKqZJKfOAx4HvCiGyJiuoEOI5hzI509joQr76dCAgWKWRahYnCPD3Y122hcNFTciJ+peL9iillrbes8JNkkNFjQT5+03/thJjkXOX8q33dxotyTA25yZyqaaDhs4JtsFwKjPN4gOHihqRErb6iiJwDKVfPMrrdaBeCDELwPE+qqqWUtY43suAg8AKoBmIEUIEOHabDdTcRo7npZR5Usq8hIQZuEobSfY2aC2H5lKjJRnG5txEatp6KaqfQLtsKZUiyNwMgSGeFm1SHC5qZFVGLOHBAePvPB2Zd7fqdqtZEzqnhTbhKvaSdyEmDSyTXnt6lAPXG4mPCGZxih5T0ly9y3cBTwHfdry/PnIHRyZRj5SyXwgRDxQA/yallEKIA8DHgBfHOn4qdHR00NDQwOCgi02qdCZ8Ddz9MlQ1Q4ObZgG4QGBgIImJibdWOPuu1pObHHn7gxqvQfsN2PAlL0g4cera+7hW18nf3zvfaFGMY04+BEdD8R5Y+OD4+3uJRSlRJEQGc+B6Ax9bOfv2O1v7VTuWZY+q4TuaYLXZOXS9ge2LkvHz00MuVxXBt4GXhRDPAJXAJwCEEHnAX0gpnwUWAD8WQthRFsi3pZTOdIS/A14UQnwTOAf8zEV56OjooL6+ntTUVEJDQw3Pz/Uo9f7KTWTwakdKSW9vLzU1NSQlJbEkNZr91xr4/Jbs2x9YtEe9z7vb80JOgoPXlWE7IwPFTvwDIftO5X6027Up9BNCsHleAnsu12G12Qm4XX+eG8dhsFu7+oH3brTR0WfVxi0ELgaLpZTNUso7pZQ5DhdSi2P7GYcSQEpZKKVcIqVc5nj/2ZDjy6SUq6WU2VLKj0sp+13750BDQwOpqamEhYVNbyUAEBKlJpYZPF5QCEFYWBipqak0NDSwdX4i791opaV7HEulaI+aRhaV4h1BJ8i+qw2kxoSSmzSORTPdmXe3SiOtPW+0JMOYcBpp8TuqrblmY08PXG8gwE+w3sD5AyPRQ827kcHBQUJD9UoT8xjBUYAdBvQI6IWGhjI4OMidCxKR8oOV9aj0tKhpZDl6WQO9AzaOljRy18Kk6b+QGI/sbYCA4r1GSzKMCaWRSgnX31IdVYMjvCfcBDhwrYG89FjDppGNxrRTBMDMeYCDI0D4Qd8U+vt4AOfffXFKNAmRwbx77TaKoHS/GiI+7x4vSTcxjpU00TdoZ9uCsTKhZxDh8Wq2b9HbRksyjOjQQFbOjWXf1fqxd2oqgpZSmH+f9wSbADVtvVyr62RLrj5uIZimimDGIPyUVdDXrlU7AD8/wZ3zEzl8vZEB6xhuq6I9EGaB1Du8K9w47LtaT2RwAKszjBkirh3ztsPNc9B5my9dA9i+KIlrdZ3caB5jGNK1N9V77g7vCTUBnFayTvEBMBWB7xMSDXYrDLo4HczNbJ2fSGe/lTMVLR/+0G5T1Z7Zd6nW2ppgt0v2XW1gU24CQQHmowF8tsKZ5wAAHsdJREFU4LrTzD1010Jlse29Ujf6DtffUvM7osfJLPIy+682MDs2lOxEvdxV5t3u64REAUIb95CTgux4ggL8RncPVZ+G3lbtsoUuVLfR1NV/60vGBBXMj0zRzj2UZglnfnIk71wZxVLpalD3WK5ebqGufitHSpq0jD+ZisDX8QtQM341UwThwQGszbTw7tX6D1cZX/2TGomYfacxwo3Bvqv1+PsJNs/Ty2w3FCGUe6V0v+oLpRHbFyZxuqLlw9lp198CJMy/1xC5xuLg9QYGrHbuWZRstCgfwlQE04GQaNVm1zrBsnsvsW1BIhXNPRQ3DKkylhKu7FK9hUL0qKp0su9KA6vSY4kO0yebQwsWPqhcj5r1Htq+KBm7hHdHBo2v74bouZC02BjBxuDtS3VYwoPIS9cv/mQqAg3ZvXs3fn5+lJeXD9teXl6On58fr78+ogDb+YWqmVVw96JkhIDdF2s/2Fh7XlUTL9CnWhXgRnMP1+s7zWyh0Uhbr4bVXN01/r5eZFFKFCnRIewd6h4a6FbT1ebfq1U1cd+gjQPXGti+KAl/TaqJh2IqAg25++67SUlJYefOncO2v/DCCyQmJnLffSN8nwHBEBCqnSJIjAphVVocb10cEtC7sguEv3Zpfc6go6kIRsE/QH2xFu1RbRs0QQjB9kXJHClupHfApjaW7leWca5ebqHC0ia6B2zcraFbCFxvMeET/POfLnPlZoch116YEsVXH1g0qWP8/f359Kc/zc6dO/nqV7+KEAIpJTt37uTJJ58kIGCU/7aQaOiqA9ugag+gCfcuSeZrf7pCSUMX2QnhalWZvh7C9DKP37xYy4JZUaQbPDtWWxY8BOd+rVbbGgX571qYxAuFFRwublRfstd2q2chbZ3Rog3j7Ut1RAYHsC5Ln2rioZgWgaY8/fTTVFZWcvDgQQAOHDhAZWUln/nMZ0Y/IDRGvfdNcnqTh7lnsZo69tbFWmi4qnrDa9TEDKC6tYdzN9q4f6leE9K0InOTqlm5opd7aHVGHFEhAey5XKeslWtvKmtAo8WQ1WbnnSv1bF2QqG1a8oywCCa7IteBzMxMNm/ezC9+8Qu2bNnCL37xC1avXs2iRWP8WwJC1Ku3DcL1aZaWHB3CyrRYdl+q46/8jwEC5j9gtFjDePN9FcN4YKlePY+0IiBYVYFffxNs39XmizbQ34+7Fiaz90odAwurCOpvh8UfM1qsYZyqaKG1Z1DLbCEneqonEwCeffZZ/vCHP1BTU8Orr746tjUAKjAWGgsDXWA1vi31UHYsTuZqbQcDF/8Ic9dApF5++Dcv1rJ0djRzLWFGi6I3Cx9U9R+ajUh9cHkKnX1Wmk/+TlWrZ24yWqRh7LlUR3CAH5ty9VmgjcRUBBrzkY98hKCgIB599FHsdjuPPvro7Q8I0dM9tGPJLNJFLUHNV7XLFqps7ub96nbTLTQRsu5U0+Q0yx4qyLKQGmYnrvpdWPiQNtYKKLfQ7kt1bM5NICxIXweMqQg0JiQkhCeeeIKjR4/yyCOPEBMTc/sDAkPUgO7eVu8IOEFSY0J5NvaC+mWBXm6hNxxuoXuXmIpgXILCIGe7ihPY9Bn6FODvx1/PLSVY9tGT+/D4B3iRY6XNNHb28/Byvcexm4pAcx5+WN3YTz/99MQOCI1VxT8apfkhJfdziBP2BZRb9coWeuP9WlbMjWF2rOkWmhBLPwk9TSpNUyPush+jTsbydke60aIM44/naogKCdBmSP1YuKQIhBBxQoh3hBDFjvfYUfbZIoQ4P+TVJ4R42PHZC0KI8iGfLXdFnunI3r17SUtLY+vWrRM7wOke0skqqDlLTE8lr9k28Op71UZLc4vSxi6u1nZwvxkknjjZ25Qf/sLvjJbkA/raiak5yKGA9ex6X58uqd39Vt6+VMd9S2cREqhPc8XRcNUi+DLwrpQyB3jX8fswpJQHpJTLpZTLga1ADzC0leHfOj+XUuo1CslArl+/zmuvvcaPfvQjvvCFL+A30VGBAcEQGK6yh3Th/G8hIITWjHv5w9lq7HY9Wmb/6cJNAO4z3UITJyBIZeVc263PYuPamwjbAP3zH+ZIcRPNXXpYw3uv1NE7aOORFXp1QB0NVxXBQ4Cz/HUnMJ6D7mPAW1JKvbpXachnP/tZHn/8ce69916++MUvTu7g0Fiw9sJgr2eEmwzWfrj0B5h/P/evyuVmex/Hy5qNlgq7XfLKmWrWZ8eTHB1itDi+xfLHwNYPl/9otCSKi69AzFxWFdyFzS7ZfWmM1tRe5tX3akiNCSUv7UOOEu1wVREkSSmdjWTqgPHyAh8FRtqU3xJCvC+E+I4QInisA4UQzwkhzgghzjQ23mZE3TTh4MGD9Pb28tJLLxEUFDS5g0NjAAE9xn/hUrRHZTEte4ztC5OIDAng92eNdw8dK22ipq2XT6yaY7Qovses5ZAwHy68aLQk0FoJpQdg2WPMnxXFvKQI/niuxmipaOjo41hJE4+sSMVPw95CIxlXEQgh9gkhLo3yemjoflL1Gh7T5hdCzAKWAHuGbP4KMB9YBcQBfzfW8VLK56WUeVLKvIQEffNxtcA/UJXZ97QYPtieCy9CRDJkbiYk0J8HlqXw1qVaOvuMzTp56XQVMWGBbDdnD0weIWDZo1B1AlrKjJXl3K/U+4o/QwjBx1bO5mxlK0X1xs7x3nXhJnYJD6/QO1vIybiKQEq5TUq5eJTX60C94wve+UV/myG1fAJ4TUp56xtASlkrFf3AL4DVrv1zTG4RHq9mAhtZU9DdDMV7YOnHVeMy4GMrZ9M3aB/eiM7LtHYPsPdyPY+sSNU+iKctSz4BCLjwknEy2Kyq/1HOXRCjLLuPrZxDkL8fvz15wzCxpJT84b0als6O1m4S2Vi46hraBTzl+Pkp4PXb7PsYI9xCQ5SIQMUXLrkoj4mToAjwDzLWPXTxFTVGc9ljtzatmBNDZkK4oe6h187VMGCz80nTLTR1olMhczNc+K0aPWoExXuhsxZWfvrWprjwIHYsSeYP71V/0JHUy7x3o5WrtR18PM937i9XFcG3gbuEEMXANsfvCCHyhBA/de4khEgH5gCHRhz/GyHEReAiEA9800V5TJwIodL8Brpg0ICBNXY7nHoeUvMg6YP+SE7z/VRFC2WNXbc5gWeQUvLS6SqWzYlhfnKU168/rVj5FLTdMG6e8dkXlNsxZ3g31Cfy0+jss/Kn928aItbOwkoiQwL4iI+4hcBFRSClbJZS3imlzHG4kFoc289IKZ8dsl+FlDJVSmkfcfxWKeUSh6vpSSml978ZpjNhFgwLGpfsg5ZSWPOXH/ro4w7z/YXCCq+LdaG6nev1nXzSh1Zr2jL/AYhKhRM/8v6126uh5B1Y8eQtt6OTVemxZCdG8BsD3EMNHX3svlj7/7d353FVlfkDxz9fLoiAW4q4AIpbLqFGkJraaGWJS2rlMtmvHLV9+U2zVFpNo7/Gxte0TE65NVnor3TGSSf7aW6ZZeYupkOKS66gqOAOyHLv9/fHuakgCJd74XK9z/v14hX3eM6530Nwv+d5zvM8X4bFRxMWXH2XlCjOzCy+ntmCrOL2uadAq/ih8cbpULuJtfZLMQ1rBzPo5qb8a0saZ3KqdoG8TzYcIiTIxr2dzdwBt9kC4dZH4cC3cHxn1b73tk+s3+lbHr7qn0SEh7o2Y/uRM6SkV22xpnmbjlDoUB6+rXmVvq+7TCLwISkpKYjIpRoF5RIaDo5COnRoz7Rp00rdbejQofTu3dvtGAE4udtagiBhbKkLgI3p0YLcAjvzNh3xzHuWw7GzuSz6IZ3hCVHUrll9FibzafG/spY/3zSz6t6zMM/qFmp1J9wQU+Iu98dFERwYwNxNVdcqKLA7+HTjIXrd2JAWPlbgyCSC611wbQ6kZ7IrdQ8Di5e4rCwbZ4ItGBJKXza7Q9M69GjdgNnrDlJgr5rWykdrD+BQePT2llXyfn4htD50Gm6NHso5VTXvuX2e9ZC4+3Ol7lI3NIhBnZvy7+T0KptpvCwlgxPn8xjV3bdaA2ASwfVPhCXfJRPbrjXNIupW/vupw/pD7TjMGsJ6DWN7tiDD2ada2c7mFDB342EGdmpCdH2zwJxHdX3SmsmePLvsfd1lL4S170LTOGh5xzV3faJXKy4W2vlw7YHKjwuYve4gzeqH0vvG6r3AXElMIqjGpk2bRnR0NGFhYdx7770cO3b5A3P48OElduVMmDCBRo0aUVBwecLWkpXfMKBPL7hwHFQ5cuQI/fv3JyQkhJiYGD788MMi5ygoKCAuLo7evXtjzRO0PPfcc4SHh5ORcY05AHkXrNVPuz5R5vX1vjGClg3DmLX2QJH3qQyfbDxEdr6dJ37RqlLfxy81ugla/AI2/b3yiyLt/BxOH4Dbf2eNjLuG1hG1GNCxCXPWHaz0Z1Hr9mWy5dBpRveI8YmZxMWZRFBNLVq0iGeeeYaBAweycOFCOnbsWGQp6rFjx7JmzRoOHLh8t3NlgfugIKsPPCcnh2+++YYB9w6Gghz04jkGDx5MSkoKs2bN4p133mHKlCmsX7/+0nmCgoKYM2cO69evZ8qUKQB8/fXXTJ06lalTp9K4cSkl9xyFkHfO6rtt0qnMawwIEEb3aMGOtLOs+6nyRjZdLLDz8fcH6HVjQzo0NUNGK0WP5+FcOmz9uPLew+GA796G8LbQtnzdnM/e2ZrsfDsfVWKrQFV5c8VumtStyYNdmlXa+1Qm3xnf5I6l4yDjP95578Ydod9klw+bNGkSiYmJTJ9uDc3r27cvJ0+evHT3fvfddxMVFUVSUhITJ04ErAL3Bw8eLFLSctWqVYSEhND9zkTI3M3Sz+ezbds2NmzYQNeuXQGIj4+nVatWtGnT5tJxHTt2ZMKECbz88sv07NmTMWPGMGzYMEaMGFF60BdOWF1DfSaU+zqHxUcxffU+/rx0F18807NS7qY+25pG5oV8nuxlWgOVptWdEHM7fPsXuHkkBNf2/HvsXQ4ndsJ9M6Gcq/G2a1yHxJsa8/G6g4y9vSV1Qzw/SODr1BNsO3yGN+7r6LMz1U2LoBoqLCwkOTmZwYOLDr28//77L30fEBDA6NGjmTNnzqVulaSkJBISEoiNjb2035IlS+jbty+2wCCoFcGmLVto1CjiUhIAaN68OfHx8VfF8eKLL9K5c2d69uzJxYsXrznqCHsBZJ+EGmHQpHO5r7VmkI3f921LSvo5vtju+QlAF/IK+duqvdzSrB7dWlavojjXFRHoM9EqWrN+qufP73BYSaZeM4h9wKVDn72zNecvFjK7EuatOBzKWyv20LxBKMMSqv9y06XxjxZBBe7IvSkzMxO73U5ERNGHTsVfjx49mtdff53Vq1dz6623smDBAt56660i+3z55Ze88cYb1ovQBmScPE1E/XqgWqSPNSIigvPniy7UZbPZGDZsGBs2bGDEiBE0aNCg9KDPZ1jnDHa962XIzZHMWnuAN5fvJjG2sUfvqt5btZcT5/P44JEEpIw+ZcNNUfFWTep171lDh2t5cHHI7XPhaDIMnupyTeLYyLr0aR/Bh9/tZ2TXZoTXKnWRY5ctTclg17Fz/HVEZ4Jsvntf7buRX8fCw8Ox2WycOFF0Db/ir2NiYujTpw9JSUnMnz8fh8PBgw9eXtdnx44dpKenk5iYaG0IsNE4uiUnMrOsu/drnBvg0KFDTJw4kbi4OGbMmEFKSilLQRVetGYvhzaoUOHwgADhlf7tST+T69HZxvtOXGDW2gMMT4ji5ugy6j0bnnHXa1YdjO/eKnvf8so5BSv+ANHdoPPICp1iXL925BbYeX2x5ya+5Rc6eHvlbtpE1GJQZ99ZTqIkJhFUQ4GBgcTFxbFoUdE1/BYuXHjVvmPHjmXBggVMmzaNIUOGFClwv2TJErp27Up4+OVhnLfe1pPjJ7PY+O0KsFsjKQ4fPkxycnKR86oqY8aMoWXLlqxfv54uXbrwyCOPFBmN5NwRzqZbrYvapTxELofurcO5o21Dpq7ex6ls90d4qCoT/+9HQmrYeDGxndvnM8opvI217MPmWZDhoTUkv5oAF8/CwHfK/WyguNYRtXm6d2sW/XCUb3Zfa5Hk8puyag/7T2Yzvn87bD44UuhKJhFUUy+//DLLli3jqaeeYsWKFbzyyissW7bsqv2GDBlCzZo1SU5OLvKQGKxEMKDYJLL+AwbQuVMnhj3+AvM+ms7ChQsZMGDAVd1OU6dOZe3atcyePZvg4GCSkpLYs2cPkyZNKhpATqY1Uqh2kwq1Bq40vn97LhbYefGzHW4PJ12x8zjf7c3kN31u9GhXgFEOd71mTTRbMNb9KnlHNlnzE7o9VWTxwop4+o5WtGoYxqufp5CTX+jWubYeOs30b35ieEIUd7a7DmpaqKrPfcXHx2tpdu7cWeq/+Zr33ntPIyMjNSQkRPv166fLly9XQFevXl1kv4ceekijo6PVbrdf2paVlaU2m023bdt21XkPHTqkfe/qrTVrBmuz6CidMWOGPvDAA9qrVy9VVd27d6+GhobqpEmTihz3/vvva2BgoG7dutXakJ+jmr5NNXOfqsOhqu7//Gd9t1+bv7RY/77mpwqfI+10jsa/vkLveedbzS+0l32A4Xl7v1L9Yx3Vxb+t+DnyslWndVd9q53qxfMeCWvj/ixt/tJi/dPiHyt8juy8Au31l6+1+59X6bncfI/EVVWALVrCZ6rXP9Qr8uUviaA8CgoKtGnTpvrqq68W2f7pp59qZGRk6Qc67KrHd6ke/cH6g3OV3a56fKfqsR2qhZf/GNz9+TscDn1s9mZtNX6JJh865fLx2XkF2u/dNRr72jLde/ycW7EYblr2spUMUr90/Vh7oeq8kap/rKu6e7lHwxq3YIfGjFusS3YcrdDxf/j8PxozbrGu/ynTo3FVhdISgeka8lH5+fls3ryZF154gaysLJ54ouhM3pEjR5KWdo3iLxIADVqC2Kxyg67MCFWFs0esh8T1mrvdJVQkLBHeHNqZxnVr8uzcbZx24XmBw6H8bv52UjPO8beRcbSOqISx7Eb53fWaNY/m86fhRKprxy5/BVIXQ+JkuPEej4b12sAOxDe7gef/8QNr92a6dOwHa35izvpDjOnRgm4trzGKzseYROCjjh49SpcuXZg7dy4zZ84kKqoCY5htNaBBK2sS2KmfrJnBZXE4rCn+uaesh8M1PT9Tt25oEO+PvIWT5/N4YPo6DmRml3mM3aFMXpbK0pQMxvdrzx1tfW+9l+tOYDAMTbJuFD7qC4c3lO+4DdOtZcy7PQ3dnvR4WCE1bMwadSstG4bx+P9uYfuRssu5qipTvtrLG1+mMrBTE8b1u74GILiVCERkmIj8KCIOEUm4xn6JIrJbRPaJyLgrtrcQkY3O7f8UkRruxONPYmJiUFWOHz/OqFGjyj6gNEEh1lK+hXnW8tEXz5W+r6MQTu2zRnDUibQeEFeSm6Pr8cmjXTmdk8+Qqd+zbl/pd27pZ3IZ+fcNfLDGGif+6O0tKi0uw0XhrWHsCmto8ZzBkLqk9H1zTsG/n4Rl46DdQLin8goW1g0NYs6YLjSoVYNHPtrEv7YcweEoeYBCgd3B5KWp/PWrPQyNj2LKL+N8es5ASUTdGJ0hIu0BBzAT+L2qbilhHxuwB7gbSAM2Aw+q6k4RmQ8sVNV/iMgMYLuqllnuKCEhQbdsueqtANi1axft27ev8DX5rbwLVtlBex6E1IewhhBYAwICrVKXOVlWK8BhhxuaQ8gNJZ7G0z//w1k5jJ29mf2Z2dwfF0m/jo3p0TqcABH2nbjAxv1ZvL1yDw6HMmHQTQyNjzITx6qj7EyYOxzSk6HVHRD3MLQbYHVRnj1itRZWvga5p611i3q9aLUoKtnhrBye/+c2kg+foXNUXcb3b0/7xnWoVTOQ7PxC5m08zMffHyTj3EUe6tqM1wfH+uSicj8Tka2qetVNu1uJ4IqTf0PpieA2YIKq9nW+Hu/8p8nASaCxqhYW3+9aykoE7dq1Mx8GFeFwwIUMa80gnL8XYgO1A2J1A9VqZC0jUQJVJTU11eOJ+PzFAiYt2cWSHcc4n1dIWA0bBQ4lv9CqYxDXrB7vjriZ5g18qxiI38nPhu//ZlUXO5cGQaHWc6afq+c1jYNB71nPFaqQqvL5D+lMXprK8XOXaxfYAgS7Q+neqgGP3d6S3m0b+vznSmmJoCqWmIgErixDlQZ0BRoAZ1S18IrtpU7PE5HHgccBmjUrfYW/oKAgcnNzCQ01a867LCAA6jS1mvEFuVbroDDfahmE1C/zoXBubu6lVU89qXbNICY/0In/GRzL9z9lsmrXcUJrBHJT0zrERtalRYMwn75L8xs1wuCO8dbd/v7VsHsZhNSzBhzUb2HNHLZV/ao3IsJ9cVHc06ExK3ceJys7n7O5BeQXOhjYqQmxkVVQx8PLyvypi8hXQElTRl9R1UUlbK8UqvoB8AFYLYLS9ouIiCA9PZ3IyEhCQkJ8PoN7RWCwS81yVSU3N5f09HQaNaq8yTU1AgO4o22EeRDs6wJs0LqP9VWNhAUHMiTOt5eKqKgyE4Gquvt/Kx2IvuJ1lHNbFlBPRAKdrYKft7ulTh1rFMvRo0evXg7BqDRBQUE0atTo0s/fMAzfURXtsM1AGxFpgfVB/0tgpKqqiKwGhgL/AEYBHmlh1KlTx3wgGYZhlJO7w0fvE5E04DZgiYgsd25vKiJfAjjv9p8FlgO7gPmq+qPzFC8BvxWRfVjPDGa5E49hGIbhOo+MGqpq1xo1ZBiGYZSstFFD19esCMMwDMNlJhEYhmH4OZMIDMMw/JxJBIZhGH7OJx8Wi8hJ4FAFDw8HXFt7tnrx9fjB96/B1+MH378GX48fvHMNzVW1YfGNPpkI3CEiW0p6au4rfD1+8P1r8PX4wfevwdfjh+p1DaZryDAMw8+ZRGAYhuHn/DERfODtANzk6/GD71+Dr8cPvn8Nvh4/VKNr8LtnBIZhGEZR/tgiMAzDMK5gEoFhGIaf86tEICKJIrJbRPaJyDhvx+MKEYkWkdUislNEfhSRX3s7pooQEZuIbBORxd6OpSJEpJ6IfCYiqSKyy1li1WeIyG+cvz8pIjJPRGp6O6ayiMhHInJCRFKu2FZfRFaKyF7nf0suol0NlBL/m87foR0i8m8RqefNGP0mEYiIDZgK9AM6AA+KSAfvRuWSQuB3qtoB6AY842Px/+zXWMuR+6opwDJVbQd0xoeuRUQigf8GElQ1FrBh1Qep7pKAxGLbxgGrVLUNsMr5urpK4ur4VwKxqtoJ2AOML35QVfKbRAB0Afap6n5VzccqhjPYyzGVm6oeU9Vk5/fnsT6AfKqunohEAQOAD70dS0WISF3gFzjrZqhqvqqe8W5ULgsEQkQkEAgFjno5njKp6hrgVLHNg4HZzu9nA0OqNCgXlBS/qq64ol77BqwKjV7jT4kgEjhyxes0fOyD9GciEgPEARu9G4nL3gVeBBzeDqSCWgAngY+d3VsfikiYt4MqL1VNB94CDgPHgLOqusK7UVVYI1U95vw+A6i8YtmVbwyw1JsB+FMiuC6ISC1gAfC8qp7zdjzlJSIDgROqutXbsbghELgFmK6qcUA21btLoghnP/pgrITWFAgTkf/yblTuU2sMvE+OgxeRV7C6fT/1Zhz+lAjSgegrXkc5t/kMEQnCSgKfqupCb8fjoh7AIBE5iNUtd6eIfOLdkFyWBqSp6s8tsc+wEoOv6AMcUNWTqloALAS6ezmmijouIk0AnP894eV4XCYivwIGAg+plyd0+VMi2Ay0EZEWIlID6yHZF16OqdxERLD6pnep6jvejsdVqjpeVaNUNQbrZ/+1qvrU3aiqZgBHRKStc9NdwE4vhuSqw0A3EQl1/j7dhQ897C7mC2CU8/tRwCIvxuIyEUnE6iYdpKo53o7HbxKB88HMs8ByrF/++ar6o3ejckkP4GGsO+kfnF/9vR2UH3oO+FREdgA3A294OZ5yc7ZkPgOSgf9g/f1Xm2UOSiMi84D1QFsRSRORscBk4G4R2YvV0pnszRivpZT43wdqAyudf8szvBqjWWLCMAzDv/lNi8AwDMMomUkEhmEYfs4kAsMwDD9nEoFhGIafM4nAMAzDz5lEYBiG4edMIjAMw/BzJhEYhmH4OZMIDMMNzkI1aSIyp9j2L0Rkj4iEeis2wygvkwgMww3OegRjgYdFZDCAiIzGqrswqjqsI2MYZTFLTBiGB4jITKziKInAamCmqr7k3agMo3xMIjAMD3DWidiBtc7/PiBeVfO8G5VhlI/pGjIMD1DVC8BiIBiYZZKA4UtMi8AwPEBEbgXWYS3v3By4yVm/wDCqPZMIDMNNIlITa43//cBwYDtWAaFBXg3MMMrJdA0Zhvv+BDQGHnOOEvoVMMBZitAwqj3TIjAMN4hID2AN8LCqzr1i+5vAY0CsqqZ5Kz7DKA+TCAzDMPyc6RoyDMPwcyYRGIZh+DmTCAzDMPycSQSGYRh+ziQCwzAMP2cSgWEYhp8zicAwDMPPmURgGIbh5/4fviNHlEtak4UAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" } } ], "metadata": { "id": "ls6qAuzLVTdT", "colab": { "base_uri": "https://localhost:8080/", "height": 284 }, "outputId": "13aec7ef-6bea-4955-dac6-7270da019cf7" } }, { "cell_type": "markdown", "source": [ "### Gradients of ReLU" ], "metadata": { "id": "dTDvLkRc7X-1" } }, { "cell_type": "code", "execution_count": null, "source": [ "x = tf.linspace(-1,1, 101)\n", "\n", "with tf.GradientTape() as tape1:\n", " tape1.watch(x)\n", " with tf.GradientTape() as tape2:\n", " tape2.watch(x)\n", " y = tf.nn.relu(x)\n", " dy_dx = tape2.gradient(y,x)\n", "d2y_dx2 = tape1.gradient(dy_dx,x) \n", "\n", "plt.plot(x, y, label='y')\n", "plt.plot(x, dy_dx, label='dy/dx')\n", "plt.plot(x, d2y_dx2, label='d2y/dx2')\n", "plt.legend(loc='upper left',fontsize=15);\n", "plt.xlabel('x',fontsize=15);" ], "outputs": [ { "output_type": "display_data", "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAELCAYAAADJF31HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgUZbbA4d9JWBJAkB0NgYCAgGBYQsAdN/Zd2bmjgBuCzFWHEWVmAB10dBx3QB0XYFgUhRE0UdyCXB0wQtgDhLATloR9hyT93T+6yXSaTtJJuru6O+d9njzprqquOqmunHw5VX1KjDEopZQKfmFWB6CUUso7NKErpVSI0ISulFIhQhO6UkqFCE3oSikVIspZteFatWqZmJgYqzavlFJBac2aNUeMMbXdzbMsocfExLB69WqrNq+UUkFJRPYUNE9LLkopFSI0oSulVIjQhK6UUiFCE7pSSoUITehKKRUiikzoIvKRiGSKyKYC5ouIvCUi6SKyQUTaeT9MpZRSRfHkssVZwDvAnALmdweaOr46AjMd30vt1KlTZGZmkp2d7Y3VKT8rX748derUoWrVqlaHolSZUGRCN8asEJGYQhbpC8wx9j68q0TkahG5xhhzsDSBnTp1isOHDxMVFUVkZCQiUprVKT8zxnD+/HkyMjIAgjOp51yCX9+Fi6etjkSFCGMMP+84QrPbBlG3xc1eX783PlgUBexzer7fMe2KhC4ijwCPADRo0KDQlWZmZhIVFUWlSpW8EKLyNxGhUqVKREVFceDAgeBM6AfWwnd/djzRAYXyjlsMrNlUP2ATuseMMe8D7wPExcUVemeN7OxsIiMj/RKX8p3IyMjgLZnlXrR/f+AraHSbtbGooLciLYsHPk6mf5so/nF/rE+24Y2rXDKAaKfn9R3TSk3LLMEvqN9DW679e5hlHTJUiDhw4jy//2QtzepcxV/7t/LZ74U3EvpS4HeOq106ASdLWz9XKiBoQldecCnHxtj5KWTnGmaMaEelCr47nopcs4gsADoDtURkPzAZKA9gjHkXSAR6AOnAOWCkr4JVyq9sOfbvYeHWxqGC2ouJW1i79wTTh7XjutpVfLotT65yGVrEfAOM9VpESgWKvISuI3RVMkvXH2DWf3Yz6pZG9LzxGp9vTz8pqlRBNKGrUkjPPM3ERRto37A6z/Zo7pdtakJXqiCa0FUJnb2Yw2NzU4gsH870Ye0oH+6fVKsJ3UcSExMJCwtj165d+abv2rWLsLAwlixZYlFkymN5J0W1hq48Z4zh2cUb2Zl1hreGtqVetQi/bVsTuo907dqVa6+9ltmzZ+ebPmvWLOrUqUPPnj0tikx5TEfoqgTmrNzD0vUHeLrL9dzSpJZftx1UR+rULzeTeuCUJdtueW1VJve+wePlw8PDefDBB5k9ezaTJ09GRDDGMHv2bEaMGEG5ckG168smTeiqmFL2HuevCanc3bwOY+64zu/b1xG6D40aNYo9e/awfPlyAJKSktizZw8jR+qVnUFBE7oqhmNnLzFuXgp1q0bw2qA2hIX5/0N1QXWkFmeEHAgaN25M586d+fjjj7nzzjv5+OOPiY+P54YbguvnKLP0g0XKQ7k2w+8/WcuRM5dYNOZmqlUqb0kcOkL3sYceeohFixaRkZHB4sWLdXQeTPSDRcpDb/2wnf/bfoQpfW6gdf1qlsWhCd3HBgwYQIUKFRgyZAg2m40hQ4ZYHZLylJZclAeWb8vkrR+3M6BdFEPjo4t+gQ9pQvexiIgIhg8fzs8//0z//v25+uqrrQ5JeUoTuipCxonzPPnpOq6vexXT+rW2vBmdJnQ/6NevH2A/SaqCiNbQVSEu5uTy+DxH063h7YisYH1pTo9UP/j2229p2LAhd911l9WhqOLQGroqxLSELazfd4KZw9vR2MdNtzylCd2Htm3bRmpqKjNnzmTy5MmEhek/REHFlgMSBsHc0135xJJ1GcxZuYeHbm1E99a+b7rlKU3oPvToo4/y66+/0qdPH8aPH291OKq4TK6WW9QVth8+zcRFG+kQU51nuvun6Zan9Gj1ocsfKFJBypajCV3lc+ZiDo/NXUPliuG848emW57So1Wpgth0hK7+yxjDxEUb2HXkLHNHd6RuVf813fJUYP15USqQ2HL0hKjKM/s/u/lqw0H+0PV6bvZz0y1PaUJXqiBaclEOKXuPMy1xC/e0qMNjt/u/6ZanNKErVRBN6Ao4euYiY+elUK9aBP8YaE3TLU/p0apUQbSGXubZm26t4+jZSyy2sOmWp3SErlRBtIZe5r35fRo/px/hhb430CrKuqZbntKE7mebNm1CREp0SWPLli2ZMWNGgfPvv/9+OnfuXPLgVH5acinTkrZl8taP6QxsX5/BHRpYHY5HNKEHiV27drFlyxZ69epldShlhyb0Mmv/8XM8+ek6WlxTlRf6tbI6HI9pQg8SCQkJtGrVigYNgmOkEBK0hl4mXW66lZtrmDm8HRHlg6fspgndx2bMmEF0dDSVK1emd+/eHDx4MG/eoEGD3JZIpkyZQt26dcnOzs6blpCQkO/G0vv27aNHjx5ERkYSExPDBx98kG8d2dnZtG3bls6dO2OMyZv+xBNPUKtWLQ4dOuTFnzJEaQ29THrhq1Q27D/Jq4NiialV2epwikUTug8tWbKEsWPH0qtXLxYvXkzr1q3ztdAdPXo0K1asYNeuXXnTnG8kXb68/Yz6uXPnWL58eV5CN8bQt29fNm3axIcffshrr73Gm2++ycqVK/PWU758eebMmcPKlSt58803Afjxxx+ZPn0606dPp169ev7YBcFNSy5lzhdrM5i7ai+P3N6YrjcE3+9IcB2tX0+EQxut2Xa91tD9b8V6ybRp0+jWrRszZ84EoGvXrmRlZeWNpu+9917q16/PrFmzmDp1KmC/kfTu3bvz3aruhx9+IDIykptvvhmAr7/+mrVr17Jq1So6duwIQPv27bnuuuto2rRp3utat27NlClTeO6557j11lsZNWoUAwcOZPDgwSXfD2WJJvQyJe3waZ5dvJH4mBpM6Hq91eGUiI7QfSQnJ4eUlBT69u2bb/qAAQPyHoeFhTFy5EjmzJmTVxaZNWsWcXFxtGr13xMxCQkJdO3alfBw+7//ycnJ1K1bNy+ZAzRs2JD27dtfEccf//hHYmNjufXWW7lw4UKhV8koF1pDLzNOX8jmsX+toXLFcrwzrG3ANd3yVHAdrcUcIVvpyJEj5ObmUqdOnXzTXZ+PHDmSF154gaSkJDp06MCiRYt49dVX8y2TmJjIiy++mPf80KFDV6zn8rpPnz6db1p4eDgDBw5k1apVDB48mJo1a5b2Rys7bDkQHtgfJFGlZ4zhmUUb2HPsHPMe6kidAGy65ang/DMUBGrVqkV4eDiZmZn5prs+j4mJ4Z577mHWrFksXLgQm83G0KFD8+Zv2LCBjIwMunXrljetXr16V6zH3boB9uzZw9SpU2nbti3vvvsumzZtKu2PVnbYckD0pGio++iX3SRuPMSErtfTqXFwD3g0oftIuXLlaNu2LUuWLMk3ffHixVcsO3r0aBYtWsSMGTPo169fvhtJJyQk0LFjR2rV+m93tw4dOnD48GF+/fXXvGl79+4lJSUl33qNMYwaNYrGjRuzcuVK4uPj+d3vfpfv6hlVCC25hLzVu4/xUuIW7m1Zl0dvb2x1OKXmUUIXkW4isk1E0kVkopv5DUQkSUTWisgGEenh/VCDz3PPPcc333zDmDFj+Pbbb5k0aRLffPPNFcv169ePiIgIUlJS8p0MhSsvVwTo0aMHsbGxDBw4kAULFrB48WJ69ux5RRlm+vTp/Pzzz8yePZuKFSsya9Ys0tLSmDZtmvd/2FCkJ0VD2pEzFxk7P4Wo6pG8OjAWCYVbDRpjCv0CwoEdQGOgArAeaOmyzPvAGMfjlsDuotbbvn17U5jU1NRC5weLt99+20RFRZnIyEjTvXt3s2zZMgOYpKSkfMsNHz7cREdHm9zc3LxpR48eNeHh4Wbt2rVXrHfPnj2ma9euJiIiwjRo0MC8++675r777jN33HGHMcaY7du3m0qVKplp06ble90777xjypUrZ9asWeP1n7UgQfteTr/JmAXDrI5C+UBOrs0M++dK02xSotmUccLqcIoFWG0KyKueDD/igXRjzE4AEfkE6AukOv9dAKo6HlcDDpTqr0wIGTduHOPGjcs3zTh90AfsV8QkJSUxatSofDeS/uabb6hXrx5t2rS5Yr0NGjS4YrT/6KOP5j1u0qQJZ8+eveJ1Y8eOZezYsSX6WcocHaGHrNe/S+OX9KO8ct+N3HBt4Dfd8pQnJZcoYJ/T8/2Oac6mACNEZD+QCDzhbkUi8oiIrBaR1VlZWSUIN7RcunSJ3377jQkTJnD06NF8CRlg2LBh7N+/36LolCb00PTj1sO8k5TOoLj6DOoQbXU4XuWtk6JDgVnGmPpAD+BfInLFuo0x7xtj4owxcbVr1/bSpoPXgQMHiI+PZ/78+bz33nvUr1/f6pCUM03oIWffsXM8+el6Wl5Tlef7Bk/TLU95crRmAM5/xuo7pjkbDXQDMMasFJEIoBZw5XV0Kk9MTMwV5RcVQPQql5ByITuXMfPWYDOGd0e0D6qmW57yZIT+G9BURBqJSAVgCLDUZZm9wN0AItICiAC0pqKCmzbnCilTv0xlU8YpXhvUhgY1K1kdjk8UmdCNMTnAOGAZsAVYaIzZLCLPi0gfx2JPAw+LyHpgAfCg0aGnCnZacgkZi9bsZ0HyXh674zrubVnX6nB8xqOj1RiTiP1kp/O0vzg9TgVu8W5oSllME3pI2HroFJO+2EinxjX4Q5dmVofjU/pJUaUKojX0oHf6QjZj5qZQNaI8bw1tS7kgbbrlKT1alSqI1tCDmjGGCZ9tYO+xcyx4uBN1rgreplueCu0/V0qVhpZcgtqHP+/im82HeKbb9cQ3qmF1OH6hCV2pgmhCD1q/7T7GS19vpesNdXn4tuBvuuUpTeh+tmnTJkSE5cuXc+rUKSZPnkx8fDzVqlWjXr169O/fn7S0tBKvPzk5mQoVKnDy5Em388+cOYOIMGvWLI/X6Ys4A57NBhhN6EEo6/RFxs1PIbp6JH8PlaZbHtKEbqG9e/fyz3/+k65du/L555/z3nvvcfDgQTp27Mi+ffuKXoEbCQkJ3HrrrVSr5r3+FL6IM+DZcuzfw/RXJJjk5NoYv2AtJ89nM3NEe6pGlK0blOjww0KNGjVix44dREZG5k277bbbaNCgAR999BGTJ08u9joTEhLy3SDDG3wRZ8AzufbvOkIPKq99l8bKnUd5dWAsLa6pWvQLQowOP3xsxowZREdHU7lyZXr37s3Bgwfz5lWuXDlfkgSoUaMGDRs25MABe8PKY8eOERERcUWJxBhD48aNefLJJ/OmHTp0iJSUlHz90xctWkSzZs2IjIzk9ttvZ+vWrfnWs2rVKsqVK8dHH32UN+3kyZNER0czfPhwj+MMOXkjdE3oweL71MPMWL6DofHR3N++bPZF0oTuQ0uWLGHs2LH06tWLxYsX07p1a0aNGlXoa7KyskhPT6dZM/sHIGrUqEH//v2vSOjLly9n165d+daXmJhIo0aNaN68OQApKSkMHjyY2NhYFi9eTO/evRk0aFC+9XTq1IkJEybw5JNPsnfvXgDGjx+PzWbjnXfe8TjOkKMJPajsPXqOpxauo1VUVSb3vsHqcCwTVEfry8kvs/XY1qIX9IHmNZrzTPwzxXrNtGnT6NatGzNnzgSga9euZGVl8cEHHxT4mqeffpoqVarw4IMP5k0bPXo0Xbp0YefOnTRubD9j//HHH9O+fXtat26dt5zr3Y3+9re/0axZMxYuXIiI0L17dy5dusSf/vSnfNucOnUqCQkJjBo1iieeeII5c+aQmJhI9erVixVnSLFpySVYXMjO5fH5awCYOTw0m255SkfoPpKTk0NKSgp9+/bNN33AgAEFvmbmzJnMnTuXDz74gJo1/3uz2rvvvpuGDRsye/ZsAE6fPs2iRYvy3a4uOzub7777Ll9CT05Opk+fPvnO8rvbfoUKFZgzZw4rVqxg8ODBPPTQQ3Tv3r3YcYaUvBF62U0OwWLql5vzmm5F1wjNplueCqrhR3FHyFY6cuQIubm5V9zn0/X5ZUuXLuWJJ57g5Zdfpn///vnmiQgjR47ko48+YsqUKSxcuJDc3FyGDRuWt8yKFSuw2Wx07tw5b9qhQ4c83n5sbCwtW7Zk/fr1PP744wX+XIXFGVK05BIUPl+znwXJ+3i883XcE8JNtzylI3QfqVWrFuHh4WRm5m8J7/oc4JdffmHIkCE89thjTJgwwe36Ro4cyb59+0hKSmLWrFn069cvX0kkISGBu+++m4oVK+ZNq1evnkfbB3jjjTfYunUrLVq0yKuhlyTOkKEJPeClHjjFn77YyE2Na/LUvSF6LqeYNKH7SLly5Wjbti1LlizJN33x4sX5nm/evJnevXvTrVs33nrrrQLXFx0dTZcuXZg8eTI///xzvnILXFk/B+jQoQNLly7NdxMN1+0DbNu2jUmTJvHXv/6VhQsXkpyczOuvv16iOEOG1tAD2qkL2Tw+b02ZabrlKT1afei5555jwIABjBkzhv79+/PTTz/lu7FzZmYm3bp1o0qVKowfP57k5OS8eVWrVqVly5b51jd69GgGDhxI/fr1uffee/Omp6enk5aWRo8ePfIt/8wzz9CxY0cGDRrE6NGj2bRpEx9++GG+ZXJzc3nggQdo27YtTz31FGFhYUydOpU//elP9OzZk+bNmxc7zpCgNfSAZYzhDwvXs+/4eT55pBO1r6pY9IvKCmOMJV/t27c3hUlNTS10frB4++23TVRUlImMjDTdu3c3y5YtM4BJSkoySUlJBnD7dccdd1yxrvPnz5ty5cqZSZMm5Zv+xhtvmNjYWLfbX7hwobnuuutMxYoVzS233GKSk5MNYD7++GNjjDEvvviiqVSpkklLS8t7TU5OjunUqZOJj483OTk5xY7TVVC+l4c2GTO5qjGbv7A6EuXivZ/STcNnvjL/XLHD6lAsAaw2BeRVMRbdWCguLs6sXr26wPlbtmyhRYsWfowo8CUmJtKrVy/S0tJo0qRJ3vQuXbrQoUMHpk2bZmF0BQvK9/LgenjvdhgyH5r3LHp55RfJu44x9J+r6NKyLjOGtytTfVouE5E1xpg4d/O05BIEDhw4wPbt25k4cSI9evTIl8wBvv32W4siC2F6UjTgZJ6+wNj5KTSoUYlX7r+xTCbzouiZhCDw/vvvc/fddxMREcHbb79tdThlQ95JUa2hB4KcXBtPzF/L6QvZzBzRjqvKWNMtT2lCDwJTpkwhJyeH5ORkGjVqZHU4ZYOO0APKq9+m8euuY7zYvzXN65W9plue0oSulDua0APGd6mHefenHQzr2IAB7cpm0y1PBXRCt+qErfKeoH0PNaEHhD1Hz/LUwnW0jqrGX3qF4OWxXhawCb18+fKcP3/e6jBUKZ0/f57y5YOw3nm5hi5aQ7fKhexcxsxNIUyEGcPblemmW54K2IRep04dMjIyOHfuXPCO8sowYwznzp0jIyOjwP4xAU1Pilpu8pLNpB48xeuDY8t80y1PBez/k1Wr2k98HDhwgOzsbIujUSVRvnx56tatm/deBhUtuVhq4ep9fLp6H+PubMJdzbXplqcC+mitWrVqcCYDFfw0oVtm84GT/PmLTdzSpCZPatOtYgnYkotSltKEbomT57MZMzeF6pUq8OaQtoSH6YeHikOPVqXc0Rq63xlj+MNn6zlw4jyfPnoTtapo063i0hG6Uu7oCN3v3luxk+9SD/Ncjxa0b1jw7Q9VwTShK+WOJnS/WrXzKK98s5Wera9h5C0xVocTtDxK6CLSTUS2iUi6iEwsYJlBIpIqIptFZL53w1TKzzSh+03mqQuMm7+WmFqVeVmbbpVKkUeriIQD04F7gf3AbyKy1BiT6rRMU+BZ4BZjzHERCcILj5Vyoncs8oucXBvjFqzl7MUc5j/ckSoVdX+Xhicj9Hgg3Riz0xhzCfgE6OuyzMPAdGPMcQBjjPsbVyoVLPSORX7x92XbSN51jJcGtKZZ3ausDifoeZLQo4B9Ts/3O6Y5awY0E5FfRGSViHRztyIReUREVovI6qysrJJFrJQ/aMnF55ZtPsR7K3YyolMD+rV1TSmqJLx1UrQc0BToDAwF/ikiV7suZIx53xgTZ4yJq127tpc2rZQPaEL3qd1HzvKHheuJrV+NP2vTLa/xJKFnANFOz+s7pjnbDyw1xmQbY3YBadgTvFLBSWvoPnP+Ui6PzV1DeLgwfXg7KpbTspa3eJLQfwOaikgjEakADAGWuizzBfbROSJSC3sJZqcX41TKv7SG7hPGGP68ZBPbDp/m9cFtqF9dm255U5EJ3RiTA4wDlgFbgIXGmM0i8ryI9HEstgw4KiKpQBIwwRhz1FdBK+Vzthx761y9hM6rPv1tH5+v2c8Tdzbhzuv1Yjhv8+j/SWNMIpDoMu0vTo8N8JTjS6ngZ8vRcouXbco4yV+WbubWJrX4/T3adMsX9JOiSrljy9FyixedPJfNmHlrqFm5Am8OaaNNt3xEhyBKuWNsOkL3EpvN8PRn6zh08gKfPnoTNbXpls/oCF0pd3SE7jXvrtjB91symdSjBe0aaNMtX9KErpQ7WkP3iv/sOMKry7bR68ZreODmGKvDCXma0JVyRxN6qR0+dYHxC9bSqFZlXr5Pm275gx6xSrljy9WEXgrZuTbGzU/h3KVcFjzcicradMsvdC8r5Y7W0Evl5a+38tvu47w5pA1NtemW32jJRSl3tORSYl9vPMgHP+/igZsa0reNNt3yJ03oSrmjCb1Edh05y4TPNxAbfTXP9WxhdThljiZ0pdzRGnqxnb+Uy5i5aygfLszQpluW0CNWKXe0hl4sxhgmfbGRbYdPM2tkPFFXR1odUpmkI3Sl3NGSS7EsSN7H4pQMxt/VlDua6b0OrKIJXSl3NKF7bOP+k0xZupnbm9Vm/N16GwQraUJXyh2toXvkxLlLjJm3hlpVKvDGYG26ZTVN6Eq5ozX0ItlshqcWrufwqQvMGNGeGpUrWB1SmacJXSl3tORSpJk/7eDHrZn8uVdL2kRfcQthZQFN6Eq5owm9UL+kH+Ef326jT+y1/E+nhlaHoxw0oSvlzuVb0KkrHDppb7rVuHYVXhrQWptuBRAdgijljs2mNXQ3snNtjJ2fwvnsXD4d0U6bbgUYfTeUckdLLm69lLiVNXuO8/bQtjSpo023Ao2WXJRyRxP6FRI2HOSjX3bx4M0x9I691upwlBua0JVyRxN6PjuyzvDHz9fTtsHVPNdDm24FKk3oSrmjHyzKc+5SDmPmrqFCuTCmD2tHhXKaNgKVHrFKuaMfLAIcTbf+vYntmWeYMyqea7XpVkDTP7VKuaMlFwDm/bqXf6/N4Ml7mnFbU226Feg0oSvljiZ0Nuw/wfNfptL5+tqMu7OJ1eEoD2hCV8qdMl5DP372EmPmplD7qoq8PqgNYdp0KyiU3SNWqcKU4Rq6zWZ4cuE6sk5f5LPHbqK6Nt0KGjpCV8qdMlxymZ6UzvJtWfy5d0titelWUNGErpQ7ZTSh/7z9CK99n0a/NtcyomMDq8NRxaQJXSlXNhtgylxCP3jyPOM/WUvTOlV4UZtuBSWPErqIdBORbSKSLiITC1nuPhExIhLnvRCV8jNbjv17GaqhX8qxMXZeChezc5k5oj2VKpStP2ahosiELiLhwHSgO9ASGCoiLd0sdxXwe+BXbweplF/lJfSyk9Re+noLKXtP8Mr9sVxXu4rV4agS8mSEHg+kG2N2GmMuAZ8Afd0s9wLwMnDBi/Ep5X9lbIT+1YYDfPzLbkbeEkPPG6+xOhxVCp4k9Chgn9Pz/Y5peUSkHRBtjEkobEUi8oiIrBaR1VlZWcUOVim/MLn272VghJ6eeYZnPt9AuwZX82x3bboV7Ep9UlREwoDXgKeLWtYY874xJs4YE1e7tn6MWAUoW9lI6Gcv2ptuRZQPZ8bw9tp0KwR48g5mANFOz+s7pl12FdAKWC4iu4FOwFI9MaqCVhkouRhjeO7fG9mRdYa3hralXrUIq0NSXuBJQv8NaCoijUSkAjAEWHp5pjHmpDGmljEmxhgTA6wC+hhjVvskYqV8rQycFJ27ag9L1h3gqXubcUuTWlaHo7ykyIRujMkBxgHLgC3AQmPMZhF5XkT6+DpApfwuxBP6un0neP6rVO68vjaPd9amW6HEoyPWGJMIJLpM+0sBy3YufVhKWSiEa+jHz15i7LwU6laN4PXB2nQr1ITeEatUaYVoDd1mM/zvp/amW5+PuYmrK2nTrVCjp7WVchWiJZe3f0znp7QsJvdpyY31telWKNKErpSrEEzoK9KyeOOHNAa0jWJYvDbdClWa0JVyFWIJPePEeX7/yVqa1bmKaf216VYo04SulKu8k6LBX0O/3HQrO9cwc0Q7IisE/8+kChYaQxClvCmERujTElJZt+8EM4a3o7E23Qp5OkJXylWIJPSl6w8we+UeRt/aiB6ttelWWaAJXSlXIZDQtx8+zcRFG4hrWJ2J3ZtbHY7yE03oSrkK8g8Wnb2Yw5h5KVSqEM704e0oH66/5mVFcB6xSvlSEH+wyBjDxMUb2Zl1hrkPdaRuVW26VZbon26lXF1O6BJ8CX32f3bz5foDPN3lem6+TptulTWa0JVyFaQll5S9x5mWuIV7WtRhzB3XWR2OsoAmdKVcBeFJ0aNnLjJ2Xgr1qkXwj4HadKusCp4jVil/CbIPFuU6mm4dPXuJxWNuplql8laHpCyiI3SlXAXZCP3NH7bzf9uP8HyfG2gVVc3qcJSFNKEr5SqIEvrybZm8/eN27m9fn8Edoot+gQppmtCVchUkCX3/8XP876fruL7uVbzQt5U23VKa0JW6QhBc5XIxJ5ex89eSm2uYOaK9Nt1SgJ4UVepKQfDBor9+tYX1+07w7oh2NKpV2epwVIDQEbpSrgK85LJkXQb/WrWHh29rRLdW2nRL/ZcmdKVcBXBCTzt8momLNtIhpjp/7KZNt1R+mtCVchWgNfQzF3MYM3cNlSuG884wbUNptI8AAA6uSURBVLqlrhRYR6xSgSAAa+jGGJ5ZtIFdR84y76FO2nRLuaV/4pVyZcuxN+YKoMsAZ/1nNwkbDvKHrtdz03U1rQ5HBShN6Eq5suUEVLllzZ7jTEvYwj0t6vLY7dp0SxVME7pSrgIooV9uunXN1RH8Y1CsNt1ShQqMo1apQGLLDYiEnmszjP9kLcfOOZpuRWrTLVU4HaEr5cqWA2HW/2q88X0av6Qf5YW+2nRLecb6o1apQGOsH6Enbc3k7R/TGdi+PoM7NLA0FhU8NKEr5criGvq+Y/amWy2uqcoL/VpZFocKPh4ldBHpJiLbRCRdRCa6mf+UiKSKyAYR+UFEGno/VKX8xMIaur3pVgo2m2Hm8HZElA+ca+FV4CsyoYtIODAd6A60BIaKSEuXxdYCccaYG4HPgVe8HahSfmPLsexDRc9/mcqG/Sd5dVAsMdp0SxWTJyP0eCDdGLPTGHMJ+ATo67yAMSbJGHPO8XQVUN+7YSrlRxaVXP69dj/zft3Lo3c0pusN9fy+fRX8PEnoUcA+p+f7HdMKMhr42t0MEXlERFaLyOqsrCzPo1TKnyxI6NsOnebZxRvp2KgGE7pc79dtq9Dh1ZOiIjICiAP+7m6+MeZ9Y0ycMSaudu3a3ty0Ut7j5xr66QvZjJm7hqsiyvP2sLaU06ZbqoQ8OWozAOebFdZ3TMtHRO4BJgF3GGMueic8pSzgxxr65aZbe46dY/5DHalzlTbdUiXnyVDgN6CpiDQSkQrAEGCp8wIi0hZ4D+hjjMn0fphK+ZEfSy4f/bKbxI2H+GPX6+nYWJtuqdIpMqEbY3KAccAyYAuw0BizWUSeF5E+jsX+DlQBPhORdSKytIDVKRX4/JTQV+8+xkuJW+jSsi6P3N7Y59tToc+jo9YYkwgkukz7i9Pje7wcl1LW8UMN/ciZi4ydn0JU9Uj+PjAWCaBWvSp4Wd+BSKlAY8uBcN81wsq1GcYvWMuJc9n8+/F4bbqlvEZPpyvlyscll9e+28Z/dhzlhX6taHltVZ9tR5U9mtCVcuXDhP7DlsNMT9rB4LhoBsVFF/0CpYpBE7pSrnyU0PcdO8eTn67jhmurMrXvDV5fv1Ka0JVyZcv1+nXoF7JzGTNvDQAzh7fXplvKJ/SkqFKuLt8k2oumfpnKpoxTfPC7OBrUrOTVdSt1mY7QlXLl5csWF63Zz4LkvYzpfB33tKzrtfUq5UoTulKuvFhD33LwFJO+2MhNjWvy9L3NvLJOpQqiCV0pV14aoZ9yNN2qGlGet4Zq0y3le1pDV8qVF5pzGWP442cb2Hf8PAse7kTtqyp6KTilCqZDBqVceaHk8sH/7eKbzYeY2K058Y1qeCkwpQqnCV0pV6VM6Mm7jvG3b7bS7YZ6PHRbIy8GplThNKEr5aoUNfTM0xcYNz+F6OqRvDLwRm26pfxKa+hKuSphDT0n18b4BWs5dSGb2aPiqRqhTbeUf2lCV8pVCUsu//gujVU7j/HqwFhaXKNNt5T/aclFKVclSOjfpR5m5vIdDI1vwP3t6/soMKUKpwldKWc2G2CKldD3Hj3HUwvX0SqqKpN7t/RdbEoVQRO6Us5sOfbvHtbQLzfdChPRplvKclpDV8pZXkL37Fdj8pLNbD5wig8fiCO6hjbdUtbSEbpSzoqR0Beu3senq/cx7s4m3N1Cm24p62lCV8qZhwl984GT/PmLTdzSpCZPatMtFSA0oSvlzJZr/15IDf3k+Wwen5dC9UoVeHNIW8LD9MNDKjBoDV0pZ6bwhG6MYcJn68k4fp5PHulErSradEsFDh2hK+WsiJLL+yt28m3qYZ7t0YK4GG26pQKLJnSlnBWS0H/deZRXlm2jR+t6jLolxr9xKeUBTehKOcuroedP6JmnLzBuwVoa1qjEy/dp0y0VmLSGrpQzNx8sysm1MW7+Wk5fyOZfo+O5SptuqQClCV0pZ25KLn//dhvJu47x2qBYmtfTplsqcGnJRSlnLgn9282HeO+nnQzr2IAB7bTplgpsmtCVcuaU0PccPcvTn62ndVQ1/tJLm26pwKcJXSlnjpOil2zCY3NTCBNhxvB22nRLBQWPErqIdBORbSKSLiIT3cyvKCKfOub/KiIx3g5UKb9wjNBnrdrPloOneH1wrDbdUkGjyIQuIuHAdKA70BIYKiKu/3+OBo4bY5oArwMveztQpfzCkdB/2HaMJ+5qwl3NtemWCh6eXOUSD6QbY3YCiMgnQF8g1WmZvsAUx+PPgXdERIwxxouxAvDk/CEcOr3J26tVCoAwbFSoV4esCt+z0baZkd9YHZEKRc1rNOeZ+Ge8vl5PEnoUsM/p+X6gY0HLGGNyROQkUBM44ryQiDwCPALQoEGDEgUcVj6CnLCIEr1WKU9cDCvHtXVqoR8dUsHGr9ehG2PeB94HiIuLK9Ho/R8DZ3kzJKWUChmenBTNAKKdntd3THO7jIiUA6oBR70RoFJKKc94ktB/A5qKSCMRqQAMAZa6LLMUeMDx+H7gR1/Uz5VSShWsyJKLoyY+DlgGhAMfGWM2i8jzwGpjzFLgQ+BfIpIOHMOe9JVSSvmRRzV0Y0wikOgy7S9Ojy8AA70bmlJKqeLQT4oqpVSI0ISulFIhQhO6UkqFCE3oSikVIsSqqwtFJAvYU8KX18LlU6gBQuMqHo2r+AI1No2reEoTV0NjTG13MyxL6KUhIquNMXFWx+FK4yoejav4AjU2jat4fBWXllyUUipEaEJXSqkQEawJ/X2rAyiAxlU8GlfxBWpsGlfx+CSuoKyhK6WUulKwjtCVUkq50ISulFIhImATuogMFJHNImITkQIv7ynoBtaOdr+/OqZ/6mj96424aojIdyKy3fG9uptl7hSRdU5fF0Skn2PeLBHZ5TSvjb/iciyX67TtpU7TrdxfbURkpeP93iAig53meXV/leaG5yLyrGP6NhHpWpo4ShDXUyKS6tg/P4hIQ6d5bt9TP8X1oIhkOW3/Iad5Dzje9+0i8oDra30c1+tOMaWJyAmneb7cXx+JSKaIuL1Ppti95Yh7g4i0c5pX+v1ljAnIL6AFcD2wHIgrYJlwYAfQGKgArAdaOuYtBIY4Hr8LjPFSXK8AEx2PJwIvF7F8DewthSs5ns8C7vfB/vIoLuBMAdMt219AM6Cp4/G1wEHgam/vr8KOF6dlHgfedTweAnzqeNzSsXxFoJFjPeF+jOtOp2NozOW4CntP/RTXg8A7bl5bA9jp+F7d8bi6v+JyWf4J7G2/fbq/HOu+HWgHbCpgfg/ga0CATsCv3txfATtCN8ZsMcZsK2KxvBtYG2MuAZ8AfUVEgLuw37AaYDbQz0uh9XWsz9P13g98bYw556XtF6S4ceWxen8ZY9KMMdsdjw8AmYDbT8KVktvjpZB4PwfuduyfvsAnxpiLxphdQLpjfX6JyxiT5HQMrcJ+5zBf82R/FaQr8J0x5pgx5jjwHdDNoriGAgu8tO1CGWNWYB/AFaQvMMfYrQKuFpFr8NL+CtiE7iF3N7COwn6D6hPGmByX6d5Q1xhz0PH4EFC3iOWHcOXBNM3x79brIlLRz3FFiMhqEVl1uQxEAO0vEYnHPura4TTZW/uroOPF7TKO/XH5hueevNaXcTkbjX2Ud5m799Sfcd3neH8+F5HLt6sMiP3lKE01An50muyr/eWJgmL3yv7y602iXYnI90A9N7MmGWOW+DueywqLy/mJMcaISIHXfTr+8rbGfreny57FntgqYL8W9RngeT/G1dAYkyEijYEfRWQj9qRVYl7eX/8CHjDG2ByTS7y/QpGIjADigDucJl/xnhpjdrhfg9d9CSwwxlwUkUex/3dzl5+27YkhwOfGmFynaVbuL5+yNKEbY+4p5SoKuoH1Uez/ypRzjLLc3di6RHGJyGERucYYc9CRgDILWdUg4N/GmGyndV8erV4UkY+BP/gzLmNMhuP7ThFZDrQFFmHx/hKRqkAC9j/mq5zWXeL95UZxbni+X/Lf8NyT1/oyLkTkHux/JO8wxly8PL2A99QbCarIuIwxzjeD/wD7OZPLr+3s8trlXojJo7icDAHGOk/w4f7yREGxe2V/BXvJxe0NrI39LEMS9vo12G9g7a0Rv/MNsYta7xW1O0dSu1y37ge4PRvui7hEpPrlkoWI1AJuAVKt3l+O9+7f2GuLn7vM8+b+Ks0Nz5cCQ8R+FUwjoCmQXIpYihWXiLQF3gP6GGMynaa7fU/9GNc1Tk/7AFscj5cBXRzxVQe6kP8/VZ/G5YitOfYTjCudpvlyf3liKfA7x9UunYCTjkGLd/aXr872lvYL6I+9jnQROAwsc0y/Fkh0Wq4HkIb9L+wkp+mNsf/CpQOfARW9FFdN4AdgO/A9UMMxPQ74wGm5GOx/dcNcXv8jsBF7YpoLVPFXXMDNjm2vd3wfHQj7CxgBZAPrnL7a+GJ/uTtesJdw+jgeRzh+/nTH/mjs9NpJjtdtA7p7+XgvKq7vHb8Hl/fP0qLeUz/F9RKw2bH9JKC502tHOfZjOjDSn3E5nk8B/ubyOl/vrwXYr9LKxp6/RgOPAY855gsw3RH3Rpyu4PPG/tKP/iulVIgI9pKLUkopB03oSikVIjShK6VUiNCErpRSIUITulJKhQhN6EopFSI0oSulVIjQhK6UUiFCE7oq80TkahHZLyJzXKYvFfvNESpZFZtSxaEJXZV5xpgT2D+i/T8i0hdAREYCPbF3fvR1L3ulvEI/+q+Ug4i8h70BWDfsfUneM8Y8Y21USnlOE7pSDiJSBdiAvQFcOtDeOLWpVSrQaclFKQdjzBngK+z3Df1Qk7kKNjpCV8pBRDoA/8He1rQhcIMx5pC1USnlOU3oSgEiEgGkYL/b+iDs/bK3GGP6WBqYUsWgJRel7P6K/b6oDzuuankQ6CkiD1oZlFLFoSN0VeaJyC3ACuB/jDHznab/HXgYaGWM2W9VfEp5ShO6UkqFCC25KKVUiNCErpRSIUITulJKhQhN6EopFSI0oSulVIjQhK6UUiFCE7pSSoUITehKKRUi/h/MeWELWyFXVwAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" } } ], "metadata": { "id": "xPAwSAmn61mO", "colab": { "base_uri": "https://localhost:8080/", "height": 284 }, "outputId": "44e4a45f-2d05-4cb1-ed7e-0928c1cdc25c" } }, { "cell_type": "code", "execution_count": null, "source": [], "outputs": [], "metadata": { "id": "sRcZXEj56-IY" } } ] }