From 2406277fcd320e3543f90bc3d62099f56cc98eae Mon Sep 17 00:00:00 2001
From: Maciej Wielgosz <maciej.wielgosz@nibio.no>
Date: Mon, 20 Feb 2023 15:19:27 +0100
Subject: [PATCH] progress towards own model (cifar experiments)

---
 cifar_example/vis.ipynb | 219 ++++++++++++++++++++--------------------
 1 file changed, 111 insertions(+), 108 deletions(-)

diff --git a/cifar_example/vis.ipynb b/cifar_example/vis.ipynb
index af4392f..a3cd485 100644
--- a/cifar_example/vis.ipynb
+++ b/cifar_example/vis.ipynb
@@ -240,141 +240,144 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "patches shape inside :  torch.Size([1, 3, 2, 2, 16, 16])\n",
-      "pos_embedding inside :  torch.Size([4, 4])\n"
+      "tensor([[ 0.0000e+00,  1.0000e+00,  0.0000e+00,  ...,  1.0000e+00,\n",
+      "          0.0000e+00,  1.0000e+00],\n",
+      "        [ 8.4147e-01,  5.4030e-01,  6.8156e-01,  ...,  1.0000e+00,\n",
+      "          1.3335e-04,  1.0000e+00],\n",
+      "        [ 9.0930e-01, -4.1615e-01,  9.9748e-01,  ...,  1.0000e+00,\n",
+      "          2.6670e-04,  1.0000e+00],\n",
+      "        ...,\n",
+      "        [ 3.7961e-01, -9.2515e-01, -4.6453e-01,  ...,  9.9985e-01,\n",
+      "          1.2935e-02,  9.9992e-01],\n",
+      "        [-5.7338e-01, -8.1929e-01, -9.4349e-01,  ...,  9.9985e-01,\n",
+      "          1.3068e-02,  9.9991e-01],\n",
+      "        [-9.9921e-01,  3.9821e-02, -9.1628e-01,  ...,  9.9985e-01,\n",
+      "          1.3201e-02,  9.9991e-01]])\n"
      ]
     },
     {
-     "ename": "TypeError",
-     "evalue": "reshape(): argument 'shape' must be tuple of SymInts, but found element of type float at pos 2",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[6], line 68\u001b[0m\n\u001b[1;32m     65\u001b[0m patch_size \u001b[39m=\u001b[39m \u001b[39m16\u001b[39m\n\u001b[1;32m     66\u001b[0m pos_embedding \u001b[39m=\u001b[39m PositionEmbedding(patch_size\u001b[39m=\u001b[39mpatch_size, in_channels\u001b[39m=\u001b[39m\u001b[39m3\u001b[39m, out_channels\u001b[39m=\u001b[39m\u001b[39m8\u001b[39m)\n\u001b[0;32m---> 68\u001b[0m pos_embedding(img\u001b[39m.\u001b[39;49munsqueeze(\u001b[39m0\u001b[39;49m))\n\u001b[1;32m     70\u001b[0m \u001b[39m# show the embedding and patches\u001b[39;00m\n\u001b[1;32m     71\u001b[0m embedding, patches \u001b[39m=\u001b[39m pos_embedding(img\u001b[39m.\u001b[39munsqueeze(\u001b[39m0\u001b[39m))\n",
-      "File \u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
-      "Cell \u001b[0;32mIn[6], line 58\u001b[0m, in \u001b[0;36mPositionEmbedding.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     56\u001b[0m embedding \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnorm(embedding)\n\u001b[1;32m     57\u001b[0m \u001b[39m# add the position embedding\u001b[39;00m\n\u001b[0;32m---> 58\u001b[0m pos_embedding \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_cosine_position_embedding(x, patch_size\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpatch_size)\n\u001b[1;32m     60\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mpos embedding: \u001b[39m\u001b[39m'\u001b[39m, pos_embedding\u001b[39m.\u001b[39mshape)\n\u001b[1;32m     61\u001b[0m \u001b[39mreturn\u001b[39;00m embedding \u001b[39m+\u001b[39m pos_embedding\n",
-      "Cell \u001b[0;32mIn[6], line 38\u001b[0m, in \u001b[0;36mPositionEmbedding.get_cosine_position_embedding\u001b[0;34m(self, x, patch_size)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m'\u001b[39m\u001b[39mpos_embedding inside : \u001b[39m\u001b[39m'\u001b[39m, pos_embedding\u001b[39m.\u001b[39mshape)\n\u001b[1;32m     37\u001b[0m \u001b[39m# get the sine and cosine embedding\u001b[39;00m\n\u001b[0;32m---> 38\u001b[0m pos_embedding \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mcat([torch\u001b[39m.\u001b[39;49msin(pos_embedding), torch\u001b[39m.\u001b[39;49mcos(pos_embedding)], dim\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\u001b[39m.\u001b[39;49mreshape(\u001b[39m1\u001b[39;49m, no_patches, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mout_channels)\u001b[39m.\u001b[39mpermute(\u001b[39m0\u001b[39m, \u001b[39m2\u001b[39m, \u001b[39m1\u001b[39m)\n\u001b[1;32m     39\u001b[0m \u001b[39m# expand the position embedding\u001b[39;00m\n\u001b[1;32m     40\u001b[0m pos_embedding \u001b[39m=\u001b[39m pos_embedding\u001b[39m.\u001b[39mexpand(batch_size, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n",
-      "\u001b[0;31mTypeError\u001b[0m: reshape(): argument 'shape' must be tuple of SymInts, but found element of type float at pos 2"
-     ]
+     "data": {
+      "text/plain": [
+       "(-0.5, 63.5, 99.5, -0.5)"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAGFCAYAAAASDy0NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7SElEQVR4nO2de5hddXnv11r7Ontue26ZXCbJJJlMkgbIjVtAQIECIqgVH9HKsVYQqQfQPmI99Txa6+lpCxREpda22nppjZeqFUGtAoJyFRLuJCSQ6ySZZGaSPbc9+7bWOn/Ys36/77udtdcwGZjk9/389Xufd1/WnjzPm997t33f9y1CiJE4r/cDEEJeP2gACDEYGgBCDIYGgBCDoQEgxGBoAAgxGBoAQgyGBoAQg4lHfeGSz98G8itXfjk4L/vOdZPqaumn8t6Z+lw+0+x6pum8l8+kcObusGrBGwAhBkMDQIjB0AAQYjCRYwDfffsXQL5+/znB+eMX/xh03x5tAfniNzwN8tPFYnBevmYf6PoqYyBne44E52FvAnTJrnGQi35ZCR1F0JV9F2QvWwbZ9T2lq8fXSrw6b3JdanKdnwjvu/Ljk+v9WI33hply+1XqyAkPbwCEGAwNACEGE9kF6IyVQH7yjnXB+c5bHwfdkruuBfnBS28H+fpd7wzOH+z6Neh+OLoa5Iu7tgXnZ0p1oFs99yDIfRV17Z/bPgy6Ya8Acn0zykW/EpzjDegeSPfBrlOv1V0Hy7IsKxniAoRc8WvqY6FvtSwnxH0I09VyAabjPtC9mPXwBkCIwdAAEGIwNACEGEzkGMC5938E5OXfUn7/z/4yBbqef0cfetFbG0De/sDS4Hzx1f+J37PlzSB/8aRNwfmu4fWgO6tlJ8jPleYG55XZw6A75KKt62waBXnYUzGOhnoZH8Dfk0irGEDFwviAk0IZYgQiPlAVPwhJ9dVKA4aa8tAUYY24RIgfXzN+EAbjA7MC3gAIMRgaAEIMhgaAEIOJHANYeQv6zJWNpwTnDz+Evvnyh7aA/LM8xggW3qtKehs+mAZd/qk2kNesV775DftWgE6PD1gWxghWNxwA3Y5yB8jdDUdAPuKpRHtrfR50o14F5ExaPZOsEYgnUdZjBE4ivMTYEnUAECMIyeVbVo0YQZi/Xeu/gBoxgjAYI5j98AZAiMHQABBiMJFdAO+VPSAf+o5K5XV/KYEfuqgL5E+/1ANy2xNbg/OzJUy5zdmC123dRRjZgV2Gy9dhd+BvBhcH508s+RnotuS7QV6SGQR5f6UpOHfWobszKu6yjWlVclwQLkAqNXkZcSyOaT/Pwuu1HZ+8jDgsRfjbN7+6UuBplfPSPTju4Q2AEIOhASDEYGgACDGYyDGA/ms3gPzwqarF94p7zwbd3o+dAbJ9P36WX9kVnL95ZCPoGp/pB1mfAtS8HZ2/Ngfbg/fsbw/OS3sxzfeN0bNAvqTtOZD3lVX6cV4aW4mPuJiqzKbVM+XFdvVMUsYAlF+fSFaETpQRh8QI7FhIitCyapT7Tq7ya/nx02oHfh02zzM+MCV4AyDEYGgACDEYGgBCDCZyDOD91/0E5HsnlL8d78Qy29OveBbkA9ctAtlfo0p6f7gNawiW7UXf/KlifXDO7sCxZDEb7VeiLxmcO2Oo23EUn/HauRgjuFcbRTY3hTGAw24jyG0pNY141MNZXfVJfMay5scn42IysYV+fCwmYwBKtmMhNQKWFV4qHOaLz+RIsGkwrToBEhneAAgxGBoAQgyGBoAQg4kcA7ghi+O3Tv7y9cE5+U587Xe6cAz4Fc9incABrU4gsxnfazvo/N0zvCY4p3cOgE5uCmrQlgw12NiCPDSIfvz8ONb77xxXMQ1ZIzCg9QlYlmW1JvQYQBJ09QmMARS0OoFUQtYBoF8fj8txYpPXAVT1EWj6Y1UjYFkR6gTCmKE+gmnB2ALAGwAhBkMDQIjBRHYB3vLS5SB3f/GF4Jy+C6/BAy5edZ00Xscbzj8UnOtvacbXdi8E+d596r2dB3eBblcZ7Vdjn/pemSJ0BvAZW4Xp6xvLBue5nZgG/E1+GcgtCTUxKOdlQNeUwPbmcU99UTouXABxjQ9LAzrONNKArzZFaFmzcrEoU4THDt4ACDEYGgBCDIYGgBCDiRwDKPzdfJDrtam7X13yPdBd9sL7QK47PQvyX/V+PTjf9vz5oBs5bynIozuUwzenWATd5sJikOv2q/ScbLWtGxBjvRwRtxhR24vaYuOg6y9inOKk+r7gnHPDYwBFX5UKp+PYKlwSrcSJmEgDajGCWFUaEGMCtubnyxShjAFAmnAapcAnXCuxgfAGQIjB0AAQYjA0AIQYTOQYQOonT4C87XNnBmeZzy5v6gR58AL8rAvqlK97ywCW9x5ejyPEm7ers5NBf/uRYXyt0z8UnI966IvXHcZnTNnYhlzIqbFfrQ6W8/YXsIz43Oax4CzLhBvjog7AV9+TiYtWYeHmVrUL+3oMQG4WFqXAIbl+O3QzEH3tAAPrC3gDIMRgaAAIMZjILkDxLaeB/P23fz44v2/Hu0DX/qMXQe4SpcK7yuoKLa/1besOg1z/C5WCs+eja7HlEJYYdx5VpcIHKvjTMgNYhiuJ5dTrG0VH4uE8ugBZR5UCb6/MA11jDF2AUU9NLs6INGDBR/ubEmlA3bWSpcAyDRhaKjytzUAzNE1ohmCZ8NTgDYAQg6EBIMRgaAAIMZjIMYDMTftBbo8pf/bwd3Hqb2fhKZA/s+i/QP7r/ouV0IvlvNd04xqhH+xUsYf8SowB5Paj/dJLhbeX54AuNYC+uZyak8wp5zEjUoRH87iBqMlRnzVYbgDdsjSmNcc9Faeoi4mtQZaMAWCcQk/1xYWP74rUqwOlwJOXCVchfOaqaULTKOcNLRWmrz4r4A2AEIOhASDEYGgACDGYyDGAH/feDXLPzz8SnFd9/2XQjbxlDcinJB8D+Ze/Pjk4t6xDZ/DyhldA/l6/ihHkLsdxYZl9FuKo1tsXJrpAFTsyBvKEj2W5qZx2tvHPkh/DeoNmR8UajpSwjmFtPU4qHvVUiXF9HNuZCz5uFZLtwHrEIF5jM5AzhVJgaBeeTkvvTMKJwq8JvAEQYjA0AIQYTGQX4PNHsfNu1S0jwdnL4RTdo3+I1+3fFDH91XWfuurufYtI5cXqQfa11N5wL16RO57A98Ya1HtfGGnBH5AbQdHDlFsqp67YcqKwN45pwXotJZcTLkCjg+nGA2X1HBlHLg3Bz03LNKF2061yD3xZCqwvBhGdgiFX5tAUoWVN77o9CycKE4Q3AEIMhgaAEIOhASDEYCLHAL755UtA7typtnoWLsK03zfW/z3In9r9dpAzj6oxP2v/fPJWYcvCduHO5YOgq/8hTuu15rQFx1eOiFbhUdwqNOgK/zuHPrZObBTTdRktr5YrYJlwo4NpwGFXTVPOxDAGkPfwGdOiFLiktQsnqtqBEb0duLoUOCSFOJ2W3uNwci/bhRHeAAgxGBoAQgyGBoAQg4kcA+j8xydB7r/u1OA8dib6vack0Wfe9ctukBcNPxqcr52P48M2DW8A2V4wNzhfMn8r6B7fvwrk8rxscM4N4jPIrUL7XYwfJHMqBy9bYhNj6DimbfXZowX04+sd/J7hiooRLEgdBZ2sA0g5cnuwFgOQW4NErl+2C+tUlwJ7mm5qpcDHaqtQTeirvybwBkCIwdAAEGIwNACEGEzkGICzvBvk91/3k+C8Nr0XdH85sBbkhfflQY71LgvO56YfB91Nz54KcscyVQfw5qZnQPf4AI4IGzu1IzgnBsKdyN2lDpBjw6qGv2Khv53A0gQrocUA8gWsY6i3sZ5/pKLagZfXYR3AuKgDkCPBylodQMyWI8GQmN4LEDIurIqwVmHLmrFcf83NwmGcYLUJrye8ARBiMDQAhBhMZBdg2ydw+u2PszuDs2yfve7fzgF5yeYtIPf/8brgLKfveJsxPXe0V51PSuD1zh3GFt+xLvUcaVwwZNlx/J7dhXaQndHx4Fzw8SqeGBMpN0u5AOUJTOVlRCpvtKxcgLQt2oE9fG+dI5eHqu9JijSgTPrF9FJgX7oAky8WrdUOHLpYlKk6xXH6t+ANgBCDoQEgxGBoAAgxmMgxgF+96Qsgv+WlK4PzRXOwRHfpt3E7jueiD1q4YDQ4P11Cn3nOFkyj6SPDMg6m3CwP/eLxLiXLcWFyC/Gucfwsf1ylKkc9mQZEP1mPefgFLDlOizTUWEWl+uqFj3+gjOPPZClwSYs1JGWZsHDd9RiATAOG+fE1XdfQ7T4zlMqLoifHBN4ACDEYGgBCDIYGgBCDiRwDOOSiz1y4VY26+oc34nbgpVsfBdk+7WSQP32y2jL01cFzQVf/3AGQ531E+dCD7jjo7AQ+U6ZL1exmfoI+v93cBHLfaBrklrz63mEP/frk+OSttk4ebWhaONyjJT0GgK3CeQ+fX8YA9DqBpCPagYWTHAvxx2UdgM5U24Ej6yzruCvLNXFcGG8AhBgMDQAhBhPZBbjyBzeCvOynauHnsv7V+OJ1KPdd0AjyFQ1quu8nH30H6Jbvx7LhN89X04a2FLOgc7JYNnxKp7rGHxmYDzqvDV2AI8NYhttcUNfzIQ8n/SZGJ58YHJ/Ae2NC3IvHS+qanxadgqMuuiHNceyaLGtpwLgjNwOJrUgwFRiR7gFMDQ7TWRFchNcDA6/qMwVvAIQYDA0AIQZDA0CIwUSOAay4Haf+jL/ltOCcuucJ0L1y60aQ557cD/Kwp6bvNG3BqTh2HH3zNzeqiUFfHzobdH5nG8hntzwcnH8yhHGHiR6cAFQexlSfXlY8UMF4QXwcfXd9Mm5MxABke3OhrGQZA5gQ24k6EzINqOIHqRppQH0q8FS2A4dOC6pF2MTg36GPrCOvGbwBEGIwNACEGAwNACEGEzkG4Odx+0/Dx/uCs7erF3TXX/pTkOXU4DuGzgzOczZj7ttZimXFqzQ3+f69+D0dXVjue1qdGlN2z3AP6CY6sC4gkZvcCe2vYH2BM4ZtvPrU4Dj+WayYKAUuFtUPSNvo44+5YiqwgzECfXNQolYdgK23A4tnEn6+3i4cOvLLssKnBs9gjcCrnhp8gpUnzzS8ARBiMDQAhBhMZBdg1424iPP55XcG59Xv/p+g+2jL7tDP+uNfqDThqhd3gC538QqQ9SlAxe2Ynhvtws9dHldXaG8MOwfzc9DWJXPioRyVFjxYyqIqXwC57KtLdky4APrEYMuyrEpJySkbL+cyDZi05dSfyScCyTSgPjVYlgLLNKA+NbhWqW9NFyH0zdN474nELP478AZAiMHQABBiMDQAhBhM5BjAzVd9DeTbjy4Pzle+7Vege2AC7crCOG7wmX+fcorco0dBN7AeHSZ9ClB2Oz7T2CJ8bUtMpQX9stjC04G+bn0fiJaTVP74wQK2A1sTGAPI+3oacPKJwZZlWV5B/YmTYsFnvoITgWSpcM5Vv0dOC9LjA5ZlWXEtvuCGTAyWyBiALCMOS5vVTiEy5Tbb4Q2AEIOhASDEYGgACDGYyDGA8+uOgPwXd7w/OD/5yTtB1/vLq0E+t+dlkLO/3h2cvUZs2+1ah1OBnyy2qvftQF988GzR0htCqQN96Lbn8b12vfK3DxWwRNcv4PcWND85MVEjj15SNjYlfOaqGIAsBa5opcC2LAXGf7r4FEaC6aXAtdqBQ938mfTx2Ur8msAbACEGQwNAiMHQABBiMJFjABuf+ADIC/7pyeD89E3oXy/8Fn7sQ288CeSl/WpzkH0q6q5ZhK3Edx9dG5yTuw6Dbv4C9NX1mgG5Nai+A9uOUzmxOUjbHjyYF1uDisMgj2qbg+KFyXPslmVZdlE5rEmROJ+oYC+ArAMoepPHAEpVdQB6OzB+jxOay6/VCxC2HTj0rTX8+OOvRuBE3BzEGwAhBkMDQIjBRHYB5t2C11Wnd0lwvupJbOFd/POnQO5y1+KXLl4YnPvXYxrw0sw+kP/6+UuC88JD2Dp8Wgdev3dqaTWnoR50S9uGQC4M40/3m9Trh8fRfciWsKx4VJvWG5sIdwFiJXVvlFuDChV8hoRoB9aXh2YcfIayaDvWNwe54q4qS4F1Sd5q5WagWpuDyPENbwCEGAwNACEGQwNAiMFEjgHYjz0L8ravrQvOCzaJ+EBbK8h1j2If7+Db1fbgI+sxvaW39FqWZZV3qDFgssX3rMZdID8xsVQ9bxbHh61q2gPyc8Oor2RVC3BxXMQHXHzGnKeeMZ6ffHOwZVmWo6UBE6JVuFjG75FpwIKWBmwWs8dkKXDC1kuBw9OAestvWIrQssJbfmtuDtbeO6WtQVH0x/p9hsIbACEGQwNAiMHQABBiMJFjALmrzgT54TfdGpw/8KELQdf/x+tA7vjyYyAPXFAMzmcv3wm6rSUs2W1+SZ0d0Tq8NoWtw58ZWB+c3dYG0K2uwxlgz48sALnUrW0DGhN2UYzJ0kd1xQpiXLfwdWPqp1qOsLelqjoAjCcUvfikuqpSYL0OQG4OtifP3TuiRkBvFbasaZYCzxCvemsQqYI3AEIMhgaAEIOJ7AKc8ZEnQd6nLba009iVN/ddmHKzHlwO4o2n3h+c5eLQf8+dAXJ2h0p/2fM7QdcVw/Tjs4fUAtCODkwnrkwdBNkfR1ej2Kx1+I2F322HXOVe2BOYuquI1Zy6CyAXh5bLeI1PijJbfXOQXBwq04D61B+5OLRq8q92dmpc46eyGcizok8Uft04wToUpwtvAIQYDA0AIQZDA0CIwUSOAXxh/hMg92y6ITi3vh0dq7t7/g7ki8//OMjXNG8Lzgkb/eAPPYkpxB5tCtDYOlwHrG8OtizLyvcp3zzfIbYT6c64ZVn+BJbWFrPqN8RHhaMoHOGjFdU6bBdFm66PMQC9i1emASsiBpAQ6bqSp28VmsJmIJkGDGsHnsZEIG4OPka8jn8L3gAIMRgaAEIMhgaAEIOJHAP4UN9GkFfcrvL3w1/FKbrSqlQuyAm98zvPlmVZiS1YwuseejE4H+3tBp30t+v7lF880YHP0BrDWgWvJCbwajGAxDioLDuO9QaDmu9uF2QMQJYCa623wtnzyuK325PXAchS4HEPf4+u90QdQGgpsBz55YeXAsP24Gm0EpPZAW8AhBgMDQAhBhPZBXj2jjUgt+RVm943V30XdNfsvALkT62+B+R/HVkWnNemsWx4zmaRrquo9NdoL6bCDrqYymvYp666Q2vw/pmy8RpveaK7Lquus3UD+FI7ie89UlIugF+S03rxWoylwMLeShdAvLfkqu+RE4PLPpY666m+kpgY7AgXwNVv8dOZCBT6zhoYWHY7G+ENgBCDoQEgxGBoAAgxmMgxgKZNj4O869MqLdgRw4Wee763DOQr/hz1vb98R3A+t+dl0NU9j5N7PG0KUPeyQ6B7sdQGckOfcrj7L0Q/uBaVrPKxk2P4XtnufKSoyaUC6AoijRYvhpTSVqUBUV/Q0oBJkQaUpcChacCQUuBYVavwFCYC1WKm/PzQlt6Z+coTFd4ACDEYGgBCDIYGgBCDiRwD8M/COoCbr/pacL6h7/dBN/8/XgF56BOYr2+7V5UOPzCxEnS9/dh2HFulxoldOm8L6B4d7wE5cTAXnLPt6LePeeir23H86cmsih8kxrC02U6jnCsoubE0ArpxD21qrBQWAxBbfIW+6OpTgbEOoOhhbUIipB24ajOQpq+9GSisHXgGc/msE3hN4A2AEIOhASDEYGgACDGYyDGAgT9DH/r8uiPB+X//x8mgWzD4G5BvGzwb5I4H9gfnSh1u6HGEvz3e0xKcL254AXQff+WdIMcODwXnZa04LuyQiz60nUR9e/NYcE6Mie3AdRhPGC0ouaEifHORn48VQ1pxS+irJ+TYcL0XwIpeByBHhst2YNefPAYw+dP+f732iilsB56S7jjEP05/D28AhBgMDQAhBhPZBXh4wzdAPuOJDwTnRd/bB7r8BWtB/u6jaGd6dysXoWNzM+isnm4Qj/aqR1yRwGvvywdx7E/PqHItVjdhm+7uCn6PnakDeWFjLjjnxrDV1q9Ht2SioKXgXDGpx8f0nBPiAojMnhUT92K9HbhWKXBGGz9cnQYMeYZaacBQLeLVdCDIbIM3AEIMhgaAEIOhASDEYCLHAH6abwd5/s3qrZW92MK757Pomy+4S6TGOjT9NhwJNvS21SCPLleOshzrFetD39zSWnFX1+EzbSvOB9luqAd5cUZNOR4Rv7WSxXhBpaB+uy9iAKMevjY0DViWaUC0x6XK5CPBCqIUuDmmyq1lGlBOG/ZCSoFd0c4cVipca+pvaHxBvNcV05RnLIV4nKbrZgreAAgxGBoAQgyGBoAQg4kcA/jU168CeeHjjwZn79y1oPvy2d8E+Y4/exPIo+csDc6ZH+CoscEN6Dcu7ekPzgcrY6Cr34cOnZ1SJbq9icOge2B4FcheE+b6l6TULPDn84tAV+lqAtmf0Oym8JlHPLElqay16Qo/18FShaotSXopcEyM6qp4cpyY/j2oC2sHDtsaZFm/YzOQ9hw124Hpb896eAMgxGBoAAgxmMguQPcXnwd56Kozg/PhszEVdlEGF2/edgiv430XdgfnVfdjie6KNXtBvrBjW3B+vDgXdI19mBqLtWSD8/w46rbmOkGON2O6rjs5GJz9Cex8LDeINGZ+8rvtqIuf6xTUc3jiGu/IUmCRV6tAGhD/xnIikF7uW9Up6OB7Pa11rfqKjxyrqT/yt88U/rSmGNfSn3hTingDIMRgaAAIMRgaAEIMJnIMwG7GVNjGj6rpvWc17gDd3+cW4pd0Y1rtXRtVO/BTq3Da8Ie7vg/y0oTyzT/ffyHoMn3jIHtz1aagFgfTcfsHsyDPy6IPPT8+rAQRAyjVi0m/qAZGRRrQ1tKAsl3WwVBJVRrQrShZlvOW/ZA0oPgcx3r1U4GdsO3ANUuBw5RMIc4GeAMgxGBoAAgxGBoAQgwmcgxg601dIN8z7+5JX9uz6T0gt74RHbqPtW8KzhevPwt0F9YNgpywVU77oT1LQdfTj68dX6smDOvvsyzLqgygb17Iou3r0JLyfglrdMsNIGIdgHB0x1zRolxSjn7Zxyy7jAFIXFc9oywFLnlis5HWLizrAORIsLAYgCwMft02A5HfMsOxEN4ACDEYGgBCDCayC/CDt34e5A/1vTE4n9u8HXQ9m7Brb8dHMOXW5qhy2dwGvG7X2biwI6ZPydmBU3y8IfzekYWLg7PsvEsP4LW4mAXRanTUn8Ir4d283Ij3sLiWBrRj+LnDFSwFtsvqau6Ka7xcHOqI+54fkgbUF4f+9r16KbCcCCRLgdXn1k4DChfBn7wbUE4TCkv11UohktcG3gAIMRgaAEIMhgaAEIOJHANojaFf/OznVAnvz8/B5aC9T+Jy0I+uPwry98fVws8LV28F3QtljAno6bksuvyWV8Ca3HEtUznmF0FXdxj90Yk56IRC7METW3gw9GDVHVJnO45/wpEKxjussp6eCy8FjompwH5FPWPVRCBRCqxvDsp7uMw0bCJQVZmwdOOZ6juh4Q2AEIOhASDEYGgACDGYyDGA837xUZB7Nz0WnBv2rQVdbEUPyFc3fwvkM554f3D+5zU4QfjrQ1gafFbjy8G5eUceH0okk8sLld9/yEV/u24Q5eHl+F7pf+tUGvC98d3aexPo84+W8U/qQymwyPvXKAW2tDoA+XQlWQdgR68D0NuFwzYH/1Y/jc1A4eoab36VsYeaY71e3ceeqPAGQIjB0AAQYjA0AIQYTOQYwKpbciCX37A2ODsPPwO6XZ/aCLIchRW/LxucN5yO3/PerTgi7EiPSsIndvWDzmvAPt3F84eC885yK+jqDmN9gduCNfxhuA3oQycm1HvtJMYARsqYg7cqqlZBuvyxcrifa7vKYU0I37Xk4fMnYSQYvri6nl/Z/bgjNweLZxDylDYDhcH6glkBbwCEGAwNACEGE9kF8HbvA3no1u7gPO9GnAL81j94BOSvDK8Eed79apKP9+d46azfgu20D8eXBOclh18AXXwxTik6vX1PcH6xsAB0iSGcIFzXjG3HeU9zERyxCagBV/jEJ7TNOiINOF7Gz01rC00LvriaV2pcg6EUGCkLFwDbgcXzT2EikHQfpsOMlRHTfThm8AZAiMHQABBiMDQAhBhM5BjAwes2gPzo+juC8+lXfhR0/znnhyCvuO+DIC/f+lRwfnAiA7o5W7DFty+r9eLKNt15WZDPbvhFcP7R0HrQ2UeGQe5sxum9Y75K0tkJ/LPU1+MzxbU0oJVCn3+8JGIAZfW5BTmtt0YaUN8eXLU52Ju8Hbjs1SgF1keC1djaG14KLCcKh5cV45unqT+O8Gfxb+ENgBCDoQEgxGBoAAgxmMgxgGuuvQfku/MdwXnjO7AUeGsZi1477sXy2FhjY3D+Sv+5oEs+vwfkbNdy9b5sM+iGu7BmYHXycHC+OTcHdE2jh0Be2DAB8pBWdmsn0Y9vrcc25NiEih/4IgYwUcK6AN/V8/Nia2+pRilwRR/dhZTd6HUAVWPBrcnHgruyViFkc1BN15b5+lkPbwCEGAwNACEGE9kF+HB2F8gnf+n64Pzkn9wBujOe+ADIix7sAzm/sTc4b30ObVDvIE4Ubn5ZlfT6XfNAN7oQ39sVV65G/yC6Cw0T6FosyWBq74Cr3BJbXOs76nDT0UihST1THb62WBR/Um0ScEFM6nHK4Wkz/eYes6KnAT1LTjsKKwWu8QzH6Bo/pRQhec3gDYAQg6EBIMRgaAAIMZjIMYBLtr0N5O4vPh+cD16L03aaNjWBXNnzIsh9H1ZtvG2b8XucelzDY2tTgEbPWgK68S70K1O2loI7LCbziDLiJakBkPeV29R31mF6cW4aU4ijEyoF6mUw7VcpianArvrecbmxp0Y7sF4K7MgJyK5IKepTgUWrcCI2eSlwrCrNF54GjKqzrPCpwTVjC9p75abn0PzjdMpuZ3HJ7kzBGwAhBkMDQIjB0AAQYjCRYwCVW+eCnGxWPtxVL/4R6LJ3PwuyM7cT5I3nqtFeh7+N48T83m6QvWe2BeejvbhxKNk1CnJRa+lNDwjbJsZ8LUwMgfyrMTW2zM9gq/CCVA7kl4sq5uG2YzuzXxTDu7RtQAUf4wVOSfrm6OvaMBJM1AGIUuBYSCmwzPVDKbDcDlwjBuD6x2gqMJkV8AZAiMHQABBiMJFdgOTPngB56+fPDM4dm/DaaMcxxZY7pxvkL8y/LThfv/1i0A1eeQrIrU9pabTlmG7cMPcgyH0VtRy0bkAs4qzDa/38GLoPu/IqDehlMF3XmcBpQnZBPUelDq/bdmlym5qXacAapcCQBhS22hWlwHrHX9VUYHnN9yfvBqx6hhoTg+BzxWtDXYTXKeXm020BeAMgxGBoAAgxGBoAQgwmcgygcDlu8fzPt90RnP/Xpy4C3dDbVoN8+GxMdy1LqKWe3jhu7BnagH5xxw9UW++qZQdAd1bLTpCfK6lUZWYAv9NpxEWibTH0BfeNtQTneAO2+M6NYwzAL2lpwDq0oXZxcudWpgGtikzPCR9abwcWdbWuKAXW/fyK3Bok0oB6jKAqRSgmAkk/3g3RTQf522cloSXIx8Hz/w54AyDEYGgACDEYGgBCDCZyDKD5pr0gZ7UktV/B7bn+uwdBfu+CbSA/MKG1o7a3gW79Ka+APL5IjQG7rPNh0K1N45ivu4bVNqD0QBF0fqsYEWajP354RMUIOhrwz9Iaw5FgVkmVHFfSYvxWSAxAtgPbZTmtF/1xB/+sgOuK79V8aDl9uGozkGb3EyHjwiyrVjvw5M9nWeHtwDU5Tn3q4w3eAAgxGBoAQgyGBoAQg4kcA7hr+c9AXvLTjwbnBZdi3nnTSbeB3CycxXe8eFVwTq3Ogu6aed8C+f/2vD84n595CXQyl//4QHdwzgyi317uxDFlGQdz/fmcGgNWbkC72OaIeIJWB1Cuix4DyHv4nXYZnXy91dayLMsO6QXwPDkSbPI6gJiILejbf2qNBQ9jOnUA04sPTOO9BOANgBCDoQEgxGAiuwA3Dy0HedUtueC8928wvTUvhlfdhI1X0iO/VKm92HpQWeemsU33pl713p6EaKcVd8G9B1RKceVRTFtOnNxuhRHLqT9FqQE/t1G4MHra0xUugPAW4K475mJLsuVGTwPK3+pPIQ0oNwOFLgetkQacisPAiUGvAdN0h3gDIMRgaAAIMRgaAEIMJnIM4DtfuhDkzj1PBed/W/cg6P70wBtBflMzlgJ33af8/B034iOkbJT1MWAyliBJ9KnYgz+KsYR8hxipJSbwJnNKX8bOYSsjvtfX0ncVXCJkxXBqmWXH1HvzrkgDVmSJrhhjFrI5yHdlua/6PVXtwJZsB1Z/Y5ki9PzJ04uSqviATGNqepnirFXqO600IYkMbwCEGAwNACEGQwNAiMFEjgHM+Qqu8T3w4VOD8+oktuk+/IN1ID902lKQFz6tSnov/z30QTeLbTmrl+0PznsrWN7bLLb9NOxTZ6+ICflCBzqVEz4668mcOssYQEq0Dlta/EDGANJHULa0GMC4K/7cIgZQlpuBYCQY2uqwOoCK8OOToh1YH09esw6gaqS49nzM8x/38AZAiMHQABBiMJFdAHsFXuOv/dCPg/NfHMYr/+Lv7Af5wMgC/KyYsjvvb/016P5h4I0gXzZHLRp9ZAIXia5M9oPc2KfVzoq0U6EDr8GjHnbipXLq9RO4B7U6/ah9tluH3xMTpcB6GnBCLPS0xCQlWWYbNhHIcuVVXfvYqk5BOfVnCpuBptUt+KrfWuOD6XocK3gDIMRgaAAIMRgaAEIMJnIMYPsnMN91XbOayHvSP18OukW7HgF53i+xBLayYWVwPjmJr/35C78H8sfOvzc4f6bvMtCNtOKEoLo+lSb0RIow3lEAecDDn57Oadt1G6LbRRkDcEpyU7L6nvEKphPlNOWSH70U2PJEGlATKzW2A+vlvlXTgqzwUmAvRDcVaoYHZsrPD93uMzNfOZvhDYAQg6EBIMRgaAAIMZjIMYCHzvsiyJdse09wXvKdAdBVzl4LsvvIMyDv+9TG4CxLcpu34NivJRepMVq/2d0NulQMfWjnkFaHW58B3YL2HMgHKrgpKJlT2378+vC2Yx2vDusL4kXx3oT6E+crGAuxXIxLTKUOwJ5CHYAcCaaX+1a39EZvB5Yus2xnZqnw7Ic3AEIMhgaAEIOJ7ALsEW1vlVtVvayz9QnQvfz3Z4C88gVcyrH4jSqFeP9EK+jmbB4HWZ+Gm3gJn+HpBiwx7jy6KzjLpaOrsodA3l3CKcHx4Qn1PQ14VS/7eM3Xa1xt4QLESsKmamnACZEGTLn4W8tyaI4bcoUWj6Q7Hq4sBa6aCKReHeYezCQz5h7Q7ZgSvAEQYjA0AIQYDA0AIQYTOQZw1fevB3nZzx4NzvaG1aD7zAXfB/lf7v4DkG9Z8qXg/Nk9b8UH2oobffa7+eCc3YH+av9cjC3M0aYAeS2oW13/HMi7ih0gO6MqBtBYj9+TF6lKvcU3VVcGXayAaUA7ofz+QgX/3EmxGagkU3BTSAPGtLiEnAhUtRlI08uJP5KwiUG1SoGZBoyG/zqWIPMGQIjB0AAQYjA0AIQYTOQYwIrbd4E8dvnpwXn/eWhH3tc0CPJnL0C/+PSU8otffGYx6JYffQzkX08offMOnAo80o1+vqW1AJc7sBR4ZeoAfu5R3Hbsj6lYQ3tGTtFFWW/xzaRxBlisKMp9tVLgQhn/3E0iBlCWrbgh7cBi0C8g6wBkO7CrtwPLOgAZhwiJEUylHVhuPq4J23ZfE3gDIMRgaAAIMRgaAEIMJnIMwC9hLrzlJlXP/8523P77/TH0zc97w/MgP1tSbbDtW0Q+uwnf+6OBtcHZ2X0QdA19jfjehvrgPNaBvnh3fBjkfaNZkJvyqlegM4N+fU741JaW228SMQCnVA+yr8UASqIOwHdljT5+jx0aA5jCWPCQXoCECCZ4NUaCuSFJa+nlh7rqrBGYFfAGQIjB0AAQYjCRXYCdN64EedvyL03ySsta8uMPgnz/JZ8D+WN7VGlw21NHQeetwLTg5l3qEXuGngZdY59YwzNHtQDnO9C2dcbwpw4O4wbQxqIqQZ6XxhrcI14aZDupXICWVB5040V0YXztteWy3DCEl+aCj8/ohLQDy4U9Me3CLa/pVaXA2mtrtQOHbQY6lqW+U04TkmMCbwCEGAwNACEGQwNAiMFEjgHc8d6vgnzzkCqlPTWzE3TLNmFqacnl6G+/8GBPcF66/SnQDfwRbhpObZ88mZTcj6m98rxscC5gt6/V4KAfX8rh9GF9S8+8ZA6fyUW/3k6qFGNLEsuTJ0r42/2U8vsrFYwB+B760GURAwhPA6Ls6O3AbvhmIL3ct2pzcFUrsdwMFL0dOHSicI1y3lnZSnwCliDzBkCIwdAAEGIwNACEGEzkGMDGdA7kT972geD8jxsxH7/8QfTr75tAn3ThfdrorgJuxzmyAXPw8+7XRli3tuBDDRwBcexU5fgX5oTM07IsK56b/KfPFWXD/WXcImSlVAygNYmjvQ+W8HvdZjXK3CvVqgPAseGOG5KDdyd3SF0vPJcPY8FrjASbCq7/Gvnt4qe7+t+xlp9+Avrx04E3AEIMhgaAEIOJ7AKc9fg1IC/8yubg3LR7DX5oF27s+fSOJSA3/UZ1D9qdc0B3zskvgXz4HxcGZ3/hXNB5z+Jrx7qUPUt34NW86OP03mRO3AW1vFRHfARUWwvzQfY1F6A9gWlAq4wugKelAf2ysLfiylzyxUThsrrausJdkGlAKAWumggkpwLrqTxZJhw+EYhTgU8seAMgxGBoAAgxGBoAQgwmcgyg62YxrWbF0uCc/gWm/fZ95HSQ/fvxsxoKappQZeMq0F3TiSXHf7vnguA8fD5O8m14Gh3h8YVKXt6KbcZHXExVpnLonzopVRrc5mCLb79s8U1r7cBxjDXYJYw1eEnt71YJz0HJNKBdCUsDouxottyT7cCWLDmePA1YNRU4xI+vtVUIPle8tmZ84HVI1/kGxix4AyDEYGgACDEYGgBCDCZyDMB6Aif7bv/G2uC84qZW0J1yxYsgD96AdQH2KSuC8+EN2Ja7MYXOrTukyn2PLkd71ZjAyb/1XaPBeVW2H3T7XPye1LDwSetUyW5rDP34QxMYA/DqlK/eFhN1ACIG4GoxALsUbm8LnogBaKXAnvShQyZoeTVKgcPqAGqNBJPxhTBYBzD74Q2AEIOhASDEYCK7AEffdybID513a3C+9Io/A913F90O8jufeQPI+7U0YXEdptHCKPRi56DTmgV5TadaAHpypg9024rzQE7lxMLPerVMtNFGuzgwgcs+6tLqz5aN4fP7FeHCpLXrNnoHVVRNBAqZCuzIUmCtlNmrsRzU05eDVpUJh08EgueTS0MmfeXvem/010r3Z1YuFQldZjoLn/e/4Q2AEIOhASDEYGgACDGYyDGAc258HOQ9FZU2W/CuXaA74mFLrB3Hr0mfPxCcL5uPLb2PFsVEW20K0Pqle0E33tEO8sbsw8F5ZeoA6O4aXg9yMofLTv1GFQNI2fi8ufE6kBN16hkbHYxLWBUxESipxwCmVgpsVfQ0YHg7MHynW6sUWJ8KLP34Y9cO7HD6zqyHNwBCDIYGgBCDoQEgxGAixwBum7cF5J5/vz44P/LuvwPdu7ZeBXLiNJzm+9kV3wrOi+PYtvs3By8B2Vuk8veXdfwCdF/vehvIp9WpDUVzY9j+u3UEx4nFh9F316f3yhjAxBiWEbt1ym422pjc90UMoJKKXgdQrCoFVo6+nLgb1g7se+EtvWF1AHKz8OvFVOoE8I3H9DFOeHgDIMRgaAAIMRgaAEIMJnIM4Oq9WM+/4naV+3feja8d/zbW3Y9dgI7ZJRndP8+A7tfPrQB5Xq/KuZ9Xh1uI7+zCx18eV052ysZW4Z1DbSAvHsHtP6WFquU3JnoBrDH0zct16vc0OqIvtyxHgqlzVR2AcHTzHj6zFboZaFKV5dcYCVbRYwC1xoKHtAtXxRbkM06hj4C8PvAGQIjB0AAQYjCRXYCtd5wEcra0PTh/YOcVoOv44TaQO3+YBnlvRU3RaXaw9Ld1syjDXa6unIvi6C6MLcSrbksM9TrjA6jz8wdBLmbF4k6N2BjaSa0K2kqLa7wvru1uWBpQuBpVE4G01uJapcCOdjUXS4Sqy3mnkOoLnQo8hWu891otDiVTgjcAQgyGBoAQg6EBIMRgIscAGr/9GMg7//Ks4Fz3XXxtZ34zyH+9+Gcg/+2hC4PzuU3YDtyxBafs7rhx8kcsdpUm1UmSA/g5/jhu/yk2a9t1hROdGEOfGWMAGDvwXTESTKsiljEAW/TLFj3xWz1tO7Dw452QcWG+Gz7Wq+KpZ3ZEbEGOJas1MiyMsBhB1TgxGSMIfW/kRyA14A2AEIOhASDEYGgACDGYyDEA75x1IN/xXrXF9wtnnwe60UvXgrw2hePE/ushpX/p5DmgS720B+QNS1RufHcF/fb5C46APOiqEd0ZG3Pq6QF0HD2xwafYovQVMeA6IZb/uFoMICW+RybhXa26NzmKL5V1AEWx0cfS6wBkO3DIZiBffo7A02oGZHygMI3twCFhCTJL4Q2AEIOhASDEYCK7ALlP4AacjelccL5jCK/iA+/pBPnpIk7n6bpP3V/3uPNBt2wEJ/9e1q42/DyQ7wHdaR342pfLquR4fnwCdJnD4s7s4TW/lFX317yH7kFiDO+24/O08l45gkZc1d20kh38M1l2DO3vhCvKkfWJQHI5aNVmIO2zxE+t6gb0Jp8IVGv5p1weGgY7/mY/vAEQYjA0AIQYDA0AIQYTOQbw4Np/A/nUxz4YnDsvxM053zjtSyB/ei9O761/eEdw7mjCCUCxFpwgfE7dQ8H5T3dj2/EfzsPy5M2F7uBcSu8DXd1A+EjeclY51aNVpcDCr9c6i6umBwlwIpBQxtDnL3qirNjTNwMhYROBLJEGlE+o+/nVE3/C04C6Xr53KtQq533V1b6MO0wJ3gAIMRgaAEIMhgaAEIOJHAO4axxz+4tuVl7a9hvQId2QRF92+wNLQV589JHg3PLUEOgqqxbh92hjwJ7Z2QW6W7pxA/BnBtQG4HQrOtypQawL8MQoskSzqlXIibbc5Bj6upW66HbTS2l1AGIqsC1iAAVXlBVrdQAlP3o7sIwBxGSFsa++V9YIyHZfWScAz1BjKvBURoZNiRn73Gnqj0N4AyDEYGgACDGYyC7A//nae0DuevLR4Pzls54H3V8N4gThhffj9TvWuyw4ezuxnPfwtRsmfYbMdlzS2fX7eGV+9pAqK56TxtY7Z2gEPyyNn9XarEqdj7g4QTgxhgs//fTkE4QlXkpdjGNlYW+FC1ByxT+HqyYeTS0NWOOZfL0bcPLFH5YVvlh0KshSZjI74A2AEIOhASDEYGgACDGYyDGARXc+B/LQ+84MzhdlngLdDd/GCUFLn3gG5EP/Y01wbv8nXPg5vB5bh1+pqPhByw50fDMOLtPM9zUE563Nc0EXG8YYgJ3B8uWuxlxwPuw24nvHMaXoZJQ/KycIV5FU+qrloKIduCBiAI5f0L4H3xpWhWvXKAXW24HlVOBjNfXXssKzZmwVnh3wBkCIwdAAEGIwNACEGEzkGIDTmgX5nBvVpN8vHl0MumWbcPaVV8INPuMXqDG7c++eB7qLVr8I8gP55cG5cQf68WUfYwL1fSqvvncethUvzveDHGvB712c2R+cD5TxvbFxfP6ktuxYThC2RImxk1J6WQdgx/HPXxQxgLS2abgs8/OhpcAoyqoFP2Tsl6wDCKsTqG4VPna1sjMWI2DsAeANgBCDoQEgxGBoAAgxmMgxgK034fjue+b9ODj3fOs60C17AUd12RtWg/zJU34anP/15D8A3dUdPwL5r/Zerj5nL/rxB13sMWjYp/zV/m7M8/sVrOf3GutBXlo3EJz3Ftvw+fMFkDPaqO+Cj58rN/4mkkrvlMSfW8QASmIseFrfDiz89rBeANuVfrzYbqzl+qWPL+sA5PafqTCV7cDkd1NjSvu04Q2AEIOhASDEYCK7AD++/A6Qr977+8G559u4PdNeswrkvvObQb6y8WBwvnk9tvTKaULPvaKmAPXmNoPuxRJe1Rv6VBlx+jC6ABK3OQ1yd1K5AE+OdOOLJ9AFaNLeWpSlwKLFN6W7ABVx7Y3ja8shm4HKwlbbXsj1ukZ1sgubgeSCTzERSKYBfb2M+Nhd471aPcw6oTXG034Uo+ANgBCDoQEgxGBoAAgxmMgxgIzIO239nBr71fgkpv1e+duNILevOQQybN/dMAy6okirZXZoLb9iC8+j47gtOHEwF5zTAxgDsFM4AqyYxdjDgph6joP5JtA5BYxxtGhjvsaFLy4n/WZSqozYKYvUpIwBVMRmIG0SsNzaWxVP0JnKZiDZDlzDifZgM5CMH0w+TmzKO4SYJnxN4A2AEIOhASDEYGgACDGYyDGA83/+pyD3fkf5/bEV6Itfe9nPQV6TxtHfXzhyanB+7/InQferAo7jatmutdO2tYLusUH062OH1ZahzABuMnIaG0AuZNHfbo+puMRQHsuE2wu4vaglqW8Sxj+hbPFtSKoYgF0KrxmoqgPQ4gslaavDRoKFjQy30FcPy/NblmUlan3Yq4Tp+tkBbwCEGAwNACEGE9kFWHUrTvkpn7MuOO+9ANNbP2l9JfSzPnT/2cH53os/B7pP7H07yI07VHrO68ZJvy8fxMfvGVVTfeoGcIqPlcXUXjGLti/rqM8aGccy4TbRSdiaVCXHox5OJrYS+Ez1cfUcE2W8TvvitZWKsMdamXHZF5OGQiYCyQxaTFy4ZUoRdFOaCBSe3DtWHX9TKhMmU4I3AEIMhgaAEIOhASDEYCLHAPy9+0HO3a580rcuwM1Av8LuWWthDEtp592v3rvsMkzPbXl2Gcgr925T33kpthnH9glfViudTQzkQeW24vcUs/jWOlv58qUx9OvlNKH2hNoknPNwk7Cc8tOUVFOLCmX0Zf0E+vWuLAXW0oCubAcOiwEIl9kRE4Gm0g5cy8/XqVVGjJ9bY6tQyEfNymlCx2lekzcAQgyGBoAQg6EBIMRgIscADvzJepB/s+7zwdkRdmTVg1eD/KZlO0BueXB3cB50x0HXthk/y82pOoBcL+rq+/AZ9ZZf5yhuEZpYvADkcjP6kTGt1dgel7t0kJa4FgNwsWzYTmCbcVNcPf+AK+oAUvhaz5UxjcnrAMJiALXS5q43eS4/rEbAsjBGEKvhi9fy82cE8fhV25s5TgzgDYAQg6EBIMRgIrsAf/JBXNjxw7E5wXllEhd2LPgWptF++aaTQV52UHUS/mgM037tW3BCkKct2yz3Ymqv4yn8nlirWurpjYyCLt8hym5bMLWnEx8Lt4ttWlpTLhKVpcBNcZUGtEUpsNuA3YyeLAXWKMuuwxAXwJGLQcTdVl8OGrNkqa+YCizShDARaApTgV1/FqbuCG8AhJgMDQAhBkMDQIjBRI4BXN2MU33W3nlDcLZPQ7+967+eRtlbg1/apVJy/7JnKeiaduwBOdahtv+c2b0bdIN9mNrzOtXEIO/QAOgm5qAfnGrGeuWiryYCJUZFPkjUpWZjKg347MRC0MkW32YtBmBVMAbgiS1IVlU7sDYRqCoNKCf5aPIUNgNJqqYPT6EU2LXke7kcdLbDGwAhBkMDQIjB0AAQYjCRYwAXvfgOkBfd+VxwHrnk90DnNONk38wj20Eeeotq6z3yFPqNDeM78YtXqxjBW9vvAtXX+7EVd2yd2iScfhr97YkO9DnnN2OL8qinRnclsDrZsuNYspt1lF9/tIzP4CfxtQ0xFWuwy1h74MVl3erktahVdQBT2A4sS7X1d1bl+X1ZQzB5u/BMlvoyRvDawBsAIQZDA0CIwdAAEGIwkWMA9q0dIDttykdruuc50B26CvP+7f+M24MPXaB84fk/xfx2rL0N5CO9qt32rDSOJfvXIRxVPtq1ODinRe7e68Ax4V0NOfwezW9OjImNv8Kvb3RUzcBgEUeN+amYeG1IHUBCjPmqhMUAorcD10rde57ux4f3AkimMhY8rFdA+vjuFPoKXq+2Xf8EjEvwBkCIwdAAEGIwkV2AxM9xiefWO88Izis/mQNd+5X7QI49iC2/1532YHB+8G+wVbh00mKQc73qvtcVx+u2V8By3nGtKrczha227R04Iag7gws/+7XJPknpAtThpqCMdhXMlXArkpcU7cBaGtByZSkw3mXDXABZCmx5YpKPdoWWN/OYdIc8vR04PA04panANaYJGUutP8vr6FrwBkCIwdAAEGIwNACEGIzt+5zVRIip8AZAiMHQABBiMDQAhBgMDQAhBkMDQIjB0AAQYjA0AIQYDA0AIQZDA0CIwfw/7TvfjA96458AAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
     }
    ],
    "source": [
-    "# create position embedding\n",
-    "class PositionEmbedding(nn.Module):\n",
-    "    def __init__(self, patch_size, in_channels, out_channels):\n",
-    "        super(PositionEmbedding, self).__init__()\n",
-    "        self.patch_size = patch_size\n",
-    "        self.in_channels = in_channels\n",
-    "        self.out_channels = out_channels\n",
-    "        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=patch_size, stride=patch_size)\n",
-    "        self.norm = nn.LayerNorm(out_channels)\n",
-    "        \n",
-    "    def get_patches(self, x, patch_size=8):\n",
-    "        # get the patches\n",
-    "        patches = x.unfold(2, patch_size, patch_size).unfold(3, patch_size, patch_size)\n",
-    "\n",
-    "        return patches\n",
-    "\n",
-    "    # define cosinusoidal position embedding\n",
-    "    def get_cosine_position_embedding(self, x, patch_size=8):\n",
-    "        # get the patches\n",
-    "        patches = self.get_patches(x, patch_size=patch_size)\n",
-    "\n",
-    "        # print shape of patches\n",
-    "        print('patches shape inside : ', patches.shape)\n",
-    "\n",
-    "        # get the batch size\n",
-    "        batch_size = patches.shape[0]\n",
-    "        # get the number of patches\n",
-    "        no_patches = (32 / patch_size) ** 2\n",
-    "        # get the patch size\n",
-    "        patch_size = patch_size\n",
-    "        # get the position embedding\n",
-    "        pos_embedding = torch.arange(0, no_patches).unsqueeze(1) / (10000 ** (torch.arange(0, self.out_channels, 2) / self.out_channels))\n",
+    "import math\n",
+    "import torch\n",
     "\n",
-    "        # print self.out_channels\n",
-    "        print('self.out_channels inside : ', self.out_channels)\n",
     "\n",
-    "        # print shape\n",
-    "        print('pos_embedding inside : ', pos_embedding.shape)\n",
     "\n",
-    "        # get the sine and cosine embedding\n",
-    "        pos_embedding = torch.cat([torch.sin(pos_embedding), torch.cos(pos_embedding)], dim=1).reshape(1, no_patches, self.out_channels).permute(0, 2, 1)\n",
-    "        # expand the position embedding\n",
-    "        pos_embedding = pos_embedding.expand(batch_size, -1, -1)\n",
+    "def sinusoidal_encoding_table(n_position, d_hid, padding_idx=None):\n",
+    "    '''Generate sinusoidal position encoding table'''\n",
+    "    encoding_table = torch.zeros(n_position, d_hid)\n",
+    "    position = torch.arange(0, n_position).unsqueeze(1)\n",
+    "    div_term = torch.exp(torch.arange(0, d_hid, 2) * -(math.log(10000.0) / d_hid))\n",
+    "    encoding_table[:, 0::2] = torch.sin(position * div_term)\n",
+    "    encoding_table[:, 1::2] = torch.cos(position * div_term)\n",
+    "    if padding_idx is not None:\n",
+    "        encoding_table[padding_idx] = 0.\n",
+    "    return encoding_table\n",
     "\n",
-    "        print('pos_embedding inside before return : ', pos_embedding.shape)\n",
-    "        \n",
-    "        return pos_embedding\n",
-    "        \n",
-    "    def forward(self, x):\n",
-    "        # get the patches\n",
-    "        patches = self.get_patches(x, patch_size=self.patch_size)\n",
-    "        # flatten the patches\n",
-    "        patches = patches.reshape(-1, self.in_channels, self.patch_size, self.patch_size)\n",
-    "        # get the embedding\n",
-    "        embedding = self.conv(patches)\n",
-    "        # flatten the embedding\n",
-    "        embedding = embedding.reshape(-1, self.out_channels)\n",
-    "        # normalize the embedding\n",
-    "        embedding = self.norm(embedding)\n",
-    "        # add the position embedding\n",
-    "        pos_embedding = self.get_cosine_position_embedding(x, patch_size=self.patch_size)\n",
+    "seq_len = 100\n",
+    "embedding_dim = 64\n",
     "\n",
-    "        print('pos embedding: ', pos_embedding.shape)\n",
-    "        return embedding + pos_embedding\n",
+    "pos_encoding = sinusoidal_encoding_table(seq_len, embedding_dim)\n",
     "\n",
+    "print(pos_encoding)\n",
     "\n",
-    "# use the position embedding\n",
-    "patch_size = 16\n",
-    "pos_embedding = PositionEmbedding(patch_size=patch_size, in_channels=3, out_channels=8)\n",
+    "# plot the position encoding\n",
+    "fig, ax = plt.subplots(1, 1)\n",
+    "ax.imshow(pos_encoding.detach().numpy())\n",
+    "ax.axis('off')\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor([[[ 0.0000,  0.0000,  0.0000,  ...,  1.0000,  1.0000,  1.0000],\n",
+      "         [ 0.8415,  0.6816,  0.5332,  ...,  1.0000,  1.0000,  1.0000],\n",
+      "         [ 0.9093,  0.9975,  0.9021,  ...,  1.0000,  1.0000,  1.0000],\n",
+      "         ...,\n",
+      "         [ 0.3796, -0.4645, -0.9086,  ...,  0.9997,  0.9999,  0.9999],\n",
+      "         [-0.5734, -0.9435, -0.9914,  ...,  0.9997,  0.9998,  0.9999],\n",
+      "         [-0.9992, -0.9163, -0.7687,  ...,  0.9997,  0.9998,  0.9999]]])\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(-0.5, 63.5, 99.5, -0.5)"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAGFCAYAAAASDy0NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9KElEQVR4nO2deZxcVZn371JLd/W+Jp10IPtCBBKiSJCAAgKK6MyoqMAwjqKIo7ihvK/juIyODjgC6ujoDChuML6KG4qCE0AFBLKAQEhIIGtn6yVdne6uru3e+/7hZ+45v+dat6vJ1p3z+/51nj619b31eeo8ux0EQWARQozEOdYfgBBy7KACIMRgqAAIMRgqAEIMhgqAEIOhAiDEYKgACDEYKgBCDCZR7QPnfPlLR/JzkGPEC2/5xrH+COQI4UzfMv5jjsLnIIRMUqgACDEYKgBCDMauthhozY4TUR6bE66fGp0Fe1uH20DuG60DOZdPh+tiAd0QfknoJF32bdzzUbT1fflfRWR8LTvuKhzH5VKvOesJkM9q3Azy4tQ+kNvdUrhucFzYq7HxXjra74tj4fV2bf72HGnoAyCExEIFQIjBVG0C7OzpArk7Ua/2yiOwtyY/A+R1o7NB3jzSGa73jjbC3lCuFuRCPhmuPWEeBNJcKGuy+K/sccwHePxEzAPJFDMXOtaivH8VXpiTl+wE+dXtG8P1qbU7YG+mi9+DJkddx4ydhL2kjeYDTYTDD00AQkgsVACEGAwVACEGU3Uq8Nn3fwDkr5/1/XB9UQYf25rpBXl6Igtye3J+uN6QRH/B9kQryAdc9eK5fAr2yiIM5evqrIw2pTDrLVvYnGC7+4HYEo/VX0fa/PKhk9wn0HTHYyA37FwG8ubz54Lcs7wpXL96Vgvsrax/HuQFyb5w3eoU8X0c/OpJn0DcdaN/4PDBK0mIwVABEGIwVACEGEzVPoDFNw6D/F7rinCt+wMsK+oTODWVF6/2vHU4yFnCJ6CtfUvYlGUUI6a7pxnvUi0Kn4DuUIj4FuJ8ApPQHxCsPAVk56EnQZ5lLQN5l6XSvH9rLbLEgxGVKgL+gD+DN6RBXnPtuiXEvfQCzFWgT+DFwytHiMFQARBiMFWbAP4LmPY5+46Tw/UnO14PezOWfg/kU1I1IC9Ijobr4ZrdsJfz8Vg/5qkU0pInj4Ii1Kel+wZyLxC6Tp7qte1I2nAkCqg9Wb5PnEkwCUOE+z+G4bnuD3SD7K/ZCHJnqzIZ9tRjGPDxWqwYbXLHwnXGLsBeMplFOfAqfkZHHPFl2jB58fAEQIjBUAEQYjBUAIQYTNU+gH3vXgHytK+pFNL8yS+Hve91rQT5E52PgNzmqJLfuckDsNebagB5sEbFFEfKadgrSp+Ap/SZL4xx36/sL/jzH7SliOXZdpzNWTlEOC6TwCfw8Eu/DfLKSz8M8syb94Lc8CfVIahp+kzY29HWDvIzGZXmPS05BHuNDoaGa+wxkF2tXtux0D8QTRtWj2VIcGLwahFiMFQAhBgMFQAhBlO1D+Dt77kH5N/+eHG4nrk6C3s/XXEqyBc3/QnkV9Yqm22aizroBOET2J9W5acHSthdeLSEOQOFkvp3ymW0EwNX+AA8YYDr2aXS5p+ACyCyrT13Qq3FjhL/M4Z2++lvfArk3fdjum/5iU3hunlLB+yNzML7saVd7c+t64e9jsRBkBtEuXCNq25IUvRv88UNcWTaN6kangAIMRgqAEIMhgqAEIOp2gfw/uatIN/+pteGaz0nwLIsK7MO8wJ+tRh9AsvTKi+g3sbY/owElh1PS6j48b5kE+wNJIVPIKls0GK5co6AZVmW7Yq2XzFThSJ5AWDYW4KplRfw8e9eCfLjV98E8spzRV7An9SHrNmKJb71CzAvYKBb5XRsbUFfw5w0PrfTxfve4Ki8AOkDcMTvlqPfH5YKTwheHUIMhgqAEIOp2gS4+LlLQJ77ZjV1ZOw2PMZ3rsc0z/85G0NJl7c8Gq6XpVEHtQqVNF0zAXYmc7DXmMT3OZhQZcc5FyfROI4IJTnijRztGCmnBsWFBSdwTB+3e9AxYPZXN4DcdxV26qk/dz/Izk9V2x9v1x7Ya+iZBvJQnzLJerqaYW9vA8ozk4MgN/mqfLhGmGvSJLAYBnzR8ARAiMFQARBiMFQAhBhM1T6A/L/hBJ87v/nlcP3m098He6lndoE8vGUeyOsWqtZRS1PYEqzBwXTSNle1D2sXIcLGZDPINYlSuE66+DolF/9VzxFhQF0eJxVYDwvGlwpbVlz7sMmAXV8P8hUbMSz4uYU/A/lTy64K13XPb4O92t2jKPc1h+u+g/g++1oxpJtNYyvp6a5KFS4GJdhLC+eJDz4B2fqNYcE4eDUIMRgqAEIMhgqAEIOp2geQvmcNyCXNtt11HuYBzH4Q0zybNs8H+ZGzlHxJ/Quw1+liem+rM6LWiRHYa0hgHkAmoUpKUwlsRV5w0Ra0Hdn2S60DsWfJ9mFHi6MwVWjTR3CcT9ud+L+e9wVsx3XNaeo3Y949aLc7+wZAru1Vdn5fFu/Hvjy2fhuoQx/BaKDyOJqED8AP8GJ42sVxJp+bZVLDEwAhBkMFQIjBVG0CFC5+GchXblGVX0vPwWGfYxk8GrZswakw6/er6TN7puFH6BRZnQ3ama7ZwVTgBleaAOqomHbx6DoqUoFjU4MjYUBx/rZjqgGnWGrwXX/1ZZA//pkLQN72z2h2tS3vDdf2DEz9lanBmT6VVuxm8T735kT35yY0/YZ91Tk6LzoIl6y41GCmBU8EngAIMRgqAEIMhgqAEIOp2geQuQ5Tdvf9UKXz3v5/boa96xe+A+S06ByT3d0Vrjcv6YS9k1NYFpqxVThITpPRp89almXVuroPAMtaExEfgAgDanIkYzdGHneK0CQo+Y2j3cUQW5BHf83n910I8lWzHw7X35/3OthLidTgdJ+6X6mssPlztSD3l0QY0Feh5ZL4nfICvLeedg/8SPcgOVGaU4R0eAUIMRgqAEIMhgqAEIOp2gdw98JfgnzJXco2POWTmObZvxxLPdvuwHTfzC6VfrphrBv2XpvBFlS1tirrbRI+gAbhA6hLKPs1KfIAEu44PgDddI+L+08UeOrk6xi8avUHQJ55MX4lHvgDvvG/vOW+cP3vC7DkuvO3aG+7B1QOQVr4AA6OYPr4gSLmjgz76juVD/B1Sxb6AOLwxYVyYsc8mQdPAIQYDBUAIQZTtQnw5UGs6POzqlvv4wUMJQ2swON267cxtFTfo45lGw52wV62RXSl1ar66kQor0GYBBltwGSNCG9JkyBqAmhdfmS3oEiObkwq8DEY7nEoLLkRh3Tu+Bwe67tvxaN552UqZXdoIV7TrnpM57Wy6rXTWUwb9kexa3NWmgCelgoc4GNLQUHI+iBRceTniT8WngAIMRgqAEIMhgqAEIOp2gfwvW9cBHLTBcrG/vR2HPy47BQcJCrLgxt6lA33wgF8bn832nvd2ifMiHCcPkDSsiwr4+o+APQlJMcpB7Z1WaaIRuz8I9Pp91iUB/tbd4L83dP+API//fHVIG8rqdDetAX9+GKdbfjaWnlwTRb9Be4wfvWyeUwNHtJ8ADkfQ4ZFURauh/pkKjDLg+PhCYAQg6ECIMRgqAAIMZiqfQDTvrkW5OdvXxqukw/Mhr2br7wN5K/MvBjk1G6VQ5Dtb4G93R6mEZ8cKHuvxkZ7rs4R+QVai7C0I3wAMjVY+gD00L4t8wAsxK6w/kschc6+h8K+d68A+ZTUIyB7Q5gncOeQevxFMzbC3kNdLwfZ0cqDU1mRlzGCvp7hPNr5Q+WYPIC48mBxP+LKgz1ODeIJgBCToQIgxGCoAAgxmKp9AM6C2SDfesZ3wvXnv44TZc9+N07xvWEexodrH1NtxJN92BJse7ED5HKtyilISh+AjXalXgsgW4K5trD3RL4/1AbExf3/4gMUkRZhQUzdwKGUBx8m3v6ee0D+TN8ykN2FONn5jueVrf6t5bfD3m9mnQ1yo/46Q1i3kRzB3JBcHmsQDpZVDcioyAMoBcIHAGvZMhzRcwZYGswTACFGQwVAiMFUbQJsuh67tr6yVh2pb1i3CfbSNr7s4EI8iKXuU6Glml48hm3PY2pwvnFzuK638SioH/kty7JqbCXXir2UCAO6ManBcYNDLeuYnNSjHKay4/c3Y9r20lvfB3LmXHxhf51645ecjnsj3fh70pRQ3wNneBT2kiMY/h0ZE2HBktYRyJflwGgK6ncybnCoZbE8WMITACEGQwVAiMFQARBiMFX7AH7/qq+AfNOBU8J14KE9/WQRQ3AHReuoab6Sa/vQRts2iiHDYe2xTQnUVzUi5FanhwFFKnBKyNIHEDfwN9olWJdlDa98rraehKnAFz93Cchz/xunOG3/LPpdOm9TKboZB0N3o92i7ZpWBh6MYglvckSkW+fRrh8pq/eNlAOLEt+S9lLesRixPIXhCYAQg6ECIMRgqAAIMZiqfQD7PbT3/uNXajLQvGUjsHdbP+qVrvloV9pJ9VqZPrTNe4abQR7ylb2HM4Qsq0YE6PXy4KgPQOQB2DIVWMsDiNj8VmVZPvYItQg7UqZt/oszQE5vXAPyJ0/eAfL3nj4zXPd7GNvPdOP3wG5SycD+wAHYS42KPIwcfmeGi7oPAL97Mi/Acxncf7HwBECIwVABEGIwVZsAb/nJtSDP/2+VzttzIXbx2bxhKcjvOO1hkB9pVsNB030YHto9hNNlBnzsFquTFGfzGq06MONit6CENAEc2Q1GrwasPDXIsg4hmjcJpwalf41Hfns53rs31q8D+du7Vaff9YVm2Dtl2h6QB9vU1Cd/917YSw6LDk1jGNobLapj/7CHw2dLkTCg+h2THYCisroJDjsG8wRAiMlQARBiMFQAhBhM1T6ARTfhBJmyZgs2f2Euvug900F+zTlPgfzwtGXh2h3A7kGlIXxuX1mFkrwgC3uy7Fj3AdTYGAZMO+N1BdY6xTiypFQQ1xV4ikWkChe/DOSeV+I1HfKxk4+dUCG4X2aXwd4rWp4H+a529b1I+uL6j2I3J3cM033zJfU5xjwM++VFWNDTLroXjBPCjcHELsHH/39ICKkIFQAhBkMFQIjBVO0DCHI4idc9aWG4/tyCH8Pe59dhl+AlojVrvrshXNc+th/2ktmZIO8rqxyDsjWAn8GWeQDK7k87YhJNJBW4cpdg2QIsvktwvJGpdwm2I+N/I4+WT4597cNB/Ud7QH5fB077uWXgDJCduSeE6/t3ot1+xXKcKvT9DnXjZXdeZwRbtiXw62UVCuoZI57wD8hJQVoegPTXxHUJ9mW7sKnmwDkM8ARAiMFQARBiMFWbANuuXQJy4Krjk94h2LIs68ZnsYJMdo4Z7lZvm16NFWWpLB7D9habw3UpwANerY2vm9bCgCk7viOQ7BKshwFl6m/EJIDNmL2jyYvsPPTzBb8CWYa+5vzi3SB3LVf7hc34zy94GZpduU712EYH026dHIYXXWEClIvq8TIMKLsC62FA/CZGuwRPmvs1SeAJgBCDoQIgxGCoAAgxmKp9ADdccTvI63JzwvULJewE4w0Ogiw7x4x2K0OsrSQm+GTRZtubV2HAnPAB1Duov1JaaK9GDA6VHYISduXy4GhHoMo+ATkMdKoZmTcNLgD5ZbXbQJ6xGv+f3pcpuRkHQlktLg78zHdo6dUpEQgcQx9AYkxMIMqrr2auHN8RSPcJeOPcDj1VmFOCeAIgxGioAAgxGCoAQgymah/AubXY1fXijLL7r9r1GthzGkC01hZaQc7PQrtfp0b4APbn1YvlZUxXkNZsuhqZCmzHlwO7MeXA8ZOCxjEk40qHJ0FLsDv+40KQv7ESA/KL/7Ad5NS7VXuuxN14XyXFDuV3sevQPxDksWVbUvgA7KL6bYr4ACKpwOprLPMAZCowQXgCIMRgqAAIMRgqAEIMpmofwMo17wD59mW3h+uHfv8S2Ju3CNt8/XIQbf4ZM5U/QZ8SZFmWlc6ird6fU23Ch/34Ns4pLUAv8wAiPgBRDuzE1gLE5JPHlgpbh62kN1JJfJhM22n/uRbkpm2nglzeh+XaV52wK1zfsQ1LhWW+R12HavluZ9AHICcFJfKiRVtB/TaNldHmL4g8gKKeBzDF8jCONTwBEGIwVACEGEzVJkDXjXjsuuLD7wzX3avxuN13GsYBd+2cD/KrZz8XrjfX42OTQ2gu7BtVk4GGRTdYiT4pKCnKgWVYMG5SUHzY7/jCWTgH5Jr7ngDZPXEWyK/NqK4/39s/D/a2inDd3DbVwanciNOjgr378H3GRFhWKwfOl/FrKoeF6pOCPGErya4/+qQgX9xpEycF8QRAiMFQARBiMFQAhBhM1T4A+1Gc7tN+5+nhuvaP2El2+18vBjm5pRHkM0/eEq63NL8SP9AQpqIWRpWPIOtjKMmyRDqp1s5qvDBgMiYM6IzTEkwPC0aifBOZFDQJUoM3fQynMS/+GKb39p/TDbJe8huIUu41YzghakmjsvOfaJoGe7YnwrI5lJ2CsvMLJfyaRsuB9VRgvKhy2g8nBSHH339ECKkaKgBCDIYKgBCDqdoHkL0C0z5bf6TixV4BbfFXnLIZ5Of+C1uKL0urycLfaq2HPXd/Ft94pEV9Bg99AF6A/gJH02dRm1+WA1dODXZiJgf/+Q/Wi0K2D4tMCjoGPPyqr4B88d98FOTsGXhvNxZVeq/TgDkcDw+2gHxR29Ph+rFmnEKcFqXdbh7zNlztbYsiD6DgozyRVGC9PFhOKzIRngAIMRgqAEIMpmoT4OUfwKqx5+9Wx3FXGxhpWZZ11TQcFvovW2aD3O2qw1ehA4/1tVv3gJwYUTpqwENzoWz1gqwPC02J3jByWKgbU/Enu8XGTwY6jF2B44aFHiFzYZcYvDn9UpzqdGXnsyD/IPvycG3PwNDeU/vxtT44475wXWjCNFt8pGXZY+L+aCZAoYTPjU4KSmhr/E2Tw0LjMHFYKE8AhBgMFQAhBkMFQIjBVO0D+MqMNSCf/lfXhOtSPdpKK2swdJTahra6Pi0414EfoWYMQ3uJYfXag2VMW5XTgtO2sg1lObCcFiwnAyViJgNNuWnBE0gxvuwn7wf5kUu/BHLGRvt7xcNqWvD0BTWwl+vBN571UvU9KDTHO1bsAqYVO5pYFj6AoggDlmLCgLJLMEF4AiDEYKgACDEYKgBCDKZqH8DVPStBTr1VdYsdG0Nb0BF6xduPPgDddh/rEDZbUZTxao1m+0uYB1AS5Zq1ml0py33HTQ3WU4HH7QqsdxDGranWPGzRTTtBdi7FfXkvk+vVPRhciI+t60G51VXRfukDsBMYy7fz6ANwC+pK+iX8DDIPoKiXAwfxv2m+loIspwaZOC2YJwBCDIYKgBCDqdoEeOoWHBhxxxf+LVyvKcyEvccLeJYKyhiC2+upUF++UxyafXFUH1H7B4qYNlwSRzi9Y0tS7MmwoBwO6thxYUD8iLEnxcOZGnwUCHIYdr1q69+A/OoOTAXuXKdCe9vehOG56b/D3xM9LFtsFtc0hcf4oChNAE0QJkDRw/fVw4BF0dnXm1q346jDEwAhBkMFQIjBUAEQYjBV+wAa73wM5I4b1VPfWDcIe1ftOgdkBxvHWM8W28J1qQPDfpKU7gMoYCpwPqgcdEsKWy81bocg5QOIKxWOMBGbf7yOwccghrjtWuzWVPMj3L/pjBkgL35GxfpmX4/h38QPsKOwTrkZfTB2jSgIFuHfhBYGtIUPIB8pB1Z2vwwDylRgGfozHZ4ACDEYKgBCDIYKgBCDqdoHEJyJeQDv71EpoTfM/A3sPfg0TgZa0p0F+Y+jalpwc/sI7NkJ/EjJEWXFZfO1sDfqV9Zfcs6rzAOQPoD4yUAT8AlI9MdOgi7AkhuuuB3kb65aBXJyBKf9lPepFPDXdg3D3gN7TwF5xM+H61QzlojbNeg/8LNDILtF3QeA163gyS7Byicgy4HH6xJsOjwBEGIwVACEGAwVACEGU7UPoO9jeZD3/vjkcP2ly9EWbFmPLzs6HyfGPNo/J1zPa+3Hx6ZSICdHlO2ezWPsuBBIS197nkjgT1ovvhw4gr4/kWnAhxHpTpiIW0Ln3NoDIH+9fwDkjgfxmvua7X5h/aOw90Av1oTs99S9a29CX09Qi68b9MvJQNr9KIoWYDG1AHqLcMuyLC+QLcK0MuNIlkDl79PxCk8AhBgMFQAhBlO1CfDwiu+C/OZr3xiu/9/c02Fv4To87u1dhbnAo3s7wvUVJz8Oe2sy0/EDjqgy0bE8poCOBpXHO7riLC5TgV17AuXAFd9l6rNyzTtAbj8PS65T9+JEKOclKsS7KClKb4fRFNxebgrXsxqysDdUh1OFLE+EZTUTQERwY8uBDyXsJ00CxwCTgCcAQgyGCoAQg6ECIMRgqvYB/DrXDnJ5pyoLnbkaS0bdTThhdvgdmBqc7FGhpKWnYyvZtfXzQHZyygdQzqMvYdjH1GDLUummSVt0sx0nFTgZEwaMpgbra9yLZPvaFdZ/iWNQHjzjBvwKbLkG7eDF6ztAHliuQrp6yy/LsixLlGdvKqjvxYkZDDc+UTcLZFv4ACAMKFKBi2W0zfNaKrAMA/oyNVjvJD1Bd4H+XNc+Pn47j4//ghDyoqACIMRgqAAIMZiqfQD/9J0rQJ55tmon3fjQNtjzDh4Eee78fSAf+Gl3uF6YxKlBfiPGoZ2DuXAdjDXD3kEfS0q9QH0mOdHGjbQJr1wOnBA5ArHlwFM9SeCxp0H8xvefAfmzqzBPoH+F+t/3lkUpdxrTezeMqtTgU+p2wd6aevQfJIX/wCmp++Ngx/BIKnBZKwv3REuwuLwA2R6sclbJ8QtPAIQYDBUAIQZTtQkw+6t4NNx4owrtLXwPHuPd5iaQL5yOz/1Zj0r3nZHA8Fy5CUN7yf0qfOTmUF8Ne/hY39IeK6sBI0d+MTzUUfuH1AFoAgTyfSIlfvBo+eTD8hmyV5wB8gWZJ0C++nx83yVL1FH+sQKmbbstzSBvzCoT7fUt+LqlejzGy+O3k1ffC0ekApdFGLAAYUDc8wPZIah6fDk8dMrbe1F4AiDEYKgACDEYKgBCDKZqH4Dd1AjyDef+MFx/ZzZOAip1YQeg8+vvBfm+npXhusXBUF6xWYSHxlQnIjePNtiwCAPq5ZwyDJi0S0IWPgHN3oumAoMoUoEtsSls9cgDJhcrP7gG5K9lMUX30pVYrr2y/vlwfffAMtjzp7eBvLtf3csZC7Drb7EO7w8Gfy3L1sOAYnhUsYzPLWmhP8+qPgxIeAIgxGioAAgxGCoAQgymah/Axuu6Qb60Xtl0//ZK7AZbqkO7S7aOcvepacJJW5R2NqOcKao8UDeHrzvioQ+gFCi7MW3HpwKnRHmwE1MOfLTyAo4YMWXGt3Rhy6/5d74H5Ecu/RLIGe1+Xb8DpwZ1zcRU4HKfugcdIphfqrcQ6SvRpgVLH4Dn4b0t+uprLPMApA9A9gE2HZ4ACDEYKgBCDKZqE+Anr/8yyD8YVt1eBs7DoSF+CY9hsnOMP6BSdqFDi2VZhWZxZNOOggl8G2uojKnAenWXTNtMyi7A4jCohwXH7wikVwNW7hZkWUelqc8hcbUWkrUsy5p/J1b4tb1Vdl3S2FIH4kGMIFo1fepiNDhiuEcDXijbxe+MXVImgz4o1LIsKxBhQH1YqKwG9IPKv3G+qECU1YEy/Hs8whMAIQZDBUCIwVABEGIwVfsAWl2MxXxitZoM9OFzMNX3qREMGcrOMX5eGfMjQQH2is3ijX1lm7tjuHWwLMOAWtdWZ7wwYOWOQI41xcN+E+Cpm08FuXEtDvy8axTTuhen9ofr5s34WtmFKNfvUtet1sahryV0H1h2QnwVNR+ADAMGZTTOy5qdP14Y0NNvpQE2/njwBECIwVABEGIwVACEGEzVPoBzfvtBkOffoVJ03/n6LbD3VO1WkB/J4+QgPVi+38N4fLGpcrJmYgxt8eGS9AFUttWlpou0BIM8ADk5OGYyUMV3/N8HxBidcVOELOuoJBE03ok2v7toPsif3XASyBecsClcN23JwV7feWh/1z6h8j/kJJ1yvbjPScwVCWJSgS2RB1D09FRg/ErL8mCC8OoQYjBUAIQYDBUAIQZTtQ9gyY1ZkL3NL4RrX+TVr0ijLfjNXow1O1op6NZSK75ui+gBrZEUPoCD0gdQ8ZmWlRT2dVweQMIRk4HEa8XmBUT2Jnew2T9rGcg7z8XmXDWr8fE/O0Pdy8Xb9sDeiTPwfiR68d7qePV4/e2UaAxe1moBSsIH44lpwb76vkXKf2VtgP4ZJn2lxpGHJwBCDIYKgBCDqdoE8LfjcMfEiar289ahQdi7phnDgg9vnwPygnbVTejZPHYTqm0SNb+OOt4lxvBoPlrC9NJ8zLQcV8iyHNiN6QgUx4TShOVjD9N0n0Nh4GOYX/36WU+C/PTlmN/rpdrDdbm3H/ZOb8dj/dMDqkNQzscJn249mnq2CAP6oyrE6JTFdROpwCXNBBg3FXiSm2RHG54ACDEYKgBCDIYKgBCDqdoHsPc9K0DWG/J+ZV077J105m6Qnc1Y+1nqUnbYhhFME57WNAyynVQfUfoARorSByAtfYWcFizDgJAKHGkNNYHy4PHSe48QujthIm6Jh077HsiyS/PrNuI/0NmpxXB9vIavqMf64GcOTAvXIwEGaevqhK8njffSyio/kSPCgI4sB/a1cmBfpALHtASTyHB21HN0/METACEGQwVAiMFQARBiMFX7AK56969A1tt+PfW1U2Dv1rlng9y8GW240W7VanpTthP25jVhbLkvpWxDdwxtzrEixo5LMfZepBzYkm3C46YDCx8ACFM7nfSXuQ6Q9ZZflmVZbkMDys/sUEJzE+wtTfWC7A+rVnADIn23tQ5LiQPhAwi0MnFHtAW3ZR6AVzkPwJfTgjVnic9UYJ4ACDEZKgBCDKZqE+C9zdtALjSpdN83/w6rvtasnAfywufxuLf3FSqUNNqPx8jzu54DuT89PVy7eUwfLRTw4+eDyv+Oa8WHATEVWFQDTvFjfhyf/s7lILsvzYLcvhKrA1P3qmGizksWw153AoeD+mMq1LfHQ1OioxY7RQ/XoiloaR2enZK4H3jrIAzoH8bUXxkWdI7DsCBPAIQYDBUAIQZDBUCIwVTtA7ho0xtA/sb8O8N1eQeWCretw8lA7rZ9II++Za4SetFunJPuA3ltrSoltscwnbRcRPt01NdfCx/riFTguK7A7jhhQF2W04ClvwAqfieaJqzvHyE3xOyvPgPy0EXYBbjnXPyQ8x9Sad0jC9B/I6dA66nCu0ptsDW95iC+b1p0jva01OyyTAXGh5Y8PRVYlANHOgIdnosqp1rLrsdThan5qQkhhwUqAEIMhgqAEIOp2gdQ/uJ0kK/4wN+F69bpGNNtX4f2ndc3AHKqW5WJOusxPjwriY8NMqru2C5gW6mggM/NB8oG9cTUYZkH4MoYr54HcCjlwFMMu6kR5MZfPgXyvA/hV6R34exwPbgQ7e2CKPnV27ltK2DK8cx0FuRnM+g/cLUpT05RdBCW5cATSQXWvgeenCRlYLcwngAIMRgqAEIMpmoTIPWbNSDnOleG6+wq7AjUeA+GlgLROWbp9L3hentfPezNcLEjkJ9RoT23fwj27CLqrxyEAdEscYSuS8pU4LhqwJhwUcQcOIRjZCBNjaPQNXjjdRiyXfxPaL59asY9IL91+UfD9egCNMl6ymh2ObXKfNuWw65Q57ZsArlci0d3XZKpwDIM6OkdgWRX4Al0BJKDQpIVHnc8wRMAIQZDBUCIwVABEGIwVfsA8pecDnLbzzaE6403Yllo/Y9GQXZF55gzW7aG696+ubDX5qId5tWrTjHOHrQ57QLayHoYUHZ7kV2B3Uior/py4GMSBoy4Aw7PlKGfveEWkN+19kMgz0v+HuSBFeraLJmHw0GfLmKo2GlQ/p1dIzg4dHoH+nO82pjfonJ8ObCnpQKXRSpwJCyoXSfxMuOif6ec4yRmyBMAIQZDBUCIwVABEGIwVfsAmq7bCXJptQrGXn7GH2FvfTu2dwpmYBroyzMPhOvf9K2CvXpRUlqqVx8xWcRUU1f4APRy4OiUF0T6APS8gKQt04Slv0CtZTlwBHju5LMbm0VQPXgrdmV+cAx/I0475YVw/ao2bN/2x5H5+FqtyvfTexDva6uLeRrlmsrXxi6htS7zAIpax2HZGdqLSQUmPAEQYjRUAIQYDBUAIQZTtQ/gFwt+A/JZr706XH+w7SbYe9vS94Gcb8OpLwuSY+oD9KMtmHHwsaV6paOCIuYBSB9AzlfPlaWecuqtE6kFqJwHIInLA4j1CUxC8/Oc334Q5P85/xaQ/37T34L88XmqNuDExCDsXdNzGcjJNpX/n8vi9W9zsG6gVBvnA0Cj3xY+AN+vnAfgHYV6iqkMTwCEGAwVACEGU7UJcMPAApCzl6mje5ODaZ69p9WC7GHjX6vN0fYHsfxUUtRNgDKe/cQp0hrx1OeITnXBf1WGAXEyUHxHIJ0p2R1IOxUvuTELW10Xogl24IEukM9eqsq1kzbe2J17sPPvjE51zd0sHsUbHJQ9aQLotpQXHwYMYsKAcR2B4g09M+AJgBCDoQIgxGCoAAgxmKp9AD/8+vkgf/9jt4Tru0awDHTktDGQ/aJo96RNUQmGsQWYnLhSqlc2WyDCQS5GBa2cp4UBx5n6Ekn31SzCaMfguMlAsiWYLB1W68noLQh29ID8oT3ngNy9Gu9P+v2aXS+m4SR70H+Q0zLAU8IHkBFh2TK6jSzbVft2WfoAxPSlmHLgyGSgmBZhvggdy++QcxxGFHkCIMRgqAAIMRgqAEIMpmofQOet60Be+gn11MvXvxb2LjnpaZCfH8Zy4CFf+Qj8AgbzxwI07Et613A5kVXkAYxqCQcl+VgHdZ3MA0hpqcHjtQWfkrH/Cuy5+jSQd/0E97ufXAvyOm1Kz3Q3B3v1OCTaGutURnMqi3tykrD0AViaD8ASPgDZEkzPAygLGz/SJhwmA4n3PA5t/PHgCYAQg6ECIMRgqjYB7EXYvfdTvflwnVqNXX/f/tGHQX60Fp+7raTpHRF6GfYx1FeuDyo+1i2gPObpXYHjkZrPiU0Frvxq43YEggdPvjPnu6++G+R7LkaTwHfxSt3Wd3a4flUTTvdp6MF7N7hUPbdhu5zMJI7mtcLMAhNApICLVGBLNwF8mfrL37g4eHUIMRgqAEIMhgqAEIOp2gew+XqM02z/+Vnhes4DfbB38j9iiCfjbAb5wZxWWuygLdjn40cq1VcOuTkiFXi0rFJRi0F8qM4V5rceFpxIKrAkYtVPpCuw3NblIxR5fE/TDpB/sQ1LeoNXLAP5vg3qfg0vxDLw2h7s7pToUPejJitqwgURH0BCfQ8iZeAiFdjy9TCgnARU2SfAcmCeAAgxGioAQgyGCoAQg6naB/DQOV8F+Z2f//tw7W16HvZkOu+cBNqKXxiaF66dOtiy9pQxpyCok0FfRULkAeQ0H8BE8wAm0hJMN82nelrwRZveALL9CvQB7DovA3LTerV+PDUb9hbtx2nBM9vVfU9kW2M/h18r8nuT2lfTEz4Z8ZWwY/MA0LESVw48Xgn58QhPAIQYDBUAIQZTtQmwQ5RreRu3hGu3GY/t94/hce/iDIaHntw/M1zPaMSuwNuL7SAn6zVzQuTdukWRClxW4cfSOKc5V8jQEWicwSBxTMgkkI89BkMsyl/Ebk47LsPfhIVLMEzof0Ldn0IL2m/eYBbkJc3q/9k2hN+fUiAq/KQJoIUBrbE8PlaW8WlP9fzqqwEJTwCEGA0VACEGQwVAiMFU7QO44i4c+Dl/hbLdc50YKvrP3Sgvm/sjkLO7G8P19BZ8n20F7B7UUKfsPygRtSzLzaOtni+rf6cYE+6xLMtypT9BDwNGusFOoCvwJEC6EuI+Yuo3a0D+9C3YJXhxei/ub7wgXDefuAjfV3R3Wlo3EK53DE+DvZwIFadrS/iZk1pp98go7MWGASfQBZj+AJ4ACDEaKgBCDIYKgBCDqdoHsOimbSA/95HZFR/r/+lEkP/QhXJml3rbUgf6C3bkMIegPaM6z+olopZlWW4BfQDDJe11J6jboBw4MjWo+nLgCHaF9SQhf8npIF/Z+KR4BJZ2e4OD4bppC+Z3BKK0e3FapQb/amQx7OV8jPtnakSLZ0gFjp8MpPsAZB6AnAwk5TjkhOlo9sjUhycAQgyGCoAQg6ECIMRgqvYBBEWM237gNb8O108Oz4K9p//jZJB/vmIZyPU9yoYb68CJsruGm0Ge19QfrvuSaI86RbTRiloeQGkcWy/SFtyqPg9AJ74F2OSn5TrM9b9rpBHkBalekN1GbX875ghY9VgbMDsxFK6DHE4RygpbvVH4AALNBxCIcmA7xgcgy4FlLYCv3XlPJEz4LAcmhJgEFQAhBlO1CbD1WgzjvL/l/nC9rX4D7L33iW6Q1207AeQ5Peq4N3ASdgvKDdWDfNa0reG6P4UhQqeAOaGlkjru5YP4f821KqcCR8KAQp6M6b8vlp8tuBfkOXe/C+TlS7aD7C9S5cPB2mdgz503G+RprnaMF2nCB3y87y1pNBHGUlpnIjHo1RHlwPrtiR7rUT6U9F8fzMTjIyTIEwAhBkMFQIjBUAEQYjBV+wBuufw2kFePKRvovFq0263N20FMb8aJs6nd+8N1/hy0BYtigkxXKhuun0lh+yqniCmiZa0rcCniA5B2vfABaCEgafO7MeXAkY7BshRX2z8kz4E0XQ+TG+KGgQUgz7sTr+mG8+aDXK/dyva1+FqlrmZ8rKPurZzu0+dhuLElNQZyLq2+X4EvrnEkDKjWZU+0AItJBZb+gSM2fmkSwxMAIQZDBUCIwVABEGIwVfsAVtZkQV5+/z+E6/96xXdgz89jG+fmLaKssu9AuMx34iSaRBY/0nQtndRKY9qwXUS70i+q1tP5ANOGLUuUmwp0u989UrZg3PTfvyQfBX749fNB7vjdoyDPspaD/MJlysae1or93A7Oip8ArLOvhK3kW1PY9mtXSn0P7EgegMjLgHJgvIgyFTgOT06UnsD98ALpN5oav61T41MSQo4IVACEGEzVJsCZj10F8gl3qqPVJztxwGTTNOwU07hlGGRvSHUUrunA8I+zqwHkjoR6bCBMAKuEJkBQUvqsKCfCyCOaTAXW0zwn0BFoqqcFd966DmS3eybIzuObQF71BfWV6ZuFKd8j3fh7Ugi0Tr8iPrq31AxyexK/M74WBnTF0dwuSRNArWVHID+SGqxVA07GFk1HGZ4ACDEYKgBCDIYKgBCDqdoH0H0D6gp7/RPheuBk7Cxb8xIMuaX/tB1kT+sIe0LrIOz1ZTGtuM1RZaJBjegIlEW70Sormy4aBkQcofugK/B4k4FiwoQRn8AhmJmB7ms4QpOD7UVzQd51IZZcd92MXX+umvZIuP6/C98De6OzMI34gKe+B04aQ4T7CpgCvqJBTCFOqfsjA3l2ubIPQNr8kenAE+gK7In7HP+NmprwBECIwVABEGIwVACEGEzVPgBrDbZ/cjrVFN+Zq4dgb/d5mOY54/4DINtJFc9f3LwP9g4OYWy51VWxZL9W+AB6xUTZotJneR8fO17HVz32H8kDsOWEGO09p3gewObra0FeOe9ZkAdW4wTglenHw/XgAvz9qOvOgrzLU3a/XYvvs38MczramtGf42k+AGl72zIVWBN9kQosfQJ67F/umQhPAIQYDBUAIQZTtQkweOUZIJfq1PFp2n8+DnuF65bGvpbT2hyuT848BXtPZrH6rEGrqvJqxHDQshgaqVkE0Y5AiBvTEci14jsCxR37ZUcg3JTPO4QjaOSp2mtP4Gj70DlfBbnBwet25nkfrvjc/EKs+jxz2h6QNxW6wrVdh0Ng+8YwLNjsYjWgV1P5f7BFV2BH7wgUSQWOl+E9K+5EkSalM0XTinkCIMRgqAAIMRgqAEIMpmofwKprHwP5ueFp4dr7Fr7MmxY/AfKTre0gBx0q3XRxGu3GVBaHkKZt9drlWkzrTIlOs06pciqwb1UO5VmW7Aocn/qLXYFjX3bSs6OM4bkZCSzPrjm3D+Q/FtQ9OG3uTthb2fwCyBvGVEg3aEAfQHYUw4ANDvoTvFTMhY1JBfY8mQosugKzHBjgCYAQg6ECIMRgqAAIMZiqfQBf6loPcm+nitte/rL3wd7ftnwN5PUn4MTZwjRlD85K4FTYxBDagroPwKtFfSWnzeh5AAWRCiw7viZt9Cc4miEp8wAOhfi8gMP2Ni+aK+7Ce9d9Kpb//vOiX4B86/6zw/XrOjCH46T0bpDv7VP5IF4T+hrGRvCr12BjWnc5HZcH4AlZrYNx8gDk9GDT4QmAEIOhAiDEYKgACDGYqn0A79x5Fsg3zLw3XPech/bdkhTGfA8uxFbfxQZlh7U7YtrPQcwJ1yeslGqF/VZCu1HPA8j5ooX4OEAtwDjlwHFtwadaefCim7aB3PuaOSBf9Dls73bN06o8+DMX3gN7zQ7+nmwdUFOfOprF1KARfGyDg9cYbp90pEQmA6l1IGz8sqwFiMkDOHyen6kDTwCEGAwVACEGU7UJsPGWl4D8jmtU9955r8Rj5JCP6aRZ0TlGj9BlhAkQ5DAsqFMWJkAgjoJ6GDDaEUhO+0kIufLRPZIaPMWO+XEERUy97vgpTgLa+Wns1NO6Tl23Ey5GU08OxBztU/uNzRh2dUfwXtaIY76nhwHF69rluDAgbB1S2M8/hGGhUwWeAAgxGCoAQgyGCoAQg6naB9Dw34+CvKvtzHD97etuhr1fj84AubAIfQJ+Uc56UQSj6APQp/pGfQCVW4IVfPzX5JQXid72y5EtwWICRNIfEJ0MpIcMcWsyeBK2XrsY5Nmfx2nB/7r/fJA71otpTDGk+tQ9KGCjaCsZ8QGICT5a1NCWNde+8OdoLcICD3/Tyn7lyUBx7cH+Eno6+VQvA/9feAIgxGCoAAgxGCoAQgymah+AvwrbdXf9+PlwvewfMc3zIy+sAnnFHGwdtXtEGYQ5H+PQflGUhWrNmj3MOI4EfeN8ABOJ6co24BOZDjzVuOXy20D+lyffDvK9D+GFWvScmhy0vYz+miZhGNf0KbnQgu+bFK6EtC3KtyEVWPxOyTwA7WsQyMlA4kZ7Mb95nrytx4mdHwdPAIQYDBUAIQZTtQmQvR6r9Nr+Sg38fLKAFWM7nsQw4N+97scgr6+dHa77hQlg+Xi8y/nqXF+uER9KHOudkpLHPEwxlmFAmbYaNxkojqlW/SdZWZMFue9teKzv/i7aXd7Bg+H6wdx82FsqOgJletV1HDgVz9ON2EA4MlnHq9HCp674nYrpCCRvXXRSkHofdgXmCYAQo6ECIMRgqAAIMZiqfQC/W/Z9kC8+/x/C9Sd34uSfDmwgbK1683aQa7R43fZyvRXHsBbq8zLx9jaGATEFdDyrXteEcR2A/tK+jkz3jbUyD+e04BfJykexY/N3X/YtkD999QX4hBYVz7u7dxps5TswlFfbp25ISZQDJ0VHIOmTgY5ALj438Ct3BLJkGDCoHAaMCwmaAq8AIQZDBUCIwVABEGIwVfsAfjGK9t6Oy5ThlXpwLuzNfQInyp6QwNZRJUtNBP5Dbh6+kYP2XtbXpwNX7wPIe2iPFmUqsMDVTEVX5gzIdmIwHXgCeQATtfntCmvLOmy1xCfcgC+84meifHZwEN/2zFPD9cat+PXJJMRk535VBp5swvuRGhFdggV+Ws8DEOXjsgxcz+EVPoByIHxBMSXA7ApMCDEKKgBCDKZqE+Czt78N5G+887/C9RdvuwL2/K1Y/SfpdtVx8OlcN+w5NZjCe8BT5kNQg0c/iaulAhc9UQ0Y+0zxOuOEAY8ngrXPgPy5fuz+7C5EE23PirpwndmMr/VUE6aAnzgwFK5bmzCPOzlSuSuUZVmWn9buQcQEiAsDiteJhAFtbS/+92+8LlLHAzwBEGIwVACEGAwVACEGU7UP4IR/fxrkC96vYm43r3kO9nwxbeaFMnYFXphUduTG7HTYczP42F5PDRZ1MuXYz6iXA+eFDyDS7UU+F9bxHgMYDir2plp58OCVZ4B8x924X/8qlIdOU6XfM3+F13hPF6Z1B0O7wnV3Qx72hkdFmbEc6ZPSfQCyIxB+D3SXjS3DgJFyYP7m6fBqEGIwVACEGAwVACEGU7UPwGltBvmrgyeGazlhNjGjC+QHcwtAntOobMOdB7Bd7Nw6tOH2lNR+qkb4AETasJ4HUBA+gNI4abdxUelDyQuYkE9APvYQJttWy6prHwN545V4r7Z+CvMyLpitukHvvGU27NXNxXvpa5OeT8xgG+CNo20g692fLcuynLSS7YTI6Sjg9w1TgWHLCmLyAGRLMBNbhPEEQIjBUAEQYjBUAIQYTNU+gI3XYZ73c7+eGa7nnzoMe7kOLP/9ZS8aZhfVqSTyfB/Gg/0GnAy0t9gcrjM12H5cTo11inotgJwKO44PQOvlFbH5Y3LCx50OPAmQ/7r+Eb/Uhf3bLtyA//vHT+kBeXF6b7j+9E7M76/fhSOAAy1eP7d2APY25WaBnA/Qv5NMabLwAcjpwHotgO3JcmDmAcTBq0GIwVABEGIwVZsAd19yC8gfeZPqJrv7XDz6+dj8xdr+Apb8PjtThYBqekXKrigb7S2oVOBGYQLIMlGnrM62JWEClCag6yIdgSZgEsQSyRt+cS9zOHnnzrNAdk7Fo/lbGtaAnNACpl42C3v1PeL+aMxOYZcoawxTgwsiFTgNJkB8RyDb11KzRSqwJ1KBPc0E8CZoDvgQY4wvZ54q8ARAiMFQARBiMFQAhBhM1T6AjI12l95Kqv6z2DbqYA7t+Mxa9BH8cYWaKlsjTMNiMzoQ9uYaw3VLGifXjkkfQEnZaKWy7AYbb3C/2HLgcdOCJ2FYUGfjzdgCrO9teJ306cyWZVlp/f8R03ySe7Mg+2nV+XemOwR7QR59AKM+XqdMWqX7BsIHEMip0OXKXYHlfff1lmCW9A/Ix07ue3c44AmAEIOhAiDEYKgACDGYqn0A5973IZCXLFITYz6z4OewtzaHk4LuueOVID/aPydcZ/rQ3s6LKbKjOdU+bGY72pF5kSLqFDUfgMgDKE4kD+AQ4v6TILQ/IRp++CjIf/1PmNb9lQMvBfnldS+Ea7etFV+sF9N9nQbVIqzdRV9CkMecgeEA72V9Siv5dcUUIeEv0F02wlUVsesnGvs/3uHVIMRgqAAIMZiqTYAlXzwA8rbL1LDQCzJ4vJuXXAfyH7YsB3nz3o5wPacPu7sMnIQhxNyokltnjMLe3iR2oLFL6vxXLqNuKwXxqZtuzOE92hFIyZOx+m8i+Kvw3ny09dsgz/nlu0B+eomqCvVnY0dnOWXInTc7XDc7+FULRGffYR87D9Vpg0YLWhfpPz9Z3A+tI5C8HXFhwEPpAOTLgbFTNDWYJwBCDIYKgBCDoQIgxGCq9gEEO3eD/Po3qHDQthJ2fJ2XxAkx9s69ILu7TgrXyb5B2Cs0ow+gOKJsw/Ykvo+V6ADR1lKBvTKmFHvj6DpH7wgkwn6OXf1s4WiHIH2Ne4fkPZDm64t8sez16Ff5PWboWl33o227vqTSvqctwA/RuAY/hNeqvge1Ntr40geQ9bGLVGNKTYjqTbbjc0UY0NZ9AOJWxZUDS/zDdVGnEDwBEGIwVACEGAwVACEGU7UPYM81p4H8i86vhuv39lwIe1+YcR/IXhZTeOt6lK3lDB6EvVKTiO2PKhu0JYH2qp1EO9/WWkX5Htqc0TwAkTOqMZ7N706kHDiO8VqEHYW84oeW/wDkpb/DuP+i320HuZRRadzZhfgBm9KYspvvUP4c147/rcl6GOtvTKhU4d6EeG4guwJXngzkxZQHH8m0YH3a8Xj/+7Fk8n4yQsgRhwqAEIOp2gS45l1Y8behqMI4Dzx0Muz9/LUvgCyHeDb0qOf6B7H6rNwiBkT0qY/Y5oowYBI/vp4K7EdSgeW/iiaAngrsinNkJCx4HIWHfjrSCfLMO9B0Ku/dB3L7elUBOHg+PtZtRfMt11H118s6UMbQcWNChQH9JN7LiKWkmQCOGAwSHQ6qXkt2BJJ4ovPQlCv1rAKeAAgxGCoAQgyGCoAQg6naSHtn006Qlz/2jnDdvRrt6W8tPRPkpg603TM9Kpznj2Kn33QT5qK6W9VkoGYXw4CB8AFYZe1zCB9AMZDDQieS3lv5sVNhOGgcn739bSDPunctyG73TJD9LTvC9RmzMQw7MA0HyI51KqO5EGDJOORIW5bVL3wATboPIIX3zg1kKrB2f8ZJBY7rDn0o5cFTFZ4ACDEYKgBCDIYKgBCDqdoHcMGzfwNy053KZss8sgn2nl91Esj1J2KaZ2K36h5b9tF/0NmE/oKDo5oPwBmDvSCFNqgzovkTRDw4mgeAOJoujEwHjpQHT6BL8CT3CZzw70+DbDc1gDxwDk4LbvqBKgt/ffs22Pta90KQxzrU/z7sY+s3O4H3brCE96czo1LE/QTeS5nUjdOBcU9e/aOVCjxV4BUgxGCoAAgxGCoAQgymah+A/UVsv9X4kLIdvTG0zTvWoyE2tBB9AC1PbdFeGO277vosyFtGVOvpBkdMl0kLazCr/Al2WfoAqm/bPF45sL4v6wJibf5DCDMHMt9gnGnH1eK0Yf7+vgsw7n/gDLzmrfe2heszax6CvX/tFhOZO5Tdf0BcUlv4b/oLWFfw0vrKtQCSuJZgvmwJpv3mTTTu72n3OhnzuKkETwCEGAwVACEGU7UJkLwPU0TtxsZw7S6cB3tNT/aBvPVvp+F+XqX7OjXYBXh2BgdM7hhRx66MOAb7Kfz4rlfZBJCpwL4M9R2mrsDHDPh3hRkSYy5svK4L5AVLdoF8aedzIP/2JavCdXcC03dHMWJotXeoUN4+0fHHrsX7ni1iN6FGV31H/NQ4R3Vfm9QUMQEqdwSKvMxhMqumEjwBEGIwVACEGAwVACEGYweB7HtECDEFngAIMRgqAEIMhgqAEIOhAiDEYKgACDEYKgBCDIYKgBCDoQIgxGCoAAgxmP8PeMDMSeU4oWYAAAAASUVORK5CYII=",
+      "text/plain": [
+       "<Figure size 640x480 with 1 Axes>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "def get_pos_encoding(max_len, d_emb):\n",
+    "    pos = torch.arange(0, max_len).float().unsqueeze(1)\n",
+    "    i = torch.arange(0, d_emb, 2).float()\n",
     "\n",
-    "pos_embedding(img.unsqueeze(0))\n",
+    "    div = torch.exp(-i * math.log(10000) / d_emb)\n",
     "\n",
-    "# show the embedding and patches\n",
-    "embedding, patches = pos_embedding(img.unsqueeze(0))\n",
+    "    sin = torch.sin(pos * div)\n",
+    "    cos = torch.cos(pos * div)\n",
     "\n",
-    "print('patches: ', patches.shape)\n",
-    "print('embedding: ', embedding.shape)\n",
+    "    pos_encoding = torch.cat((sin, cos), dim=1).view(1, max_len, d_emb)\n",
     "\n",
-    "# # plot the patches\n",
-    "# no = int(32 / patch_size)\n",
-    "# fig, ax = plt.subplots(no, no)\n",
-    "# for i in range(no):\n",
-    "#     for j in range(no):\n",
-    "#         ax[i, j].imshow(patches[i * no + j, :].permute(1, 2, 0))\n",
-    "#         ax[i, j].axis('off')\n",
+    "    return pos_encoding\n",
     "\n",
-    "# # plot the embeddings\n",
-    "# no = int(32 / patch_size)\n",
-    "# fig, ax = plt.subplots(no, no)\n",
-    "# for i in range(no):\n",
-    "#     for j in range(no):\n",
-    "#         ax[i, j].imshow(embedding[i * no + j, :].detach().numpy().reshape(1, -1))\n",
-    "#         ax[i, j].axis('off')\n",
-    "    \n",
+    "seq_len = 100\n",
+    "embedding_dim = 64\n",
     "\n",
-    "# plot results of get_cosine_position_embedding\n",
-    "pos_embedding = pos_embedding.get_cosine_position_embedding(img.unsqueeze(0), patch_size=patch_size)\n",
+    "pos_encoding = get_pos_encoding(seq_len, embedding_dim)\n",
     "\n",
-    "print('pos_embedding: ', pos_embedding.shape)\n",
+    "print(pos_encoding)\n",
     "\n",
-    "# plot the embeddings\n",
-    "no = int(32 / patch_size)\n",
-    "fig, ax = plt.subplots(no, no)\n",
-    "for i in range(no):\n",
-    "    for j in range(no):\n",
-    "        ax[i, j].imshow(pos_embedding[0, :, i * no + j].detach().numpy().reshape(1, -1))\n",
-    "        ax[i, j].axis('off')\n",
-    "\n"
+    "# plot the position encoding\n",
+    "fig, ax = plt.subplots(1, 1)\n",
+    "ax.imshow(pos_encoding.squeeze().detach().numpy())\n",
+    "ax.axis('off')\n"
    ]
   }
  ],
-- 
GitLab