diff --git a/example/image_models/cnn_pytorch.ipynb b/example/image_models/cnn_pytorch.ipynb deleted file mode 100644 index 08959ab6c..000000000 --- a/example/image_models/cnn_pytorch.ipynb +++ /dev/null @@ -1,239 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torchvision\n", - "from torchvision.transforms import v2\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/omilod/Desktop/projects/GigaTorch/env/lib/python3.11/site-packages/torchvision/transforms/v2/_deprecated.py:41: UserWarning: The transform `ToTensor()` is deprecated and will be removed in a future release. Instead, please use `v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)])`.\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Files already downloaded and verified\n", - "Files already downloaded and verified\n" - ] - } - ], - "source": [ - "# Device configuration\n", - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "\n", - "num_epochs = 10\n", - "batch_size = 4\n", - "learning_rate = 0.001\n", - "\n", - "transform = v2.Compose(\n", - " [v2.ToTensor(),\n", - " v2.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", - "\n", - "train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n", - "test_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n", - "\n", - "tarin_loder = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n", - "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Epoche [1/10], Step [2000/12500], Loss: 2.2467\n", - "Epoche [1/10], Step [4000/12500], Loss: 2.2539\n", - "Epoche [1/10], Step [6000/12500], Loss: 2.2477\n", - "Epoche [1/10], Step [8000/12500], Loss: 2.4960\n", - "Epoche [1/10], Step [10000/12500], Loss: 2.4319\n", - "Epoche [1/10], Step [12000/12500], Loss: 2.3993\n", - "Epoche [2/10], Step [2000/12500], Loss: 1.7446\n", - "Epoche [2/10], Step [4000/12500], Loss: 1.7958\n", - "Epoche [2/10], Step [6000/12500], Loss: 1.9598\n", - "Epoche [2/10], Step [8000/12500], Loss: 2.0397\n", - "Epoche [2/10], Step [10000/12500], Loss: 1.8597\n", - "Epoche [2/10], Step [12000/12500], Loss: 1.1347\n", - "Epoche [3/10], Step [2000/12500], Loss: 1.2750\n", - "Epoche [3/10], Step [4000/12500], Loss: 1.4700\n", - "Epoche [3/10], Step [6000/12500], Loss: 0.8299\n", - "Epoche [3/10], Step [8000/12500], Loss: 2.4020\n", - "Epoche [3/10], Step [10000/12500], Loss: 0.8445\n", - "Epoche [3/10], Step [12000/12500], Loss: 1.2414\n", - "Epoche [4/10], Step [2000/12500], Loss: 1.1043\n", - "Epoche [4/10], Step [4000/12500], Loss: 1.6044\n", - "Epoche [4/10], Step [6000/12500], Loss: 1.8810\n", - "Epoche [4/10], Step [8000/12500], Loss: 1.4032\n", - "Epoche [4/10], Step [10000/12500], Loss: 1.9440\n", - "Epoche [4/10], Step [12000/12500], Loss: 1.4680\n", - "Epoche [5/10], Step [2000/12500], Loss: 0.9280\n", - "Epoche [5/10], Step [4000/12500], Loss: 1.0829\n", - "Epoche [5/10], Step [6000/12500], Loss: 0.8902\n", - "Epoche [5/10], Step [8000/12500], Loss: 1.4200\n", - "Epoche [5/10], Step [10000/12500], Loss: 2.7336\n", - "Epoche [5/10], Step [12000/12500], Loss: 1.7483\n", - "Epoche [6/10], Step [2000/12500], Loss: 1.4049\n", - "Epoche [6/10], Step [4000/12500], Loss: 0.8819\n", - "Epoche [6/10], Step [6000/12500], Loss: 2.0509\n", - "Epoche [6/10], Step [8000/12500], Loss: 1.5775\n", - "Epoche [6/10], Step [10000/12500], Loss: 1.4660\n", - "Epoche [6/10], Step [12000/12500], Loss: 1.2865\n", - "Epoche [7/10], Step [2000/12500], Loss: 1.3137\n", - "Epoche [7/10], Step [4000/12500], Loss: 1.3415\n", - "Epoche [7/10], Step [6000/12500], Loss: 0.9533\n", - "Epoche [7/10], Step [8000/12500], Loss: 1.1399\n", - "Epoche [7/10], Step [10000/12500], Loss: 1.3637\n", - "Epoche [7/10], Step [12000/12500], Loss: 0.6954\n", - "Epoche [8/10], Step [2000/12500], Loss: 0.9204\n", - "Epoche [8/10], Step [4000/12500], Loss: 1.1020\n", - "Epoche [8/10], Step [6000/12500], Loss: 1.0856\n", - "Epoche [8/10], Step [8000/12500], Loss: 1.2919\n", - "Epoche [8/10], Step [10000/12500], Loss: 0.5048\n", - "Epoche [8/10], Step [12000/12500], Loss: 1.6188\n", - "Epoche [9/10], Step [2000/12500], Loss: 0.7271\n", - "Epoche [9/10], Step [4000/12500], Loss: 0.8624\n", - "Epoche [9/10], Step [6000/12500], Loss: 1.6987\n", - "Epoche [9/10], Step [8000/12500], Loss: 0.8891\n", - "Epoche [9/10], Step [10000/12500], Loss: 1.0421\n", - "Epoche [9/10], Step [12000/12500], Loss: 0.9009\n", - "Epoche [10/10], Step [2000/12500], Loss: 1.8559\n", - "Epoche [10/10], Step [4000/12500], Loss: 1.9273\n", - "Epoche [10/10], Step [6000/12500], Loss: 1.1227\n", - "Epoche [10/10], Step [8000/12500], Loss: 1.3718\n", - "Epoche [10/10], Step [10000/12500], Loss: 1.6195\n", - "Epoche [10/10], Step [12000/12500], Loss: 0.9517\n", - "Finished training\n", - "Accuracy of the network: 58.32 %\n", - "Accuracy: 73.4 %\n", - "Accuracy: 60.8 %\n", - "Accuracy: 39.0 %\n", - "Accuracy: 38.7 %\n", - "Accuracy: 50.4 %\n", - "Accuracy: 38.7 %\n", - "Accuracy: 73.6 %\n", - "Accuracy: 66.8 %\n", - "Accuracy: 75.0 %\n", - "Accuracy: 66.8 %\n" - ] - } - ], - "source": [ - "class ConvNet(nn.Module):\n", - " def __init__(self) -> None:\n", - " super().__init__()\n", - " self.conv1 = nn.Conv2d(3, 6, 5)\n", - " self.pool = nn.MaxPool2d(2, 2)\n", - " self.conv2 = nn.Conv2d(6, 16, 5)\n", - " self.fc1 = nn.Linear(16 * 5 * 5, 120)\n", - " self.fc2 = nn.Linear(120, 84)\n", - " self.fc3 = nn.Linear(84, 10)\n", - "\n", - " def forward(self, x):\n", - " # -> n, 3, 32, 32\n", - " x = self.pool(F.relu(self.conv1(x))) # -> n, 6, 14, 14\n", - " x = self.pool(F.relu(self.conv2(x))) # -> n, 16, 5, 5\n", - " x = x.view(-1, 16 * 5 * 5) # -> n, 400\n", - " x = F.relu(self.fc1(x)) # -> n, 120\n", - " x = F.relu(self.fc2(x)) # -> n, 84\n", - " x = self.fc3(x) # -> n, 10\n", - " return x\n", - "\n", - "model = ConvNet()\n", - "model.to(device)\n", - "critertion = nn.CrossEntropyLoss()\n", - "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", - "\n", - "n_total_steps = len(tarin_loder)\n", - "for epoch in range(num_epochs):\n", - " for i, (images, labels) in enumerate(tarin_loder):\n", - " images = images.to(device)\n", - " labels = labels.to(device)\n", - "\n", - " # Forward pass\n", - " outputs = model(images)\n", - " loss = critertion(outputs, labels)\n", - "\n", - " # Backward and optimize\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " if (i+1) % 2000 == 0:\n", - " print(f'Epoche [{epoch+1}/{num_epochs}], Step [{i+1}/{n_total_steps}], Loss: {loss.item():.4f}')\n", - "print(\"Finished training\")\n", - "\n", - "\n", - "with torch.no_grad():\n", - " n_correct = 0\n", - " n_samples = 0\n", - " n_class_correct = [0 for i in range(10)]\n", - " n_class_samples = [0 for i in range(10)]\n", - " for images, labels in test_loader:\n", - " images = images.to(device)\n", - " labels = labels.to(device)\n", - " outputs = model(images)\n", - " # max returns (value ,index)\n", - " _, predicted = torch.max(outputs, 1)\n", - " n_samples += labels.size(0)\n", - " n_correct += (predicted == labels).sum().item()\n", - " \n", - " for i in range(batch_size):\n", - " label = labels[i]\n", - " pred = predicted[i]\n", - " if (label == pred):\n", - " n_class_correct[label] += 1\n", - " n_class_samples[label] += 1\n", - "\n", - " acc = 100.0 * n_correct / n_samples\n", - " print(f'Accuracy of the network: {acc} %')\n", - "\n", - " for i in range(10):\n", - " acc = 100.0 * n_class_correct[i] / n_class_samples[i]\n", - " print(f'Accuracy: {acc} %')\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/example/image_models/pytorch/cnn.ipynb b/example/image_models/pytorch/cnn.ipynb new file mode 100644 index 000000000..b11949ea9 --- /dev/null +++ b/example/image_models/pytorch/cnn.ipynb @@ -0,0 +1,246 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "import torchvision\n", + "from torchvision.transforms import v2\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/orifmilod/Desktop/projects/gigatorch/env/lib/python3.10/site-packages/torchvision/transforms/v2/_deprecated.py:41: UserWarning: The transform `ToTensor()` is deprecated and will be removed in a future release. Instead, please use `v2.Compose([v2.ToImage(), v2.ToDtype(torch.float32, scale=True)])`.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100.0%\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./data/cifar-10-python.tar.gz to ./data\n", + "Files already downloaded and verified\n" + ] + } + ], + "source": [ + "# Device configuration\n", + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "\n", + "num_epochs = 16\n", + "batch_size = 8\n", + "learning_rate = 0.001\n", + "\n", + "transform = v2.Compose(\n", + " [v2.ToTensor(),\n", + " v2.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", + "\n", + "train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\n", + "test_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n", + "\n", + "tarin_loder = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n", + "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoche [1/16], Step [2000/6250], Loss: 2.3213\n", + "Epoche [1/16], Step [4000/6250], Loss: 2.2904\n", + "Epoche [1/16], Step [6000/6250], Loss: 2.2973\n", + "Epoche [2/16], Step [2000/6250], Loss: 2.2808\n", + "Epoche [2/16], Step [4000/6250], Loss: 2.1523\n", + "Epoche [2/16], Step [6000/6250], Loss: 2.2306\n", + "Epoche [3/16], Step [2000/6250], Loss: 2.3663\n", + "Epoche [3/16], Step [4000/6250], Loss: 2.1092\n", + "Epoche [3/16], Step [6000/6250], Loss: 1.2913\n", + "Epoche [4/16], Step [2000/6250], Loss: 1.6886\n", + "Epoche [4/16], Step [4000/6250], Loss: 2.6763\n", + "Epoche [4/16], Step [6000/6250], Loss: 1.7113\n", + "Epoche [5/16], Step [2000/6250], Loss: 1.2147\n", + "Epoche [5/16], Step [4000/6250], Loss: 1.8217\n", + "Epoche [5/16], Step [6000/6250], Loss: 1.5523\n", + "Epoche [6/16], Step [2000/6250], Loss: 1.2940\n", + "Epoche [6/16], Step [4000/6250], Loss: 1.6208\n", + "Epoche [6/16], Step [6000/6250], Loss: 1.4940\n", + "Epoche [7/16], Step [2000/6250], Loss: 1.7596\n", + "Epoche [7/16], Step [4000/6250], Loss: 1.1859\n", + "Epoche [7/16], Step [6000/6250], Loss: 1.7896\n", + "Epoche [8/16], Step [2000/6250], Loss: 1.6281\n", + "Epoche [8/16], Step [4000/6250], Loss: 1.8791\n", + "Epoche [8/16], Step [6000/6250], Loss: 1.3839\n", + "Epoche [9/16], Step [2000/6250], Loss: 1.7757\n", + "Epoche [9/16], Step [4000/6250], Loss: 1.4942\n", + "Epoche [9/16], Step [6000/6250], Loss: 1.2496\n", + "Epoche [10/16], Step [2000/6250], Loss: 1.0990\n", + "Epoche [10/16], Step [4000/6250], Loss: 0.9228\n", + "Epoche [10/16], Step [6000/6250], Loss: 1.6966\n", + "Epoche [11/16], Step [2000/6250], Loss: 1.4388\n", + "Epoche [11/16], Step [4000/6250], Loss: 1.3887\n", + "Epoche [11/16], Step [6000/6250], Loss: 1.8761\n", + "Epoche [12/16], Step [2000/6250], Loss: 1.4832\n", + "Epoche [12/16], Step [4000/6250], Loss: 1.6177\n", + "Epoche [12/16], Step [6000/6250], Loss: 1.6273\n", + "Epoche [13/16], Step [2000/6250], Loss: 1.7118\n", + "Epoche [13/16], Step [4000/6250], Loss: 1.1288\n", + "Epoche [13/16], Step [6000/6250], Loss: 0.6880\n", + "Epoche [14/16], Step [2000/6250], Loss: 1.3681\n", + "Epoche [14/16], Step [4000/6250], Loss: 1.9006\n", + "Epoche [14/16], Step [6000/6250], Loss: 1.3762\n", + "Epoche [15/16], Step [2000/6250], Loss: 1.1387\n", + "Epoche [15/16], Step [4000/6250], Loss: 1.4813\n", + "Epoche [15/16], Step [6000/6250], Loss: 1.3308\n", + "Epoche [16/16], Step [2000/6250], Loss: 1.0389\n", + "Epoche [16/16], Step [4000/6250], Loss: 1.1721\n", + "Epoche [16/16], Step [6000/6250], Loss: 1.0445\n", + "Finished training\n", + "Accuracy of the network: 55.18 %\n", + "Accuracy: 3 56.6 %\n", + "Accuracy: 5 62.4 %\n", + "Accuracy: 3 60.8 %\n", + "Accuracy: 8 27.8 %\n", + "Accuracy: 3 39.0 %\n", + "Accuracy: 5 44.2 %\n", + "Accuracy: 1 60.9 %\n", + "Accuracy: 7 69.0 %\n" + ] + }, + { + "ename": "IndexError", + "evalue": "index 8 is out of bounds for dimension 0 with size 8", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 72\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m10\u001b[39m):\n\u001b[1;32m 71\u001b[0m acc \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m100.0\u001b[39m \u001b[38;5;241m*\u001b[39m n_class_correct[i] \u001b[38;5;241m/\u001b[39m n_class_samples[i]\n\u001b[0;32m---> 72\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mAccuracy: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mlabels\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00macc\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m %\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "\u001b[0;31mIndexError\u001b[0m: index 8 is out of bounds for dimension 0 with size 8" + ] + } + ], + "source": [ + "class ConvNet(nn.Module):\n", + " def __init__(self) -> None:\n", + " super().__init__()\n", + " self.conv1 = nn.Conv2d(3, 6, 5)\n", + " self.pool = nn.MaxPool2d(2, 2)\n", + " self.conv2 = nn.Conv2d(6, 16, 5)\n", + " self.fc1 = nn.Linear(16 * 5 * 5, 120)\n", + " self.fc2 = nn.Linear(120, 84)\n", + " self.fc3 = nn.Linear(84, 10)\n", + "\n", + " def forward(self, x):\n", + " # -> n, 3, 32, 32\n", + " x = self.pool(F.relu(self.conv1(x))) # -> n, 6, 14, 14\n", + " x = self.pool(F.relu(self.conv2(x))) # -> n, 16, 5, 5\n", + " x = x.view(-1, 16 * 5 * 5) # -> n, 400\n", + " x = F.relu(self.fc1(x)) # -> n, 120\n", + " x = F.relu(self.fc2(x)) # -> n, 84\n", + " x = self.fc3(x) # -> n, 10\n", + " return x\n", + "\n", + "model = ConvNet()\n", + "model.to(device)\n", + "critertion = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n", + "\n", + "n_total_steps = len(tarin_loder)\n", + "for epoch in range(num_epochs):\n", + " for i, (images, labels) in enumerate(tarin_loder):\n", + " images = images.to(device)\n", + " labels = labels.to(device)\n", + "\n", + " # Forward pass\n", + " outputs = model(images)\n", + " loss = critertion(outputs, labels)\n", + "\n", + " # Backward and optimize\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " if (i+1) % 2000 == 0:\n", + " print(f'Epoche [{epoch+1}/{num_epochs}], Step [{i+1}/{n_total_steps}], Loss: {loss.item():.4f}')\n", + "print(\"Finished training\")\n", + "\n", + "\n", + "with torch.no_grad():\n", + " n_correct = 0\n", + " n_samples = 0\n", + " n_class_correct = [0 for i in range(10)]\n", + " n_class_samples = [0 for i in range(10)]\n", + " for images, labels in test_loader:\n", + " images = images.to(device)\n", + " labels = labels.to(device)\n", + " outputs = model(images)\n", + " # max returns (value ,index)\n", + " _, predicted = torch.max(outputs, 1)\n", + " n_samples += labels.size(0)\n", + " n_correct += (predicted == labels).sum().item()\n", + " \n", + " for i in range(batch_size):\n", + " label = labels[i]\n", + " pred = predicted[i]\n", + " if (label == pred):\n", + " n_class_correct[label] += 1\n", + " n_class_samples[label] += 1\n", + "\n", + " acc = 100.0 * n_correct / n_samples\n", + " print(f'Accuracy of the network: {acc} %')\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/language_model/gigatorch/CBOW.py b/example/language_model/gigatorch/CBOW.py new file mode 100644 index 000000000..2d9f53db3 --- /dev/null +++ b/example/language_model/gigatorch/CBOW.py @@ -0,0 +1,20 @@ +from gigatorch.embedding import Embedding, prepare_data, make_context_vector + + +def main(): + raw_text = """We are about to study the idea of a computational process. + Computational processes are abstract beings that inhabit computers. + As they evolve, processes manipulate other abstract things called data. + The evolution of a process is directed by a pattern of rules called a program. + People create programs to direct processes. In effect, + we conjure the spirits of the computer with our spells.""" + data, word_to_index, index_to_word = prepare_data(raw_text) + model = Embedding(100, 10) + + for target, context in data: + context_vector = make_context_vector(context, word_to_index) + print(model(context_vector)) + + +if __name__ == "__main__": + main() diff --git a/example/language_model/makemore_part2_mlp.ipynb b/example/language_model/makemore_part2_mlp.ipynb deleted file mode 100644 index 74696a1ae..000000000 --- a/example/language_model/makemore_part2_mlp.ipynb +++ /dev/null @@ -1,482 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn.functional as F\n", - "import matplotlib.pyplot as plt # for making figures\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [], - "source": [ - "# read in all the words\n", - "words = open('names.txt', 'r').read().splitlines()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z', 0: '.'}\n" - ] - } - ], - "source": [ - "# build the vocabulary of characters and mappings to/from integers\n", - "chars = sorted(list(set(''.join(words))))\n", - "stoi = {s:i+1 for i,s in enumerate(chars)}\n", - "stoi['.'] = 0\n", - "itos = {i:s for s,i in stoi.items()}\n", - "print(itos)" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [], - "source": [ - "# build the dataset\n", - "\n", - "block_size = 3 # context length: how many characters do we take to predict the next one?\n", - "X, Y = [], []\n", - "for w in words:\n", - " \n", - " #print(w)\n", - " context = [0] * block_size\n", - " for ch in w + '.':\n", - " ix = stoi[ch]\n", - " X.append(context)\n", - " Y.append(ix)\n", - " #print(''.join(itos[i] for i in context), '--->', itos[ix])\n", - " context = context[1:] + [ix] # crop and append\n", - " \n", - "X = torch.tensor(X)\n", - "Y = torch.tensor(Y)" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(torch.Size([228146, 3]), torch.int64, torch.Size([228146]), torch.int64)" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "X.shape, X.dtype, Y.shape, Y.dtype" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([182625, 3]) torch.Size([182625])\n", - "torch.Size([22655, 3]) torch.Size([22655])\n", - "torch.Size([22866, 3]) torch.Size([22866])\n" - ] - } - ], - "source": [ - "# build the dataset\n", - "block_size = 3 # context length: how many characters do we take to predict the next one?\n", - "\n", - "def build_dataset(words): \n", - " X, Y = [], []\n", - " for w in words:\n", - "\n", - " #print(w)\n", - " context = [0] * block_size\n", - " for ch in w + '.':\n", - " ix = stoi[ch]\n", - " X.append(context)\n", - " Y.append(ix)\n", - " #print(''.join(itos[i] for i in context), '--->', itos[ix])\n", - " context = context[1:] + [ix] # crop and append\n", - "\n", - " X = torch.tensor(X)\n", - " Y = torch.tensor(Y)\n", - " print(X.shape, Y.shape)\n", - " return X, Y\n", - "\n", - "import random\n", - "random.seed(42)\n", - "random.shuffle(words)\n", - "n1 = int(0.8*len(words))\n", - "n2 = int(0.9*len(words))\n", - "\n", - "Xtr, Ytr = build_dataset(words[:n1])\n", - "Xdev, Ydev = build_dataset(words[n1:n2])\n", - "Xte, Yte = build_dataset(words[n2:])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [], - "source": [ - "C = torch.randn((27, 2))" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([228146, 3, 2])" - ] - }, - "execution_count": 69, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emb = C[X]\n", - "emb.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "metadata": {}, - "outputs": [], - "source": [ - "g = torch.Generator().manual_seed(2147483647) # for reproducibility\n", - "embdedding_size = 10\n", - "C = torch.randn((27, embdedding_size), generator=g)\n", - "W1 = torch.randn((block_size * embdedding_size, 200), generator=g)\n", - "b1 = torch.randn(200, generator=g)\n", - "W2 = torch.randn((200, 27), generator=g)\n", - "b2 = torch.randn(27, generator=g)\n", - "parameters = [C, W1, b1, W2, b2]" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "11897" - ] - }, - "execution_count": 71, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sum(p.nelement() for p in parameters) # number of parameters in total" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "metadata": {}, - "outputs": [], - "source": [ - "for p in parameters:\n", - " p.requires_grad = True" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.971953272819519\n" - ] - } - ], - "source": [ - "epochs = 200000\n", - "# learning_rates = 10**(torch.linspace(-3, 0, epochs))\n", - "used_lrs = []\n", - "lossi = []\n", - "# stepi = []\n", - "for i in range(epochs):\n", - " \n", - " # minibatch construct\n", - " ix = torch.randint(0, Xtr.shape[0], (32,))\n", - " \n", - " # forward pass\n", - " emb = C[Xtr[ix]] # (32, 3, 10)\n", - " h = torch.tanh(emb.view(-1, embdedding_size * block_size) @ W1 + b1) # (32, 200)\n", - " logits = h @ W2 + b2 # (32, 27)\n", - " loss = F.cross_entropy(logits, Ytr[ix])\n", - " # print(loss.item())\n", - " \n", - " # backward pass\n", - " for p in parameters:\n", - " p.grad = None\n", - " loss.backward()\n", - " lr = 0.01 if i < epochs / 2 else 0.001\n", - " for p in parameters:\n", - " p.data -= lr * p.grad\n", - "\n", - " # track stats\n", - " used_lrs.append(i)\n", - " # stepi.append(i)\n", - " lossi.append(loss.item())\n", - "\n", - "print(loss.item())" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 82, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAGdCAYAAADJ6dNTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSUklEQVR4nO3dd3hUVfoH8O8kpAEpQEgjoUOoCU0gAamhySKoi4i4gCIqgsKqyOJaEFfDT1ZZbChKURFBlKKIIC3SQkkgdCKhhZKCQAol/fz+CBlmkin3ztyZuTP5fp4nz5PcuTP33JnJ3HfOec97NEIIASIiIiKVcHN0A4iIiIh0MTghIiIiVWFwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERESkKgxOiIiISFVqOLoBUpSVleHKlSvw9fWFRqNxdHOIiIhIAiEE8vPzERYWBjc36f0hThGcXLlyBREREY5uBhEREVng4sWLCA8Pl7y/UwQnvr6+AMpPzs/Pz8GtISIiIiny8vIQERGhvY5L5RTBScVQjp+fH4MTIiIiJyM3JYMJsURERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhWrgpM5c+ZAo9Fg2rRpJvdbtWoVWrVqBW9vb7Rv3x4bNmyw5rBERETkwiwOTg4cOIAvvvgCUVFRJvfbs2cPRo8ejQkTJuDQoUMYMWIERowYgWPHjll6aCIiInJhFgUnN2/exJgxY/Dll1+iTp06JvedP38+Bg8ejOnTp6N169Z455130KlTJ3zyyScWNZiIiIhcm0XByeTJkzF06FDExcWZ3TcxMbHKfoMGDUJiYqLR+xQWFiIvL0/vh4iIiKoH2Qv/rVixAgcPHsSBAwck7Z+ZmYng4GC9bcHBwcjMzDR6n/j4eLz99ttym+YStp7Mwu2iUgyLDnN0U4iIiBxCVs/JxYsXMXXqVHz33Xfw9va2VZswc+ZM5Obman8uXrxos2OpSVmZwISvk/DC94dwNb/Q0c0hIiJyCFk9J8nJycjOzkanTp2020pLS7Fjxw588sknKCwshLu7u959QkJCkJWVpbctKysLISEhRo/j5eUFLy8vOU1zCULn97yCYtT3rX7PARERkayek/79++Po0aNISUnR/nTp0gVjxoxBSkpKlcAEAGJiYrB161a9bZs3b0ZMTIx1LSciIiKXJKvnxNfXF+3atdPbVqtWLdSrV0+7fezYsWjQoAHi4+MBAFOnTkXv3r3xwQcfYOjQoVixYgWSkpKwcOFChU6BiIiIXIniFWLT09ORkZGh/Ts2NhbLly/HwoULER0djR9//BFr166tEuQQERERARbM1qksISHB5N8AMHLkSIwcOdLaQxEREVE1wLV1iIiISFUYnBAREZGqMDghIiIiVWFwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERESkKgxOiIiISFUYnBAREZGqMDghIiIiVWFwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERESkKgxOiIiISFUYnKiIEMLRTSAiInI4BicqpXF0A4iIiByEwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQnZnBACQghHN4OIiJwEgxOyuQlfJ+HBT3ajtIwBChERmVfD0Q0g17ftVDYA4FRmHtqG+Tu4NUREpHbsOSG70UDj6CYQEZETYHBCREREqsLgREWYkUFERMTgRLU0Gg6BEBFR9cTgpBq5U1SKD39PxdFLuY5uChERkVEMTqqRT7afxkfb0jDsk12ObgoREZFRDE6qkZMZ+Y5uAhERkVkMToiIiEhVGJwQERGRqjA4ISIiIlVhcEJERESqwuCEiIiIVIXBiRElpWUoKC51dDMUJQRr0JJyTmbkYcrygzj31y1HN4WIXAxXJTai99wEZOcX4OisQfD2cHd0c4hUZ8Snu1FYUoajl3Pxx/S+jm4OEbkQWT0nCxYsQFRUFPz8/ODn54eYmBj89ttvRvdfunQpNBqN3o+3t7fVjbaHyzl3UFwqkJZ909FNIVKlwpIyAMCFa7cd3BIicjWygpPw8HDMmTMHycnJSEpKQr9+/TB8+HAcP37c6H38/PyQkZGh/blw4YLVjXa0Q+k38LePd2Lf2WuObgoREZHLkTWsM2zYML2/3333XSxYsAB79+5F27ZtDd5Ho9EgJCTE8haq0GML96KwpAyjFu7F+TlDHd0cIiLVen3tUdSt5YWXBrR0dFPIiVicEFtaWooVK1bg1q1biImJMbrfzZs30ahRI0RERJjtZalQWFiIvLw8vR81qejOtqUyJq8SkZNLy87Hsr3p+GjraUc3hZyM7ODk6NGjqF27Nry8vPDcc89hzZo1aNOmjcF9IyMjsXjxYqxbtw7Lli1DWVkZYmNjcenSJZPHiI+Ph7+/v/YnIiJCbjMdSolZMT8cuKhAS/SpPdy5ml+ItGyu/0PlTmbkcSaQkysotv0XOXJNsoOTyMhIpKSkYN++fZg0aRLGjRuHEydOGNw3JiYGY8eORYcOHdC7d2+sXr0a9evXxxdffGHyGDNnzkRubq725+JF5S/UtlJQXIq4D//ASytTrHqc9UcylGmQE7nv3S2I+3AHLt1ggmV1l3O7CEPm70Tf/yY4uilE5ACygxNPT080b94cnTt3Rnx8PKKjozF//nxJ9/Xw8EDHjh2RlpZmcj8vLy/tjKCKH2eRkJqNM1dvYfWhy45uitM6dlldw3hkf5dz7ji6CUTkQFYXYSsrK0NhYaGkfUtLS3H06FGEhoZae1iHEELgdlGJo5tBRETk0mTN1pk5cyaGDBmChg0bIj8/H8uXL0dCQgI2bdoEABg7diwaNGiA+Ph4AMDs2bPRvXt3NG/eHDk5OZg7dy4uXLiAp59+WvkzsYOXfzisSI/I1pNZSL5wA68MjISbm0aBlhEREbkOWcFJdnY2xo4di4yMDPj7+yMqKgqbNm3CgAEDAADp6elwc7vXGXPjxg1MnDgRmZmZqFOnDjp37ow9e/YYTaBVO6WGaiZ8nQQAaBXqhwejwxR5TCIiIlchKzhZtGiRydsTEhL0/p43bx7mzZsnu1HO5MatItSp5WnRfTNz1T2uLoTA+iMZaBXiixbBvo5uDhERVRNc+M9K/T5IUOyxbF3apPLjZ+YW4OOtp/HXTcM5QwmpV/HC94cwYN4O2zaMqBKW+SGq3hicmHH8Sq7J22/cLrbJcTV2SEV5YtE+fLD5Tzz/3UGDtx+5ZPrciYiIbIHBiRkzfjqK0Qv3orTM9b7KVSxquP/cdQe3hIiI6B4GJxIknr2GpPPOeQG/drMQ2XkFjm4GOdCVnDvYciJLkcrFRHLwLUeWYnAikb07TjSa8nUpXv7hMC5cs6yEd1mZQOf/bEHX97biTlGpwi0kZxE7Zxue/iYJvx3LdHRTiIgkYXCi49vE8/gm8bzs+209mYWHP9ut+DogjyxIxE8HL2Hc4v0W3b+o9N66FlfzpRXKI9eVeOaao5tARCSJrKnEriy/oBhvrCtfMXlExway7ltRt2TayhRM6t1MsTbl3ilPtj1/zTFrzQiFlwq0R5IvERE5P/ac3FVceu9CXFRSdSXN/ALzs3IOX8xRskmK4/AvERE5AwYnKO+h+OXwFZP7zNl4yk6tKaeB8t0Mtk6I3J32FyZ/d1DyEFLO7SJt7xAREVEFDusAePrrAzhw/obJfc5eVTafxBWN+WofgPLhoM/GdDa5b2FJKTrM3gwAOPPeA3DnGkNEspzKzMN/1p/EywNbomPDOo5uDpGi2HMCVAlMrOlgkJNXYeo46deVzTOxJH/E0ufhco75qct/3SzS/s6Vnonke+Kr/diV9hce+myPo5tilNJ5a1R9MDhxYdYmoPJjhUi9jC07QeQKGJy4gO2p2Zi0LBnXbxWZ35nICbB4V1XXbxXh58NXUFDMmkXk+hicuIAnlxzAb8cy8d6Gk45uCpEiqvNwwNFLuXhkwR4kX9CvSj164V68+P0h/J+dk/OJHIHBiQFrDl1ydBMskl3NC61l5xdg7OL9+P04K6EawjozzmH0l3uRfOEGHlmQqLc9NSsfAPDrkQxHNMsi7AEjSzE4MeC9DfxmYk9KfX69/csJ7PjzKp75NlmhR6wevt5zHo9+kYibhUxMVgO+DkQMThTnyC+nZ+6uMmyI2r7B2OJ5umYkQTDndhH+veYoDqXfwL6z13Dur1soLi3Dt3svKL7kgK0cvpiDM1eNv77WeOvn49h/7joW7Txnk8cnIpKLdU5cyOWcO8o+oImIJvdOMfx9PJQ9no3M/uUEVh+6jO/2pWu3/WtIK8z5rbyH7PycoY5qmiTZeQUY/uluALZt6x0VJVqqLZgmIvtiz4kLs0WVWaB8zDv67d9tmJin7JXJUI9D0vnrBvZUp4s3HLO2EhGRozA4IaOMhQizfilfIHFBwhnzj8GvwEREJBOHdRysrExAowE0ZqZSFJeWoaikDLW8nOAlEwIbj2XiuWXJqFvL09Gtqbbe/fWE3oKWRETOgj0nDlRcWoZ+HyTgqaUHzO7b6/3taPvWJkmrIxtjz06M55aVz5gxVhhONxaT266ikjI8+kUi4jecZJVMI24WluDLneewdM95RzeFyOldvH4bz3yT5FTDwc6OwYkD7Tt3Heev3cb21Ks4a2YmRkZu+Xo1Ry7lWnQsS+IStY7IbDiagf3nruOLHWfR5T9bkGZilhIAly7wsXDHGYN1eUrLqr54rvssENnWiysO4fcTWfj754nmdyZFMDixk5zbRcgz0esh9U2/O+0vXFR4UcAK124WYsX+dNXXWSgqLdP725mKUikpLTsf7204hX+uPKzI46kphlNpXEzV1KUbCs+EJLMYnNhBQXEpOszejKhZv6NM5xutbs/E9VtFkkp2f5ZwBve/v13ScfWHTsw/9tjF+/Gv1Ufx2uqjkh7fVqYsP2S+N8RKZ66qp75JWZnADwcu4vTdCqBS3bh9L9g1FfgSETkbBicK001sFULg/zaewmc6s1oqf+tXk+NX8gAAG485tvx7SZnA41/urbJdyZk/aiq+tjblMl796QgGzNth8WN8ui1Nf4MDux52p/2FR79ItHmASeqnxh6wEhV/BtM9DE5sKOnCDSxIOIOPtp52dFNgIAXBofRqsBhoW+V1gkpKyzD809148ftDVfZ19kXiDl/Msfoxcu9Y1nPy0soUq49d2Ziv9mH/uevapGiiCskXbmDC0gM476AvB9tTsxH5xkasSrrokOOTdAxOFKY7bH/DyEwVe9Fty87TV62+iC/ZfQ495mzDVQcsMJh84QaOXMrFz4evyL6vrVIp8guK8cqqw9jx51UbHcG28gqKsfrQZZs9PmdSUWWPLNiDraeyHRa4Pv11EkrLBKb/eMQhxyfpGJyoyF83bRfM3C6SX5q8cjDz9i8nlC+RL7kt6vPR1tP4MfkSxi7e7+imWESwd5tszNhQ7GUmmJIZDE4cqPLFv8ecbYo87ppDl7D/nPXz8dU6lVhp64/I740pKC7FlzIXysu5XaSXEC2FEMKiITlLesnsNVnnm8TzeGD+TpM9K85SWTjndhH2nr3mNO0lchYMThQmjPxuL8cu5+KfKw/j0S8SzVadNUbOhe3VHw8jI9ex34KsvS5MWX5IdnGltTKHQ45fyUWH2Zvx1Nf3Cu5dzrmDrSezTD7b45YcwMhK08xvF5XglpHp3t/vT8dPyZdUHVi+ue44TmTkYf4W47lYKm6+ngHzduCxhXstGm60lLM8N44ghMCz3yZh1s/HHd0UshKDE4Vl5xdof88vMF0vZHfaNauOdTnnDj7/Q399G1vVQDHmh6RLmPzdQb1txj48dS+Yt4rM11KRO7XWGqd1ZpYkX7iO+N9OosDEKr2mbjNk2d4LAICE1Hv5KT3mbMOEr5Pw+/Eso/ernM9SWibQ5s1NaPvWJhSX6I/LXLtZiJmrj+LlVYdRbGBGgqXBqq3IfQ7VqCL/auqKFNm9YqS841fysOl4FisjuwAnWKjFuehegA1dCmavP6HIcd799QRW7L+I/ErfoKUmIVo6u8OQP7PkTRk9c/Um+n/wh/ZvYz01A+btwPk5Q61qGyC/uNgjC8p7Knw83DEtrqXVxzcnM6/A/E536RbIu1rptdbNKypVc9eJizp/7Raa1q/t6GY4BVu9O0sYILoM9pzY0BUDyaPL96Ur8thf7jxXJTABgDfWSevOjH77d0XaAZRfMP+35U/J+3+beEGxYwO2+6D735bTePXHw7K+4c/+5QQe/mw3ikpsn236c4r9hhJswdUuI7bomTI2fEe2d0cn2Nd9Za/k3EHiGet6vck8BicKe33tMe3vH2yWfsG2tbmbUmUPI8m5ePzPRP5AhVm/HMexy5atDWSSDXsJfki6hEW7pCe+Lt59DgfTc7DtVLbN2lRhqx2OIZWhJQ/MvSw/Jl/CkUs5tmmQk6pcfuDdDScd1BLX9/TXB3DASK7ZtlNZaP3mRoM1qmLnbMPoL/dq71tQXIpD6Tc4rKewah2clJUJh02NrY72n7uOv328y67HPHD+usWLJVawpGBUmYOHVaQcvvJwmjVf/N/91bLhygc/2W1we3UclSooLkXHdzbrbduT9peDWmPeztNXkZZtv7wwpW05mV0l2bzCzLtLeHxo4gtmRXAybvF+PPTZHnydeF7xNlZn1To4eeXHw4pN33UlxaUCaw9dxm0VdCn3mbsdqZn5Fk1zzb1djJGfJxpcoddajrp2nsrIc9CRTTtw/oajm+D0KlYeV4uTGXl499cTyLldtf7SyYw8/GPRfsR9aHrJBVv9nxSXliHTwc/X+xtTkZqZj313yzZ8v1+ZIXsqV62Dk9UHbVcd09lNW5mCrxXODTHG1Lfk89duY+qKSiXrK+2/5WS2djaMrusGPlRtRU4A9FPyJYuPM2ph1TWHXN1XO886ugl6HF3T5Gp+oV0uhEPm78SXO88ZnJb7p5Uz6ax9Dv/+eSK6x2/FwXTHBsUjP98jaT9Hv2ecUbUOTkgdzP3bmlss8URGHl5fewypmfofmPb8QPin2TVq7vX9vLzqsMk9hRA4eilXLyFPLkvP/GZhCfaevYayMgEhhFVtUMp/flVP3sU3iefR9b2tdp3mbshMO64cXrEgqJpUrEf1o4RAv7DEdu/hPDPlIoDypObecxPw7zXKvGbFpWXYnfYXbksox+DMGJy4sMUyEjnN+fVIhmKPZYkrEgq93bh9b3p0aZkwms9gC0oW4Vpz6DKGfbILoxYaHg83ZNPxTL3aJpYGZqMX7sVjC/fi68TzmPhNElq/uRGXbti3do5Svtp5Fgt3nDF6e+KZaxjz1V6cvSp9Kvyb647jan4hXlPoQgMAM1cfwaB5O2x6EVWabikCNQSwFSoP/77642FEvr5R9irkSo4Er025jPTrt/GdQjM1P9z8J8Z8tQ/PfuvaC2syOHFhSmb6T15+0PxONvTPlaZ7Gyo7cSXP4AwStSsoLtUOUclJ5L1xuxifJRi/EEvxV34Rjt6dTfVj8iVsOVk+G2hVkvlvp/a8QN0pKsWB89dNDqXdLCzBf349ifc2nDKYMwEAo7/ci91p1/D8d/Lf20p2yn2//yJSs/Kx5YTxYny2cOnGbYvzNnRr6pSUlQfFuXeKsS7lsqq+0f9w9737pcyhQSUXN1XivXLpxm0M+3gX1h66jGV3h9t3nlZvsrQSGJyQS7J2BWb9x7KfmauP4mB6jkX3/e2odb1bK61YRv5yzp0qFyXrhtWM33fckv0Y+XmiyVwU3eq55mrOZDtglW1DjD1dmbkFig9R3ikqRc//247u8VsVSxif+E0Spq5Iwetrjpnd1xVn3WoMpO2XlQm98hKWenPdcRy9nItpZoePXQeDE3I6aRK64U/KnNUi97Pf3P5CAIfSbxj9FmmsQu8amWv22IPUp+aEidyEtOybmLL8YJW8IEvsNzI74trNQkUv4kIInLl6U/Jj6l6a7hSVIuVijtXtOX/tNrrHb8U765XNu9HtGTC01IElKl6XNSnm38N3bLR0gRJ18KRW2Tbl8MUcHL+Si3PXqg4nZeTewfJ96bKKO9qqF/jCtVuqrc/C4IQUJ/9Cb+5Kr//nwh3mu2hn/CQ/JyAztwAv/ZBSZfuvRzJwMiMPm09kSU6E/CHpIh76bA8eMzK7RskKvRVuyRhaWbr7HL6SuKqyoUJUcv1j0T6sP5KBRxZIm90g17qUy+j8ny2KJs/O23Ia/T/4A+9ZMDz6+Fd7MeLT3Vh5wPLeKF2LdyuXP6Y0S9cIy5axbIM9WTpEWREY5RUUY/inuzH0o10oKa362fa3j3bhtTVH8d9NqdY002rL9l5A77kJmP7jEYe2wxgGJ6R6Zy0ogmaJl35IMTi9/E5xKYbM34mJ3yRhwDzTdR0q/HF3wT5rC8BZ6uxV48/ZnaJSzPrlBD7Znqb4cT9LuPeYut/2Kmp4KPkN8Py129h3tvzCWNGzIKearzkVQdmXEoM4oDxI2ngsE4fuDs1ZM1TmLJ5bZlliZtf3tlZZ2FIuqV+ErO1Qk9Mjk3PrXq+ooV6pa3erAO84bd25W2v+3ff3TwctL21gSwxOyOHWpVxxeB2Ayzm3JQ8FWbKarr0XBB67eL/R24rLbLfuz/sb730btEdvsaV1X5IvGC5bLpWh1/ParSJMXZEi62J9XadcfVa+dT0JRy/l4qWVKQbX9FJSSWmZYmtHyU1UdTaWfqwlnb+OIfN3aofKqiOuSkwAlBt3BuSXbp+9/gSa1q+l2PEt8el26TNdqmM9peu3ijB+yX483LEB+kQGIaCmh+T7WjKGb8vnOL+gWLvytNKPK8fZqzfRT2d17mV7rZtqOuyT8qUhLt64jVXPxVr1WKb0en87rkic5WPuddx5+i+UlJahhrsy35MNJaU6o7/fLav/6BeJslZmv1VYglpernFZd42zIKvN/sWytVEMsaRo0ykFEiXV6satIiQZWWDM3kpKy7DvrPy2fLItDUcu5ZYPUxl5r1y4ZrgeSpf/bJF9PFvGfzm35QURtqJkbRxdZ0wM6Vmici+RucBEf3fzr+Ti3efwTK9msttlLzsdPPwiVeKZaxj95V6Mj22MWQ+2dXRzrMZhHQIAfGug/DsZlyyjbHaf/ybgzyzphb5sRaMpz6OY+E2S7PtKmV1hrvKto+SbyXNx9JCiIyg51d5aG45mytr/f1t0F+MrP4/1R65gd6VFEi9eV6Z44AyJCaMFxaUOrfHy/qZTAICle84b3eeWE9V+YnBCdmfqn8cZ7Er7S1bFXGPThh3hGwuD0EMOXsNErjydIZb+H/yh6tV9lfDh75bP/LB3PpS1/rdFf/ZY+rXbmLL8EMZ8tU9v+/3vb9f5y/bB2Plrt9HmzU16OV1KBIHrJEzNluKrnWfR9q1NWK3SBNjKGJwQybQ9NdvRTbCINR0EzjTstnjXOfSem6C37b8mLt4Vyx7cKSqVPI1UbbkNH21TfuaVpcoEtLOobE0IIFtGIrEtViiv7NpNZRccnboiRX+Dzlvv9bVHJS+/UDHN/qUf1NnDWRmDE1IFRy9/LsdyhdbIsLele85bNNPI1krLBGauPippETcpZq+Xnz917HIuWr+5Ea3f3GhxcrjGTBfEtZuF2HQ8EyV3H99WAY4tHvVOUamsYRJDs6g+NlMvJzuvAI3/9Ssa/+tXq1buNmavnQIme1q2Nx2PfqF8crcayApOFixYgKioKPj5+cHPzw8xMTH47bffTN5n1apVaNWqFby9vdG+fXts2LDBqgaTa3L2oR5nUVBsu2nElvrtWAa+35+OV3RyVir38nyWkKbojLLKdEvhbz8lrWdM7nTaBz/ZjWe/TcZXu86hrExg0S7HTaPdeExenkfvudtx//vbTVYBNueDzX+avD12zjbt75bmLxmLD4WA5KKD1tINguwxnf4vC3pqhBCKrh9kC7KCk/DwcMyZMwfJyclISkpCv379MHz4cBw/ftzg/nv27MHo0aMxYcIEHDp0CCNGjMCIESNw7Jj1aw0Qke2YKuKmNCkfku9vTMV3JvNllLsKPPNtMnZVWlTtP+tP6NUkAYARn8pb9fry3fojG49lYv3RDOQVOC45Mf63U7L2r1h/aMtJ2y1OWGLBlVxu2QJLyTnMhzpBmKlk6z+zbuLwxRwrWmU5NeXBGSMrOBk2bBgeeOABtGjRAi1btsS7776L2rVrY+9ew4WQ5s+fj8GDB2P69Olo3bo13nnnHXTq1AmffPKJIo0nItuw1donhsRvkHah/DPb+Ni60qsiP7FIP7nyq13nMHO1/qyNE5WK9smZ9WPLIQZLE1wzcwvskpNRmbEjbpawSvMPElbMBqzLt5Ja08USUld71y3Glu/AoNaeLM45KS0txYoVK3Dr1i3ExMQY3CcxMRFxcXF62wYNGoTERNccIyMi+YoUGK6Rs66QpY5d1glGrEjsSLmYo7q8pT1pf6F7/FY0e20DPt56Wq+CbrqEXJPEM8oHWxO/SYIQAt/uveDwSqlKrVz9S6XaNrrVfLPzC3BNgUUHXYXsImxHjx5FTEwMCgoKULt2baxZswZt2rQxuG9mZiaCg4P1tgUHByMz0/R4Z2FhIQoL771IeXmWj3MSkeOUyAw8ks5fR5fGdQ3etnxfOmpbWP3S3BdnJfoL3jZQnM4ZSqhooME3ifeGzD7Y/Cc+2Hzvdil5SqO/tGwZAXN2pf2FN9aWpwHIqZRa2Y3bxvMyhBBmk5ktVfnlf+H7Q3p/V3RU3SkqRdd3twIAzr73gE3a4mxk95xERkYiJSUF+/btw6RJkzBu3DicOKFcdVEAiI+Ph7+/v/YnIiJC0ccnIvuQO3PG3CKPUlakdpRzBtruiGGSCrl3ig0msJqbNaOUPWesry1zXqFFP38/kYXCEsO9a0Pm78TfF+xBrg0qBy9IkLYshu50aCXWvnKFwoKygxNPT080b94cnTt3Rnx8PKKjozF//nyD+4aEhCArS3/cMCsrCyEhISaPMXPmTOTm5mp/Ll50/ZU9iVyR7jdyOeR8uDpiaQCpwwxHL+eitEwonhNjWnkvQI852/DARzuRfOFeAb0rOXcMzprZeFza7B0517yfki/bpedoxQFp1wdj5QpOZeYj6cINRM/+HYfSbxhdhsESUvJmbKHre1v1EnOdkdV1TsrKyvSGYHTFxMRg69atets2b95sNEelgpeXl3a6csUPkdqpsYaIs3pq6QHJ+1YskmbOR1tP46UfUuw+1DLs411o/eZGux7zTlEpbt4tVZ6gUzTQ2kTnNIkFv5Rys9B8e5WqjwMAD322p0oytL0p8f68ml+Ij+72kBWVlOHPLOcpolhB1gDuzJkzMWTIEDRs2BD5+flYvnw5EhISsGnTJgDA2LFj0aBBA8THxwMApk6dit69e+ODDz7A0KFDsWLFCiQlJWHhwoXKnwmRgw2ct8PRTXAZtkhwrfgm2btlfYvuf1kneVGOyrN67OGNdbYp11A5oVMRQuDY5VyDN/3fRnlTnp2tFH8FS99bugwFNZtPZFm0lpYayApOsrOzMXbsWGRkZMDf3x9RUVHYtGkTBgwYAABIT0+Hm9u9zpjY2FgsX74cr7/+Ol577TW0aNECa9euRbt27ZQ9CyIVkDKrgUx7a91xDO8QZtNjFBpI8FyXYpsVgh3FWdZPqXD8iuHgRNf0VYftUtTMEXaeti4/5531J5BioGaKscDktl2HGS0jKzhZtGiRydsTEhKqbBs5ciRGjhwpq1FEVD3dKS7FzJ+OOroZTs3Zeg/Sr99GUan5qGOVlcM3f7nwNN1Fu+RVv12xX11T2Q3h2jpEpCqrDymzCquumzqFq5RYKdaVODqYuXG7WDtd2Fqm1iu6YYPZOEqx90tQaMOlIJTC4ISIXN5pE9VlXc2twhLVDH/YuxlK5G6QOjA4IaJqxQVKQJhUOZ/A3PkmpF61YWvsyxmTP6/k3MFnEuuhKCXhlPpfcwYnRFStuHhsUsWiXecwaVmy0Wq9znhBdyW6qzED9gmeU51gajGDEyIiF3anuBS/HcuUXGiNHMveNXHUisEJEVUrzliQSgnOMH2UqAKDEyKqVpbsPu/oJjjE/C2nkXfHvjNWXD2/h2yHwQkRUTVwOecOZv183NHNIBWK33DS0U2ogsEJEVE1cfiS+UqsVP18seMski/YfwFNUxicEBGR4n46eAmfbE9zdDNIokcWJCJNRfWAGJwQEZFN7PhT/fU06J7UTPUkizM4ISIiIlVhcEJERESqwuCEiIiIVIXBCREREakKgxMiIiKCUNHKUwxOiIiISFUYnBAREZGqMDghIiIiVWFwQkRERLh8446jm6DF4ISIiIhwq6jU0U3QYnBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiAgSLsBEREREZxOCEiIiIVFS8nsEJERERQVWjOgxOiIiISF0YnBARERFXJSYiIiIyhsEJERERMeeEiIiIyBgGJ0RERKQqDE6IiIhIRemwDE6IiIgIzDkhIiIiMorBCREREbHOCREREZExDE6IiIgIeXeKHd0ELQYnREREhOJSDusQERERGcTghIiIiDiVmIiIiNSFs3WIiIhIXdQTmzA4ISIiInVhcEJERESqwuCEiIiI1DSqw+CEiIiIAKGi6ToMToiIiEhVGJwQERERh3WIiIiIjGFwQkRERM5bITY+Ph733XcffH19ERQUhBEjRiA1NdXkfZYuXQqNRqP34+3tbVWjiYiISFkqik3kBSd//PEHJk+ejL1792Lz5s0oLi7GwIEDcevWLZP38/PzQ0ZGhvbnwoULVjWaiIiIXFcNOTtv3LhR7++lS5ciKCgIycnJ6NWrl9H7aTQahISEWNZCIiIisjmXmUqcm5sLAKhbt67J/W7evIlGjRohIiICw4cPx/Hjx03uX1hYiLy8PL0fIiIish31hCZWBCdlZWWYNm0aevTogXbt2hndLzIyEosXL8a6deuwbNkylJWVITY2FpcuXTJ6n/j4ePj7+2t/IiIiLG0mERERORmNsLAfZ9KkSfjtt9+wa9cuhIeHS75fcXExWrdujdGjR+Odd94xuE9hYSEKCwu1f+fl5SEiIgK5ubnw8/OzpLkGNf7Xr4o9FhERkTMb2j4Un47ppOhj5uXlwd/fX/b1W1bOSYUpU6Zg/fr12LFjh6zABAA8PDzQsWNHpKWlGd3Hy8sLXl5eljSNiIiInJysYR0hBKZMmYI1a9Zg27ZtaNKkiewDlpaW4ujRowgNDZV9XyIiIrKNMhUlxMrqOZk8eTKWL1+OdevWwdfXF5mZmQAAf39/+Pj4AADGjh2LBg0aID4+HgAwe/ZsdO/eHc2bN0dOTg7mzp2LCxcu4Omnn1b4VIiIiMhSKopN5AUnCxYsAAD06dNHb/uSJUswfvx4AEB6ejrc3O51yNy4cQMTJ05EZmYm6tSpg86dO2PPnj1o06aNdS0nIiIilyQrOJGSO5uQkKD397x58zBv3jxZjSIiIqLqi2vrEBEREYSKKp0wOCEiIiJV5ZwwOCEiIiJVYXBCREREqsLghIiIiFSUccLghIiIiMCcEyIiIlId9UQnDE6IiIiIPSdERERExjA4ISIiIhUN6jA4ISIiIpVhcEJERESS1s+zFwYnRERExGEdIiIiUpcyFUUnDE6IiIhIVRicEBERETSOboAOBidEREQErxrqCQnU0xIiIiJymNahfo5ughaDEyIiIuJsHSIiIiJjGJwQERGRqlb+Y3BCREREqsLghIiIiJhzQkREROqiolEdBidERESkLgxOiIiISFUYnBARERGEirJOGJwQERGRqjA4ISIiIlVhcEJEREScrUNERETqoqLYhMEJERERqQuDEyIiIuKwDhEREZExDE6IiIhIVRicEBEREYuwERERERnD4ISIiIhUNZeYwQkRERGpKTZhcEJERESAUNFcYgYnREREpCoMToiIiAgajcbRTdBicEJERESqwuCEiIiImHNCRERE6qKi2ITBCREREXEqMREREZFRDE6IiIhIVRicEBEREXNOiIiISF24KjERERGREQxOiIiISFVkBSfx8fG477774Ovri6CgIIwYMQKpqalm77dq1Sq0atUK3t7eaN++PTZs2GBxg4mIiEh5Tptz8scff2Dy5MnYu3cvNm/ejOLiYgwcOBC3bt0yep89e/Zg9OjRmDBhAg4dOoQRI0ZgxIgROHbsmNWNJyIiImWE+Hs7uglaGmFFvdqrV68iKCgIf/zxB3r16mVwn1GjRuHWrVtYv369dlv37t3RoUMHfP7555KOk5eXB39/f+Tm5sLPz8/S5lbR+F+/KvZYREREzmz5090Q2zxQ0ce09PptVc5Jbm4uAKBu3bpG90lMTERcXJzetkGDBiExMdHofQoLC5GXl6f3Q0RERNWDxcFJWVkZpk2bhh49eqBdu3ZG98vMzERwcLDetuDgYGRmZhq9T3x8PPz9/bU/ERERljaTiIiIJFBRyonlwcnkyZNx7NgxrFixQsn2AABmzpyJ3Nxc7c/FixcVPwYRERGpUw1L7jRlyhSsX78eO3bsQHh4uMl9Q0JCkJWVpbctKysLISEhRu/j5eUFLy8vS5pGRERETk5Wz4kQAlOmTMGaNWuwbds2NGnSxOx9YmJisHXrVr1tmzdvRkxMjLyWEhERUbUgq+dk8uTJWL58OdatWwdfX19t3oi/vz98fHwAAGPHjkWDBg0QHx8PAJg6dSp69+6NDz74AEOHDsWKFSuQlJSEhQsXKnwqREREZCmnrXOyYMEC5Obmok+fPggNDdX+rFy5UrtPeno6MjIytH/HxsZi+fLlWLhwIaKjo/Hjjz9i7dq1JpNoiYiIyL7UtLaOrJ4TKSVREhISqmwbOXIkRo4cKedQREREVE1xbR0iIiJSFQYnREREpCoMToiIiMh5E2KJiIiIbI3BCREREUGjcXQL7mFwQkRERKrC4ISIiIiYc0JERETqoqLYhMEJERERqQuDEyIiIlIVBidEREQkaYkae2FwQkRERKrC4ISIiIhUhcEJERERqQqDEyIiIlIVBidERETEOidERESkMiqKThicEBEREcCF/4iIiIgMY3BCTqFNqJ+jm0BE5No4rOMc/hnX0tFNICIiqnYYnBhxcvZgTI1r4ehm0F3ubioaDCUiIpticGKEj6e7o5tQxQv9mju6CQ7z/t+jHN0EIiKyEwYnTiSiTk1HN8FhWjPnhIjIpur7ejm6CVoMTszoG1nfZo/do3k9fDamk80en4iISCo1TTxgcGKGu5vtnqLvnu6OB9qHSt4/MsTXZm0x5uGODex+TCIiD3fmmVVnDE7s6Junuhrcvub5WLP3/fqprqjlpUwezMOdGHAQkbppNAxOqjMGJ3bUq6XhIaKODeuYvW9vI/eVq0ujOhjYJliRx3owOkyRx7GXuNbB8POu4ehmEJFKhNfxcXQTyAgGJ9XMj5NiUUPiUNX8xzqYvP2VgZEY1SUCAPBMr6ba7bW9pAUA5+IfkLTf+NjGkvYzx9e7hmpqDHVrUheT+zbD9EGRjm4KkSrZut+keVBt7JrRz8ZHIUsxODHg3w+01v5uTfby8bcH4UUnnv47vIPp4Z+Iuj5496F2WDe5B2YMbqXdLrUkiUajwQ/Pxpjd742/tZH2gFaKCvfHNDvVtvH19sD0Qa0wua/zvj/I9pz580PtRndt6OgmkAkMTgwIC7jX1VenpofFj1PLqwa8VVAv5fFulv8Tjrvba9GpYYDe9nmjoqHRaFDD3Q3REQEWF0nr2qSu2X2UKsDWqJ7pqdjP9mqGaTKqAtsjkNn5al+bH8NWmgbWMrg9oi670qV6tnezalnjR+1DxjVV8LluC2pK82FwIpOnu/qfsn2v9cfi8V3QIqg2fp7SA/0ig/RulzO0ER0RgJQ3B+DLsV0Ua9/org2xfGI3xR7PkLeGtakynvxc72aKfuhNvL+p+Z10NAiQf1GOqFs1oLLkcSrc19h8fpMSXh/aGp41pP2vBNb2tHFrnJeHE3ze2IK5IWVHerhjA0WXNglSUW0RName73wFTYtrgXmjorFpWi98/kRn7fb/jGgHABB2TnII8vVCsJ83+rUKxuaXeiMqPKDKPkJmowJqelbJnO/apJ7J+7z3UHujt8U/3B6xzQJltUGuJ3s0qfIB5+3hbrchIl1HZw3EqXcGY9eMe70gIzpaHiTNGNLK/E5GaOy0JvrT9zeFn7fhXsfKb79xMY2x/oWeePNvbfAvK87NHqb0bY7+rYJM7qPb0+clMUAzZPH4LpIDPFfSs3kgNBqNavLDKvvnAMOBSeWyCyM7h+PZ3lW/wHytM2vzwegw1KvN4MSQ6vfOV1iInzce6hiOyBBfDG4Xot1ubNpvxQVzyfj77NE8xVQeWjH07f3JHo0BAK890BqPd2uIyGD9uiz+Ph7Y+nJvm7VRCm8PdxydNdDmx/HxuPf6+3p7wNvDXS/Aq1vT8t4CtXR5mwv0DA1HGApaPWq4oV0DfzzVswm8VX4xfmVQpKRhxmd6NcWD0WGYPbytxcfq16p8Vp2/j+VDy7bw9VNdcfY9acnslniks7pLHRjqzQRQJYP3/x6JwvN9quYM6c68VNMwitqo+5PAjlY9Zz4xUwnDOzTA2fceQF8z377URsoH5Jt/a4P9r/XHY0YSzXbO6Itm9WtLOt7LBr6dGOpKHSMxn2bFM921v/sa+UavFs4yzFGvlul2Ng6shVo6Y/NP9mhsVf6TrZga6tr3Wn/t77++2BNA+XmZ89oDrfHR6I6SygSY466yK1jvlvXhZoeFOIdFqSMIB1Dli5ZBlbp67PEcmWKq99oZMDi5677G9xIzTb2nlPickPumbddAHSWFn+1lOsdCo9EgyM/b4G2N69U02s1vSPOgqkGMoYTWLiZzKO49z92bmh6GUpqwUaf0vFHRAIAv/tHZzJ62FervjYZmEowB4LepvbS/G5tibs+egSMGes10//crC/bzxp//GYKDbwxA2zB/AMCL/VvgH90bGb2Pbi5SSykXNTPq1DL//CiRZCwlOd1alRPrDakYepz1oLQhWKmlC3T5+3iYrX4d5n/vsyw6wl/2MZQmt9aVGr8IyMHgRMe4mEaIDvdH/9aGi5S9+1C7Ktu8PGz7FP40KRYrn5Heq2PLcdpJfZohKtxfct6G7gVa7jdIa4LAFgYCm8p6Nq+a8yI397CGncpr7/lXeS2Gfq2C8FDHcADAoLYh+Oaprvj1xZ74fmJ3g/eTOvzTu2V9rHymO1ZLqFSsS8qFVzeA6RARYHAfpYoCSmEoQDaXdOpZww11dXqJanvVwDsjqn4WAMDE+5vglYHGkyVjmtbTnu/QKGlLV3RqWAdTjEw5n/v3KPwzriVWyPiMMOaHZ2Pw+RO2XeurSaC0nlOg/HmWMoRmag9DU7F/mhSLg28MQB0zPX/bp/fR/j6gTQjmP9YBW14qD7a7Na0ayD3Zowk2Trsf7Rr4YcmTyg/bazTltaHO2HBITU0YnOh4e3g7rJvSUy8JTfciOaab/relns0DMbS9bbseOzeqg1p3vxk0DayNaCMf8HJYGsAE1PTEz1N6YkLPJrLvO+tBeWPv1iSJfTvB/Ewg3cBpfGxjdIgI0I7xS+VVw93okgRKCgvwwal3BmPROP0ZU71a1kfbMH/ENKtXpZhbqxBffDS6o6SZPRoN0K1pPXSyYgiibdi93r3Ks6QSXumDz5/ohH5GhjLdZEaivpWq/B58Y4DB/f4m8eL/lAXvZ2O6NamHGiaCne+f6Y55ozrg8yc6Ya7EKcIajQavDIrE0KhQ+N79LPD2cMPZ9x7AyC4RmBrXwqoZXJYy15NqLY1Gg2OzBuHY2+U/xowzUaTRy6Nq7l/nRnXMBj3REQHwqnHvvhqUD8k3DyoPyA1NNAio6YFWIX5Y/8L96Bspbdi+fm0vvHr3f/eJ7qZ7OjQof06UKq2gdgxOrLDs6W5Gs+nr1y7vElRyCWo3Nw3WPh9rcMhDKVHh5d2XSo9Xyum63zWjr15CqSmVZ/3c3yIQIf6Gh5aMmfVgW6yd3EP7WhrqVTHG2JIEAT7K5o1UTqg1p2LfLS/1tqpWjy5T1Wxf1SnCV7knp3FgLQxuF2q0/XKD5RGVigMaqznxyePSegH8fTzQSoFFNbs1qYs+ElYxr+VVA4PbhaKmp7zhiE8f74QjswYi7d0hOPXOEIfnNMx8oDUSZ9q2wqqPpztqe9UwOnSz8pnuNqk3tGaSvF5Eub4c2wV/iwrFi3Et0LdVEA6/ORCzHzTcG2cJtSVRW4LBicIWjOmEqf1boEfz8hyHhzs2wD+6N8JnY6R3l5r6Nq7RVJ0MqsQHa4XxsY1x6p3BDh2vDK9jOJfB0LUt2M8bya/HKXr8j0Z3NLjd0LCeMUOjQjGyczjef0T/23FFsmu7cMNj2Muf7gYlC3f7eLrj5YH3gorY5vJyb2J0cnV0q9mamo1u7lt8hJHXVwmWTr1VIiF+5bMxJntNDE0r3fZyb6PDcoZUFD6UKzoiABPvl9JDJO+9F+rvIzvfRalgeXxsY3RrarqnylK2DvwGtAnGJ4930g4z+tf0UHTmTv/W5ntuwvy9EaDQa2ELDE4UNqR9KP45oKX2W2INdze8M6IdHmgvrYsZAOrJnK2hxOqdTevXwkMdG+DB6DB4S+y1UAul6wTUNTIWXa+W9OPUcNNg7shoPHpfhN72Pf/qj6OzBhrMfXhlYEvEyui1kWp014b45PGO2P2vfpjUpxn+75H2mCmhnsibf2sDY8swVU74bRpYC61CfNG1SV2zH7IfjopGd50xe926O9bOYvu/Ryzr8bPHe76bgYTTpvVrI6aZ7ZO1fTzcMMTEZ1DnRsaH9Kz5Fv5xpUC/lpe73nCJIVI/znSHin+bej/eGmb/Gkbm2KNoZ5/I+pjzsP77XspQ6dKnukqe7egIDE6cULsGymeOj76vIeaN6qDot5CKtXmU7Nlxdp413Ow+ldndTYO/RYWhQYAPvGq4Y9R9DdFAJy/E0MfY/0Z1MJuLoRtUuLtpsOHF+7Hyme5mg+VQfx98P7E7+rUKQt/I+nrBYKN6tbT1cowet1JgpNuLY2mROQ93Nyyb0A2Lx1tWCdnDTsnRtlIRLFryPcfYVOdHOoVjWKWk7Kn9lRmC+anSsEvrUD882aPq+9WSInhvy8yPM8XH011Wj2tl5T2pptVw0xgt3/D60PJ14oxVtH2hXwvVrjHE4ESmOlYUz5KqobEiP3fNGtZWr4u4QYC8HAt7ebZXUyx58j6slLC4n6sJc0CCopLkdmtrNOX3kdqLp9FosHj8fVjyZNcq9wk2Mh1divbh/gis7YVoI8NmpvRsESg7KbqCud4AezCW/wTA4IVbqfVhAmR8Jtar7WX1NPvWoX4me3oqdIgIwOPdGsrK+5vct5nBBFtrWlx5IoVU5+If0OtJNfa/ZWqB1qfvb4r9/+6PqUbycrw93DGhZ2Pt3/auaG4KgxOZFo+/D9Hh/lgmYUaIpXy9PbB3Zn+jsxD8a3pg5pDWWP50NwxuG4J3ZSav2qMbGSgf0uobGWS2W7hy168aDG4bUmWblETk4R3C8HTPJrJydiryJO5vIa+OgS30iawPX+8a6Hs3sbO5xKJ5thLi5437GteRHGx4urth78x+WPN8D4uPWbGOVA+Z+TmO9vkTnfDq4EiMi9G/GO6a0ReDDLyf1cpczpKp8Lfivm1C/bB2cg/U9KyB7a/0wc9TemBEhzB88rjpz5onTNSvMUfpC7vUQN9cDacgX8PBfsXQT8UsUHsV15NKfvWaaq5NmB/WTelp8+NImXES2zxQUo5C5SRBP28PnHpnML5NvIB3N5wEYPt6LaYMMFDnQm73cnS4Pw5fysXILvdyPKxZUGvBE53w/f6LWHkgHTMGtwI0QLP69yqDGprS/f4jUVVyTKTY/1p/XMkpQBvtdFzbf32JahCg/V0332LJ+PtQUia0tT9eGRQJjUaDBzs4plrn+hd7IrC2Fx76bLd2W10zuT/mhib3/KsfYudsM3r7gDbBODl7MLzt+D/h7eGGguIyqx6jpmcNPN+nOb7ec15ve0WCuZ+3bT7udd+t4XV8cOnGHQDy/4ffGd4W0GjQxURRPHO+n9gdXyee1yt3UNurBqLCA/C/x0wHJs2DaiPU3/l6PC3piXqgfYj286xiFqjaMDipBno0D0Rc6yBE6uR+eHu4Y2KvpigqLcMfqVfxaBf5F1WleHu4I651MLaczLL4MVY+G4OzV2+hdei9c4yoWxPzH+tgcChuUu/m2J12zWgtDI1Gg8e7NazSA3LojQE4c/WmIvVmKgTU9JTVNS5FezNVhRvWq4nxsY2xLuUyXtcpqqfRaPTyJ3y9PQzWqLFl92+oTmAeaCDZeWxMI3y09bT2b932Spl9EBbggxpuGpSUGT8JH4WGPKRa+UwMnlp6ANduFQHQr04q14PRYXjr5+NVtjcP8sX0QZGYuym1ym2RIeXvF93Ceh0bBqBHs0B8u/eCdpu553fJ+PswYN4Oi9od0yzQ6jIJDevVtHhxzxoO7jXQaDRoElgL5/66ZfVjjYtpbPS21c/HVqlppMSkCqUxOKkG3N00+Gqc4YqFk/s215si6ihS8nCbmqgu6e3hrtPzcI+x8dieLQKR/Hqc0Zk5xtSp5YkutQx/swv0VceaOJP7NsOzvZuZ3W/Wg21lF8fTpZtnIWdpAnOGRYXhz6x8dGlk+HmuPLOmhrsbfn2xJ0pKharXTTKVyB4dEYDVz8ei99wEAOXF2ixVp5Yn1r/QE7PXn8BLldaomty3uV5wsv6Fnjh0MQfD7gbpTQJr4YdnY1Cvtqd2HaxvEs9r9/9jel9UoROptpBZrr++rxeu5hfKuo8ri21Wz2hwIjVgjm1WD+0NDIPu/lc/XLx+26pii/bE4IRUyVDQ0D7cH58+3gllQmDjsUyDdSPkUGoK8sejO+LYlVzJVSFtbfog89OEleBZww3rX+iJkjKhrWKsBDc3TZVzMPe9rmLdG0cRJrqSDr81ELcKS4yO/RtiTVIwUB4I/SAhEb1dA/8qQZOpNXbkTCuW8l18yz97I3r273f/cmw2php6D14a0BKnMvPxaJdw7bb3H4nC0j3ntTNvzDH2GjUI8HFIJWFLVevgpCJPgdQn1N8H8x/rUOUbecV6JJWnKDrSsOgwVbXHnmwxrd3V+Pt4uETFTmOsCSkqL0XgzIL8rP+yU6+2V5Vp0o/eF2Eyl03u8g/OolrP1hnpwDwLpVWstxDlQheL4R0aWF2Ui8hZ6A6T2eOC46yXNFs9NZ4matUY6hmL06nCmvR6nF2LV1Ys0DikXUiVvCxXiVVcJ2y1wGP3ReBWYYndptba0qZp92PlgYt4TkKugRqpaX49GTc0KhS/Hsmw+/tMDV3uthbi740pfZvDx9Pd4jL8jmDsf9dZeov+OzIa87f+if+OjJZ1vxmDW2HLyWwA5s+1bi1PXL+b7KyEwe1CcX7OUMUeT42qdXBSw93NbOKgLdcBUVLzIF/8e6j6yjeTa5k/qgOm9m+BFjZcfNKQ9x5qj1ELE/FCP8srjP5ihxIA1nrFxOKKalV5KusHI6OxNuUyXjBSDVZtX0T+3jkcf+8cbn7HSnRrgpgLnZdN6IYHPtop+xjVmezwfMeOHRg2bBjCwsKg0Wiwdu1ak/snJCSUL1ZX6SczM9PSNtvV3zuH4/k+zfDtBOOL8RFVFzXc3dAy2NfuPRmRIb44+PoAvfoVcgxpF2JwBoNSOjQMsNljO5tHOofj2wndtL0J74xQbrVdZ9UmzA+Lx3fBxmn3O7opTkN2z8mtW7cQHR2Np556Cg8//LDk+6WmpsLP795Uz6Ag58glqOHuprccPBE5RsU3Vd1yFC/2Mz0N/pcpPbHiQHqVKbVNAmvhdPZNxdr2v1Hqq3JsL+Z6QupLXMhUbT0qSrN0aYTqSnZwMmTIEAwZMkT2gYKCghAQECD7fkTVTWyzQPx8+AoCZaxOXQ1SMrRqetbA+NjGKCwpw0sDTQ+DtA/3R/vwqss7fDm2C97fdAqTeltf4+f5Ps1krd9S3cS1Dkb/VkEGCxdWp/ctyWO3nJMOHTqgsLAQ7dq1w6xZs9Cjh/G1LwoLC1FYeK8wT15enj2aSA40vEMD/H4iC011SsRXV/95qB3ahPkZrV5LsKp4HAA0DqyFz8Z0tuoxNJryb/tqWBPJkcz1eNRwd8Oi8YaLQBIZY/OU8NDQUHz++ef46aef8NNPPyEiIgJ9+vTBwYMHjd4nPj4e/v7+2p+ICNeZ8mtMRfe0s862sdYD7UPw85Qe+NkJkhZtzc/bA8/1bqZdE4XU6cC/4/DTpBiXmO1nynsPl/c8TTOysq29qaW3RW3DUK1Dy9MmHu4oP7lXjWzecxIZGYnIyHtdr7GxsThz5gzmzZuHb7/91uB9Zs6ciZdeekn7d15enssHKP8c0BIPdQpH43rV84Kk0WgQFR7g6GYQSRZY28vg2j9qNzamEb5JvIDpEmcG/S0qDL1b1je6NEBMs3o4kZHnVNOf5WpQR/2VVdc8H4v067f11kdyZg6ZSty1a1fs2rXL6O1eXl7w8nK+f3prVCz6RERkS28/2BYT72+KiLrSvwiZWrPolYGRCK/jg/5WJnyqrCMCALDime64fOOO2eUR1FCHx9vD3WUCE8BBwUlKSgpCQzmeTkRkbxqNRlZgYo6Ppzue7GHZFG81XNRN6d7UtYfs1Ex2cHLz5k2kpaVp/z537hxSUlJQt25dNGzYEDNnzsTly5fxzTffAAD+97//oUmTJmjbti0KCgrw1VdfYdu2bfj999+NHYKIiFyUl4frDv+QcmQHJ0lJSejb996y2RW5IePGjcPSpUuRkZGB9PR07e1FRUV4+eWXcfnyZdSsWRNRUVHYsmWL3mMQEVH1sGBMZ0z6LhkzLKwf5UpDF2Sc7OCkT58+JpcHX7p0qd7fr776Kl599VXZDSMiItfTroE/dr7aT/b91r/QEz8dvIRp/Vua35mcXrVeW4eIiJxDuwb+aOdCq66TaRz8IyIih3OWVYzvUeP8ItfBnhMiInKYReO64GZhCYL9vB3dFIupe86Rc2JwQuQCNPx4JCfVvzUXxKOqOKxDREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERDK5qXxdIGfH2TpEREQyNQmshSHtQuDv4wE3NwYqSmNwQkREJJNGo8GCJzo7uhkui8M6REREpCoMToiIiEhVGJwQERGRqjA4IXIBrUJ9Hd0EIiLFMCGWyAX0aVkf/x0ZjVYhDFKIyPkxOCFyARqNBn/vHO7oZhARKYLDOkRERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREquIUqxILIQAAeXl5Dm4JERERSVVx3a64jkvlFMFJfn4+ACAiIsLBLSEiIiK58vPz4e/vL3l/jZAbzjhAWVkZrly5Al9fX2g0GsUeNy8vDxEREbh48SL8/PwUe1w1cfVz5Pk5P1c/R56f83P1c7Tl+QkhkJ+fj7CwMLi5Sc8kcYqeEzc3N4SHh9vs8f38/FzyDafL1c+R5+f8XP0ceX7Oz9XP0VbnJ6fHpAITYomIiEhVGJwQERGRqlTr4MTLywtvvfUWvLy8HN0Um3H1c+T5OT9XP0een/Nz9XNU4/k5RUIsERERVR/VuueEiIiI1IfBCREREakKgxMiIiJSFQYnREREpCrVOjj59NNP0bhxY3h7e6Nbt27Yv3+/o5uE+Ph43HffffD19UVQUBBGjBiB1NRUvX369OkDjUaj9/Pcc8/p7ZOeno6hQ4eiZs2aCAoKwvTp01FSUqK3T0JCAjp16gQvLy80b94cS5curdIepZ+jWbNmVWl7q1attLcXFBRg8uTJqFevHmrXro1HHnkEWVlZTnFuFRo3blzlHDUaDSZPngzA+V6/HTt2YNiwYQgLC4NGo8HatWv1bhdC4M0330RoaCh8fHwQFxeH06dP6+1z/fp1jBkzBn5+fggICMCECRNw8+ZNvX2OHDmC+++/H97e3oiIiMD7779fpS2rVq1Cq1at4O3tjfbt22PDhg2y2yLn/IqLizFjxgy0b98etWrVQlhYGMaOHYsrV67oPYah13zOnDmqOD9z5wgA48ePr9L+wYMH6+3jrK8hAIP/jxqNBnPnztXuo+bXUMp1QU2fnVLaYpaoplasWCE8PT3F4sWLxfHjx8XEiRNFQECAyMrKcmi7Bg0aJJYsWSKOHTsmUlJSxAMPPCAaNmwobt68qd2nd+/eYuLEiSIjI0P7k5ubq729pKREtGvXTsTFxYlDhw6JDRs2iMDAQDFz5kztPmfPnhU1a9YUL730kjhx4oT4+OOPhbu7u9i4caN2H1s8R2+99ZZo27atXtuvXr2qvf25554TERERYuvWrSIpKUl0795dxMbGOsW5VcjOztY7v82bNwsAYvv27UII53v9NmzYIP7973+L1atXCwBizZo1erfPmTNH+Pv7i7Vr14rDhw+LBx98UDRp0kTcuXNHu8/gwYNFdHS02Lt3r9i5c6do3ry5GD16tPb23NxcERwcLMaMGSOOHTsmvv/+e+Hj4yO++OIL7T67d+8W7u7u4v333xcnTpwQr7/+uvDw8BBHjx6V1RY555eTkyPi4uLEypUrxalTp0RiYqLo2rWr6Ny5s95jNGrUSMyePVvvNdX9n3Xk+Zk7RyGEGDdunBg8eLBe+69fv663j7O+hkIIvfPKyMgQixcvFhqNRpw5c0a7j5pfQynXBTV9dpprixTVNjjp2rWrmDx5svbv0tJSERYWJuLj4x3Yqqqys7MFAPHHH39ot/Xu3VtMnTrV6H02bNgg3NzcRGZmpnbbggULhJ+fnygsLBRCCPHqq6+Ktm3b6t1v1KhRYtCgQdq/bfEcvfXWWyI6OtrgbTk5OcLDw0OsWrVKu+3kyZMCgEhMTFT9uRkzdepU0axZM1FWViaEcO7Xr/IHf1lZmQgJCRFz587VbsvJyRFeXl7i+++/F0IIceLECQFAHDhwQLvPb7/9JjQajbh8+bIQQojPPvtM1KlTR3t+QggxY8YMERkZqf370UcfFUOHDtVrT7du3cSzzz4ruS1yz8+Q/fv3CwDiwoUL2m2NGjUS8+bNM3oftZyfEIbPcdy4cWL48OFG7+Nqr+Hw4cNFv3799LY502tY+bqgps9OKW2RoloO6xQVFSE5ORlxcXHabW5uboiLi0NiYqIDW1ZVbm4uAKBu3bp627/77jsEBgaiXbt2mDlzJm7fvq29LTExEe3bt0dwcLB226BBg5CXl4fjx49r99E9/4p9Ks7fls/R6dOnERYWhqZNm2LMmDFIT08HACQnJ6O4uFjvmK1atULDhg21x1T7uVVWVFSEZcuW4amnntJbtNKZXz9d586dQ2Zmpt5x/P390a1bN73XLCAgAF26dNHuExcXBzc3N+zbt0+7T69eveDp6al3Pqmpqbhx44akc5bSFiXk5uZCo9EgICBAb/ucOXNQr149dOzYEXPnztXrLneG80tISEBQUBAiIyMxadIkXLt2Ta/9rvIaZmVl4ddff8WECROq3OYsr2Hl64KaPjultEUKp1j4T2l//fUXSktL9V4kAAgODsapU6cc1KqqysrKMG3aNPTo0QPt2rXTbn/88cfRqFEjhIWF4ciRI5gxYwZSU1OxevVqAEBmZqbBc6u4zdQ+eXl5uHPnDm7cuGGT56hbt25YunQpIiMjkZGRgbfffhv3338/jh07hszMTHh6elb50A8ODjbbbjWcmyFr165FTk4Oxo8fr93mzK9fZRXtMXQc3bYGBQXp3V6jRg3UrVtXb58mTZpUeYyK2+rUqWP0nHUfw1xbrFVQUIAZM2Zg9OjRegukvfjii+jUqRPq1q2LPXv2YObMmcjIyMCHH37oFOc3ePBgPPzww2jSpAnOnDmD1157DUOGDEFiYiLc3d1d6jX8+uuv4evri4cfflhvu7O8hoauC2r67JTSFimqZXDiLCZPnoxjx45h165detufeeYZ7e/t27dHaGgo+vfvjzNnzqBZs2b2bqYsQ4YM0f4eFRWFbt26oVGjRvjhhx/g4+PjwJbZxqJFizBkyBCEhYVptznz61edFRcX49FHH4UQAgsWLNC77aWXXtL+HhUVBU9PTzz77LOIj49XVUlwYx577DHt7+3bt0dUVBSaNWuGhIQE9O/f34EtU97ixYsxZswYeHt76213ltfQ2HXB1VTLYZ3AwEC4u7tXyR7OyspCSEiIg1qlb8qUKVi/fj22b9+O8PBwk/t269YNAJCWlgYACAkJMXhuFbeZ2sfPzw8+Pj52e44CAgLQsmVLpKWlISQkBEVFRcjJyTF6TGc6twsXLmDLli14+umnTe7nzK9fxWOZOk5ISAiys7P1bi8pKcH169cVeV11bzfXFktVBCYXLlzA5s2bzS4r361bN5SUlOD8+fMm267bbkeeX2VNmzZFYGCg3nvS2V9DANi5cydSU1PN/k8C6nwNjV0X1PTZKaUtUlTL4MTT0xOdO3fG1q1btdvKysqwdetWxMTEOLBl5dPMpkyZgjVr1mDbtm1VuhENSUlJAQCEhoYCAGJiYnD06FG9D5OKD9Q2bdpo99E9/4p9Ks7fXs/RzZs3cebMGYSGhqJz587w8PDQO2ZqairS09O1x3Smc1uyZAmCgoIwdOhQk/s58+vXpEkThISE6B0nLy8P+/bt03vNcnJykJycrN1n27ZtKCsr0wZmMTEx2LFjB4qLi/XOJzIyEnXq1JF0zlLaYomKwOT06dPYsmUL6tWrZ/Y+KSkpcHNz0w6FqPn8DLl06RKuXbum95505tewwqJFi9C5c2dER0eb3VdNr6G564KaPjultEUSyamzLmbFihXCy8tLLF26VJw4cUI888wzIiAgQC+T2REmTZok/P39RUJCgt6Uttu3bwshhEhLSxOzZ88WSUlJ4ty5c2LdunWiadOmolevXtrHqJgyNnDgQJGSkiI2btwo6tevb3DK2PTp08XJkyfFp59+anDKmNLP0csvvywSEhLEuXPnxO7du0VcXJwIDAwU2dnZQojyKWgNGzYU27ZtE0lJSSImJkbExMQ4xbnpKi0tFQ0bNhQzZszQ2+6Mr19+fr44dOiQOHTokAAgPvzwQ3Ho0CHtbJU5c+aIgIAAsW7dOnHkyBExfPhwg1OJO3bsKPbt2yd27dolWrRooTcNNScnRwQHB4t//OMf4tixY2LFihWiZs2aVaZp1qhRQ/z3v/8VJ0+eFG+99ZbBaZrm2iLn/IqKisSDDz4owsPDRUpKit7/ZMUMhz179oh58+aJlJQUcebMGbFs2TJRv359MXbsWFWcn7lzzM/PF6+88opITEwU586dE1u2bBGdOnUSLVq0EAUFBU7/GlbIzc0VNWvWFAsWLKhyf7W/huauC0Ko67PTXFukqLbBiRBCfPzxx6Jhw4bC09NTdO3aVezdu9fRTRIADP4sWbJECCFEenq66NWrl6hbt67w8vISzZs3F9OnT9erkyGEEOfPnxdDhgwRPj4+IjAwULz88suiuLhYb5/t27eLDh06CE9PT9G0aVPtMXQp/RyNGjVKhIaGCk9PT9GgQQMxatQokZaWpr39zp074vnnnxd16tQRNWvWFA899JDIyMhwinPTtWnTJgFApKam6m13xtdv+/btBt+T48aNE0KUT4984403RHBwsPDy8hL9+/evct7Xrl0To0ePFrVr1xZ+fn7iySefFPn5+Xr7HD58WPTs2VN4eXmJBg0aiDlz5lRpyw8//CBatmwpPD09Rdu2bcWvv/6qd7uUtsg5v3Pnzhn9n6yoW5OcnCy6desm/P39hbe3t2jdurV477339C7sjjw/c+d4+/ZtMXDgQFG/fn3h4eEhGjVqJCZOnFgliHXW17DCF198IXx8fEROTk6V+6v9NTR3XRBCXZ+dUtpijubuiRMRERGpQrXMOSEiIiL1YnBCREREqsLghIiIiFSFwQkRERGpCoMTIiIiUhUGJ0RERKQqDE6IiIhIVRicEBERkaowOCEiIiJVYXBCREREqsLghIiIiFSFwQkRERGpyv8DJoZUBUW5RZgAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "plt.plot(used_lrs, lossi)" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(2.3539, grad_fn=)" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emb = C[Xtr] # (32, 3, 2)\n", - "h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 100)\n", - "logits = h @ W2 + b2 # (32, 27)\n", - "loss = F.cross_entropy(logits, Ytr)\n", - "loss" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(2.3602, grad_fn=)" - ] - }, - "execution_count": 76, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "emb = C[Xdev] # (32, 3, 2)\n", - "h = torch.tanh(emb.view(-1, 30) @ W1 + b1) # (32, 100)\n", - "logits = h @ W2 + b2 # (32, 27)\n", - "loss = F.cross_entropy(logits, Ydev)\n", - "loss" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp8AAAKTCAYAAABfKmNzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVFklEQVR4nO3de3xU9Z3/8ffM5IJAAgkxXEIkKCgihlhCEG/ddhFvtdjSeK1a16pbpVXYomJVtLZq0VW2LS2r1db+WjeCVLFKKRSl3pBAMAYEBSQJEEggJCQhaDKZOb8/6MSEzExmMiffzCSv5+PB42HOnHPmOx8jvud7vheHZVmWAAAAAAOcPd0AAAAA9B2ETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDFxPd2AUHi9Xu3bt09JSUlyOBw93RwAAAAcx7IsNTQ0aMSIEXI6A/dvxkT43LdvnzIzM3u6GQAAAOjEnj17NHLkyICvx0T4TEpKknTswyQnJ/dwa6KH2+3WqlWrNH36dMXHx/d0c2IWdbQHdYwcNbQHdbQHdYxcX6thfX29MjMzW3NbIDERPn2P2pOTkwmfbbjdbvXv31/Jycl94pe6u1BHe1DHyFFDe1BHe1DHyPXVGnY2RJIJRwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAC9lNdr9XQTOojr6QYAAADAHlsq6rR04x4VltVo54EjcnssxbscGpM+UHlZqcrPzdSEjEE92kbCJwAAQIwrq27U3ctKVFhaI5fTIU+bHk+3x9K2/Q3aXnVEL6wrV97oVC2Yma2stAE90lYeuwMAAMSw5cUVmv702yoqr5WkdsGzLd/xovJaTX/6bS0vrjDWxrbo+QQAAIhRy4srdFdBscIZ2enxWvLI0l0FxZKkGTkZ3dK2QOj5BAAAiEGl1Y2au7QkrODZliVp7tISlVU32tmsThE+AQAAYtA9y0rksSKbze6xLN29rMSmFoWG8AkAABBjNu+tU2FpTcDxnaHyeC0VltZoS0WdTS3rHGM+AQAAYszLRXsU53SoxU/4PCHepZ99a4IuPmOYGpta9Mw7uzTt9KHauq9eP319a4fzXU6Hlm7cY2wJJsInAABAjCksq/EbPCXpvktP15TRqbrljxt16Eiz5l58ms4Ykayt++r9nu/xWtpQVtudzW2Hx+4AAAAxZueBI36P909w6crJI/Xoim16/7ND+rSqQf+15CPFOYNHvh0HGrqjmX4RPgEAAGKI12vJ7fHf6zlqSH8lxrlUvPtw67G6z93aVe0/rPq4PZaxrTgJnwAAADHE6XQo3uWw9Z7xLoecTnvvGQjhEwAAIMaMSR/o93j5oaNqbvEq56TBrceST4jT6E620hybnmRn84JiwhEAAECMyctK1faqIx2WWjra7NGSjXt036Wnq/aoW4eONGnuRacp2BN1l9OhyVkp3dziLxE+AQAAYkx+bqZeWFfu97VHV2xT/wSXnrsxV41NLXr2nVIl9YsPeC+P11J+bmZ3NbUDwicAAECMmZAxSHmjU1VUXuu393POko80Z8lHrce+Pi7d731cTocmjUoxtsanxJhPAACAmLRgZrZcjsgmCbkcDi2YmW1Ti0JD+AQAAIhBWWkD9ER+troaPx2SnsjPVlYnk5HsxmN3AACAGDUjJ0OSNHdpiTyWFXCv96uf+aD1n11Oh1wOh57Iz2693iR6PgEAAGLYjJwMrZp9gSaNOjZj3RVgvU7f8dxRKVo1+4IeCZ4SPZ8AAAAxLyttgJbcNlVbKuq0dOMebSir1Y4DDXJ7LMW7HBqbnqTJWSnKz800OrnIH8InAABALzEhY1C7cOn1WsZ2LgoVj90BAAB6qWgLnhLhEwAAAAYRPgEAAGAM4RMAAADGED4BAABgDOETAAAAxhA+AQAAYAzhEwAAAMYQPgEAAGAM4RMAAADGdCl8Llq0SFlZWerXr5+mTJmiwsLCoOcvXLhQp512mk444QRlZmZq9uzZ+uKLL7rUYAAAAMSusMPnSy+9pDlz5mj+/PnatGmTJk6cqIsuukgHDhzwe/6LL76oe++9V/Pnz9e2bdv03HPP6aWXXtJ9990XceMBAAAQW8IOn0899ZRuueUW3XTTTRo/frwWL16s/v376/nnn/d7/vvvv69zzz1X1157rbKysjR9+nRdc801nfaWAgAAoPeJC+fk5uZmFRUVad68ea3HnE6npk2bpnXr1vm95pxzztGf/vQnFRYWKi8vT7t27dKKFSt0/fXXB3yfpqYmNTU1tf5cX18vSXK73XK73eE0uVfz1YKaRIY62oM6Ro4a2oM62oM6Rq6v1TDUz+mwLMsK9ab79u1TRkaG3n//fU2dOrX1+N13361//vOfWr9+vd/rfvnLX+rHP/6xLMtSS0uL/vM//1O//e1vA77PQw89pIcffrjD8RdffFH9+/cPtbkAAAAw5OjRo7r22mtVV1en5OTkgOeF1fPZFWvXrtWjjz6q3/zmN5oyZYp27typO++8U4888ogeeOABv9fMmzdPc+bMaf25vr5emZmZmj59etAP09e43W6tXr1aF154oeLj43u6OTGLOtqDOkaOGtqDOtqDOkaur9XQ96S6M2GFz7S0NLlcLlVVVbU7XlVVpWHDhvm95oEHHtD111+v73//+5KkM888U42Njbr11lv1k5/8RE5nx2GniYmJSkxM7HA8Pj6+T/zLCxd1sQd1tAd1jBw1tAd1tAd1jFxfqWGonzGsCUcJCQmaNGmS1qxZ03rM6/VqzZo17R7Dt3X06NEOAdPlckmSwnjiDwAAgF4g7Mfuc+bM0Y033qjc3Fzl5eVp4cKFamxs1E033SRJuuGGG5SRkaHHHntMknT55Zfrqaee0llnndX62P2BBx7Q5Zdf3hpCAQAA0DeEHT6vuuoqHTx4UA8++KAqKyuVk5OjlStXaujQoZKk3bt3t+vpvP/+++VwOHT//feroqJCJ554oi6//HL9/Oc/t+9TAAAAICZ0acLRrFmzNGvWLL+vrV27tv0bxMVp/vz5mj9/flfeCgAAAL0Ie7sDAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInuszrtXq6CQAAIMbE9XQDEDu2VNRp6cY9Kiyr0c4DR+T2WIp3OTQmfaDyslKVn5upCRmDerqZAAAgihE+0amy6kbdvaxEhaU1cjkd8rTp8XR7LG3b36DtVUf0wrpy5Y1O1YKZ2cpKG9CDLQYAANGKx+4IanlxhaY//baKymslqV3wbMt3vKi8VtOfflvLiyuMtREAAMQOej4R0PLiCt1VUKxwRnZ6vJY8snRXQbEkaUZORre0DQAAxCZ6PuFXaXWj5i4tCSt4tmVJmru0RGXVjXY2CwAAxDjCJ/y6Z1mJPFZks9k9lqW7l5XY1CIAANAb8NgdHWzeW6fC0hq/rxXcerY+rWyQJH3rKxlq8Vj60wflemr19g7neryWCktrtKWijlnwAABAEj2f8OPloj2KczoCvj5z0kh5vJau+PV7evivH+v754/W1ZMz/Z7rcjq0dOOe7moqAACIMfR8ooPCshq1BFlAfv/hz/XT17dKknZVN2rcsCTdfN5oFWzoGDI9Xksbymq7ra0AACC20POJDnYeOBL09Q/3HG7386bdh5WVNkCBOkt3HGiwqWUAACDWET7Rjtdrye2xd9tMt8diK04AACCJ8InjOJ0OxbsCj/eUpJzMwe1+PitzsMqqGxUoX8a7HHIGGUMKAAD6DsInOhiTPjDo6yMGn6D7LztdJ6cN0DcnjtCN52Tp9++VBTx/bHqSzS0EAACxiglH6CAvK1Xbq44E3ErzL5v2ql+8S6/OOlder6Xfv1emFwt3+z3X5XRoclZKdzYXAADEEMInOsjPzdQL68oDvt7isfTT1z/W/a9u6fReHq+l/Fz/yzABAIC+h8fu6GBCxiDljU6VK8Jxmi6nQ3mjU1lgHgAAtCJ8wq8FM7PlckQYPh0OLZiZbVOLAABAb0D4hF9ZaQP0RH62jo+fVz/zQesC88E4JD2Rn62stAHd0j4AABCbGPOJgGbkZEiS5i4tkceyAk5AasvldMjlcOiJ/OzW6wEAAHzo+URQM3IytGr2BZo06tiM9UDjQH3Hc0elaNXsCwieAADAL3o+0amstAFacttUbamo09KNe7ShrFY7DjTI7bEU73JobHqSJmelKD83k8lFAAAgKMInQjYhY1C7cOn1WuxcBAAAwsJjd3QZwRMAAISL8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGO6FD4XLVqkrKws9evXT1OmTFFhYWHQ8w8fPqw77rhDw4cPV2Jiok499VStWLGiSw0GAABA7IoL94KXXnpJc+bM0eLFizVlyhQtXLhQF110kT799FOlp6d3OL+5uVkXXnih0tPT9fLLLysjI0Pl5eUaPHiwHe0HAABADAk7fD711FO65ZZbdNNNN0mSFi9erDfeeEPPP/+87r333g7nP//886qpqdH777+v+Ph4SVJWVlZkrQYAAEBMCit8Njc3q6ioSPPmzWs95nQ6NW3aNK1bt87vNa+99pqmTp2qO+64Q8uXL9eJJ56oa6+9Vvfcc49cLpffa5qamtTU1NT6c319vSTJ7XbL7XaH0+RezVcLahIZ6mgP6hg5amgP6mgP6hi5vlbDUD9nWOGzurpaHo9HQ4cObXd86NCh+uSTT/xes2vXLr355pu67rrrtGLFCu3cuVO333673G635s+f7/eaxx57TA8//HCH46tWrVL//v3DaXKfsHr16p5uQq9AHe1BHSNHDe1BHe1BHSPXV2p49OjRkM4L+7F7uLxer9LT0/XMM8/I5XJp0qRJqqio0BNPPBEwfM6bN09z5sxp/bm+vl6ZmZmaPn26kpOTu7vJMcPtdmv16tW68MILW4c0IHzU0R7UMXLU0B7U0R7UMXJ9rYa+J9WdCSt8pqWlyeVyqaqqqt3xqqoqDRs2zO81w4cPV3x8fLtH7KeffroqKyvV3NyshISEDtckJiYqMTGxw/H4+Pg+8S8vXNTFHtTRHtQxctTQHtTRHtQxcn2lhqF+xrCWWkpISNCkSZO0Zs2a1mNer1dr1qzR1KlT/V5z7rnnaufOnfJ6va3Htm/fruHDh/sNngAAAOi9wl7nc86cOXr22Wf1wgsvaNu2bfrBD36gxsbG1tnvN9xwQ7sJST/4wQ9UU1OjO++8U9u3b9cbb7yhRx99VHfccYd9nwIAAAAxIewxn1dddZUOHjyoBx98UJWVlcrJydHKlStbJyHt3r1bTueXmTYzM1N///vfNXv2bGVnZysjI0N33nmn7rnnHvs+BQAAAGJClyYczZo1S7NmzfL72tq1azscmzp1qj744IOuvBUAAAB6EfZ2BwAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDFdCp+LFi1SVlaW+vXrpylTpqiwsDCk6woKCuRwOHTFFVd05W0BAAAQ48IOny+99JLmzJmj+fPna9OmTZo4caIuuugiHThwIOh1ZWVl+vGPf6zzzz+/y40FAABAbIsL94KnnnpKt9xyi2666SZJ0uLFi/XGG2/o+eef17333uv3Go/Ho+uuu04PP/yw3nnnHR0+fDjoezQ1Nampqan15/r6ekmS2+2W2+0Ot8m9lq8W1CQy1NEe1DFy1NAe1NEe1DFyfa2GoX5Oh2VZVqg3bW5uVv/+/fXyyy+3e3R+44036vDhw1q+fLnf6+bPn6+SkhK98sor+t73vqfDhw/r1VdfDfg+Dz30kB5++OEOx1988UX1798/1OYCAADAkKNHj+raa69VXV2dkpOTA54XVs9ndXW1PB6Phg4d2u740KFD9cknn/i95t1339Vzzz2n4uLikN9n3rx5mjNnTuvP9fX1yszM1PTp04N+mL7G7XZr9erVuvDCCxUfH9/TzYlZ1NEe1DFy1NAe1NEe1DFyfa2GvifVnQn7sXs4GhoadP311+vZZ59VWlpayNclJiYqMTGxw/H4+Pg+8S8vXNTFHtTRHtQxctTQHtTRHtQxcn2lhqF+xrDCZ1pamlwul6qqqtodr6qq0rBhwzqc/9lnn6msrEyXX3556zGv13vsjePi9Omnn+qUU04JpwkAAACIYWHNdk9ISNCkSZO0Zs2a1mNer1dr1qzR1KlTO5w/btw4bd68WcXFxa1/vvnNb+prX/uaiouLlZmZGfknAAAAQMwI+7H7nDlzdOONNyo3N1d5eXlauHChGhsbW2e/33DDDcrIyNBjjz2mfv36acKECe2uHzx4sCR1OA4AAIDeL+zwedVVV+ngwYN68MEHVVlZqZycHK1cubJ1EtLu3bvldLJxEgAAADrq0oSjWbNmadasWX5fW7t2bdBr//CHP3TlLQEAANAL0EUJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMIn0A38Xqtnm4CAABRJ66nGwD0Flsq6rR04x4VltVo54EjcnssxbscGpM+UHlZqcrPzdSEjEE93UwAAHoU4ROIUFl1o+5eVqLC0hq5nA552vR4uj2Wtu1v0PaqI3phXbnyRqdqwcxsZaUN6MEWAwDQc3jsDkRgeXGFpj/9torKayWpXfBsy3e8qLxW059+W8uLK4y1EQCAaELPJ9BFy4srdFdBscIZ2enxWvLI0l0FxZKkGTkZ3dI2AACiFT2fQBeUVjdq7tKSsIJnW5akuUtLVFbdaGezAACIeoRPoAvuWVYijxXZbHaPZenuZSU2tQgAgNhA+ATCtHlvnQpLawKO7wyVx2upsLRGWyrqbGoZAADRjzGfQJheLtqjOKdDLQHC51dPPVGzvj5Gpw1NksdradPuWj38163aXXO0w7kup0NLN+5hCSYAQJ9BzycQpsKymoDBU5JOSHDpd++U6vJfv6vrfrdeXkv63+snyeHoeK7Ha2lDWW03thYAgOhCzycQpp0HjgR9feWWynY/3/3yR/rwwekamz5Q26s6XrvjQIOt7QMAIJoRPoEweL2W3J7gYz2zhvTXnAtPVU5milIGxMv5ry7PEYNP8Bs+3R5LXq8lp9NP1ygAAL0M4RMIg9PpULzLETSAPnfjZFUc/lz3/qVEVfVNcjqk1XO+qgSX/1Eu8S4HwRMA0Gcw5hMI05j0gQFfG9w/XqekD9Sv3tyh9z87pM8OHtGgE+KD3m9sepLdTQQAIGrR8wmEKS8rVdurjvhdaqnuc7dqGpt1Td5JOtDQpBGDT9A9F48LeC+X06HJWSnd2VwAAKIKPZ9AmPJzMwOu8WlZ0g//b5POzBikVXddoAe/MV6PrdgW8F4er6X83MzuaioAAFGHnk8gTBMyBilvdKqKymv9htD3dh7ShU+/3e5Y1r1vdDjP5XRo0qgU1vgEAPQp9HwCXbBgZrZc/hbuDIPL4dCCmdk2tQgAgNhA+AS6ICttgJ7Iz1ZX46dD0hP52cpKG2BnswAAiHo8dge6aEZOhiRp7tISeSwrpL3eXU6HXA6HnsjPbr0eAIC+hJ5PIAIzcjK0avYFmjTq2Ix1V4D1On3Hc0elaNXsCwieAIA+i55PIEJZaQO05Lap2lJRp6Ub92hDWa12HGiQ22Mp3uXQ2PQkTc5KUX5uJpOLAAB9HuETsMmEjEHtwiVbZgIA0BGP3YFuQvAEAKAjwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjCF8AgAAwBjCJwAAAIwhfAIAAMAYwicAAACMIXwCAADAGMInAAAAjOlS+Fy0aJGysrLUr18/TZkyRYWFhQHPffbZZ3X++ecrJSVFKSkpmjZtWtDzAQAA0HuFHT5feuklzZkzR/Pnz9emTZs0ceJEXXTRRTpw4IDf89euXatrrrlGb731ltatW6fMzExNnz5dFRUVETceAAAAsSXs8PnUU0/plltu0U033aTx48dr8eLF6t+/v55//nm/5//5z3/W7bffrpycHI0bN06/+93v5PV6tWbNmogbDwAAgNgSF87Jzc3NKioq0rx581qPOZ1OTZs2TevWrQvpHkePHpXb7VZqamrAc5qamtTU1NT6c319vSTJ7XbL7XaH0+RezVcLahIZ6mgP6hg5amgP6mgP6hi5vlbDUD+nw7IsK9Sb7tu3TxkZGXr//fc1derU1uN33323/vnPf2r9+vWd3uP222/X3//+d3388cfq16+f33MeeughPfzwwx2Ov/jii+rfv3+ozQUAAIAhR48e1bXXXqu6ujolJycHPC+sns9IPf744yooKNDatWsDBk9JmjdvnubMmdP6c319fetY0WAfpq9xu91avXq1LrzwQsXHx/d0c2JWb6vjtv31euXDChWV12rXwSNyey3FOx06+cSBmjQqRd86K0OnD7f/v6PeVseeQA3tQR3tQR0j19dq6HtS3ZmwwmdaWppcLpeqqqraHa+qqtKwYcOCXvvkk0/q8ccf1z/+8Q9lZ2cHPTcxMVGJiYkdjsfHx/eJf3nhoi72iPU6llU36u5lJSosrZHL6ZDH63uo4VCTRyrZd0QfVzbq9+v2KG90qhbMzFZW2gDb2xHrdYwG1NAe1NEe1DFyfaWGoX7GsCYcJSQkaNKkSe0mC/kmD7V9DH+8BQsW6JFHHtHKlSuVm5sbzlsCCMHy4gpNf/ptFZXXSlKb4Nme73hRea2mP/22lhez6gQAwKywH7vPmTNHN954o3Jzc5WXl6eFCxeqsbFRN910kyTphhtuUEZGhh577DFJ0i9+8Qs9+OCDevHFF5WVlaXKykpJ0sCBAzVw4EAbPwrQNy0vrtBdBcUKefC2joVQjyzdVVAsSZqRk9EtbQMA4Hhhh8+rrrpKBw8e1IMPPqjKykrl5ORo5cqVGjp0qCRp9+7dcjq/7FD97W9/q+bmZn3nO99pd5/58+froYceiqz1QB9XWt2ouUtLwgqebVmS5i4t0cSRg7vlETwAAMfr0oSjWbNmadasWX5fW7t2bbufy8rKuvIWAEJwz7ISeUJfsMIvj2Xp7mUlWnJb4KEzJnm9lpxOR083AwDQTYzOdgdgn81761RYWhPxfTxeS4WlNdpSUacJGYNsaFl4tlTUaenGPSosq9HOA0fk9liKdzk0Jn2g8rJSlZ+b2SPtAgB0D8InEKNeLtqjOKdDLX4mF717z9f0/Lulev69stZjK350nlZtrdLCf+zocL7L6dDSjXuMhrzAs/Mlt8fStv0N2l51RC+sK+/W2fkAALPC3l4TQHQoLKvxGzy7wuO1tKGs1pZ7hYLZ+QDQd9HzCcSonQeO2Hq/HQcabL1fIMzOB4C+jZ5PIAZ5vZbcHnt6PX3cHktem3pSA7Frdn5ZdaOdzQIAGET4BGKQ0+lQvCvwjHCvV3I42r8e5wr+n3u8y9FhlrndYdTO2fkAgNjEY3cgRo1JH6ht+/0/Kq9pbNKJSV9uUTswMU6ZKf2D3m9selK3zjzvbHa+wyHdev7JuibvJA0f3E/VR5r14vrdWvTWznbn9fTsfABAZAifQIzKy0rV9qojfifrvP/ZIX1n0kit2Val+s9bNGf6qUF7HF0Oh6obm/SNX73bbTPPg83Ol6R7Lhqnq/My9cjrW7WhrFbpSYk6Jd3/Lmg9MTsfAGAPHrsDMSo/NzPgLPHfrP1M60tr9Nz3Juv5myZr1ceV2n0o8DhJj2WpuqHp2D9308zzYLPzByS4dNO5WXrsb59o2aYK7a45qo3ltXppw56AbTE5Ox8AYB96PoEYNSFjkPJGp6qovLZDYDzS1KIf/t+H7Y4t2xQ8LIY6vLOrM8+Dzc4fkz5QifEuvbezOrRGyNzsfACAvej5BGLYgpnZcjl6ZivKcGaedzY7/wu3N+z3NzE7HwBgP8InEMOy0gboifxs9dRO6KHOPO9sdn7ZoUZ93uzRuWPSQn5vf7PzAQDRj/AJ2KgneuJm5GRo4dU5SnA55QoxjNmV2drOPO/MmACThySpqcWrxf/8TPMuGadvfyVDJ6X211mZg3VlbmbAa8amJ3WpzQCAnsWYTyAC3bk0UThm5GRo4sjBAfdK9/EdPzEpUdUNTfL3JDzB5dS8S8fp8okjlJQYp5KKOj3y+laV7PUfMH0zz++/9LSgbQw2O1+SfvnmDrV4Lc258FSlJ/XTgYYv9OL63QHfc3JWStD3AwBEJ8In0AVl1Y0Bg55dSxOFKyttgJbcNrU1EG8oq9WOAw2tgXhsepImZ6UoPzdTc1/+SFX1TX7vM+/ScbpkwnD9eMlH2nv4c/3nV0/WH/8jT199Yq3qPnd3OD/Umef5uZl6YV15wNctS1r01s4O63r64/Fayg/SKwoAiF6ETyBMy4srNHfplzv1hLo00RP52Ub2JJ+QMahdb6vXa3UYGxlo5vkJ8S5dN2WUfrz0I63dflCSdO+yzXr3nhN11eRMPfP2Lr/XhTLzPNjs/HC4nA5NGpXCGp8AEKMY8wmEYXlxhe4qKFazxxtygPJ4LTV7vLqroDjstTH9CXdcqb8tMwPNPB81pL8S4pwqKv+yJ7PFa+mjvYeDjtkMdea5HbPzXQ6HFszMjugeAICeQ88nEKLS6kbNXVqirvbZ+ZYmmjhycFiP4O0eV+qbeR5s6aNwhTrz3Dc7/66C4i7V0SHpifzuH8IAAOg+9HwCIbpnWUnQLSpDEerSRNKxcaVX/u86feNX7+pP63dr2/6G1sDoG1f6p/W79Y1fvasr/3ddSOtt+gTqxSw/dFRNLR5NGvXlZJ44p0PZIwdpR1XgReLDmXneldn5LqdDCS6nFl6dY2ToAgCg+9DzCYRg8946FZbW+H3N4ZB+8NVTdE3eSToxKVGl1Y365Zod+tuWyg7ntl2aKFhvZXePKw008/xzt0d//mC37rv0dNV97lbFvyYcnRDv0ksb7Zt57pudf8eLm/TxvvqA5zkdx3Zeyh2Vol8YmLQFAOh+hE8gBC8X7VGc0+F3b/Lb/22MvnVWhn7yymaVHmrUlNFDtPCqHNU0Fmq9n8DqW5ooUPj0jSv1FzcLbj1bW/fV66evb213PNwtL4PNPP/Fyk/kcEhPXTlRA/+11NINzxeq/vMWv+d3Zea5b7WAj/fVy+WQ3yWfpGPB84wRyQRPAOhFCJ9ACArLavwGzwSXU3d87RR993frtWn3YUnSnpq9ys1K0bVTTvIbPoMtTdTZuNLb/l+RWjyBt6IMdVxpsJnnTS1ePfzXrXr4r1sDXP2ltjPP3e6OyzD506FXt5ORDJ9UNhhdLQAA0L0In0AIAi1NNGpIf/VPiNP/u3lKu+PxLqe27gu860+gpYk6G1fqb53N4/nGlS65bWrQ8xbMzNb0p9+Wp8tTqMKfeR6sVzeQcHt1AQDRjfAJdCLY0kQDEo/9J/Qff9igyvov2r3W3BK4h9K3NFHbGeLBxpX6BHrs3lao40pNzzzvqdUCAADRhdnuQCd8SxP5s6OqQU1uj0YMPkHlh462+7O/7gu/10j+lybyjSu1g29caWdMzjzvrtUCwl33FADQs+j5BEIwJn2gtu3v+Ki8sdmjZ97ZpQe+MV5Oh7ShrFZJ/eKUm5WqI1+4tWyT/0Xl/S1NFGhcaVeEuuWlFP6+8F2ZeR5Kr24ofL26s/5cpM+qG21Z9xQAYBbhEwhBoKWJJOm/V21XTWOzbv+3McpM7a/6L9z6uKJOi9Z+5vdegZYmCjSutKtC2fLSJ5x94bsS7AKtFpA6IEF/v+t8/f69Mv3mX/X6ykkpKrj1bH3v94V6/7NDfu/3xubKdo/vfeuebq86ohfWlStvdKoWMEMeAKIS4RMIQbCliSTp9++V6ffvlYV0L39LEwUbV9pV/saVdiaUfeG7IlCvbk1js+a+XKJnrs/VOzuqtevgET191UT9cV1ZwOApKeC40a6uewoAMIfwCYQg2NJE4Wi7NFFbPbnlZTB2BE8peK/u2k8PqmDDbi28Okeb99bpaLNHC1Z+GtH7MUMeAKIXE46AEC2YmS2XI7IwFmxpokBbXnZVOFtedqdQenV//sY2xTkduvTM4bqroFjNQdYyDYdvhnw4W48CALoX4RMIkW9poq7Gz86WJsrLSg15xnlnurLlZXcJtlqAz6gh/TU0uZ+cDmlk6gm2vr+/GfIAgJ5D+ATC0J1LE+XnZnb6SP/qZz4IusanT1e2vOxOwXp1410OLbwqR6+X7NNTq7fr8W9na8iABNveu+26pwCAnkf4BMI0IydDq2ZfoEmjjvUsBgqhvuO5o1K0avYFnY479I0rjbT30+V0KG90alQtNxSsV/fH009TUr94PfTaVv32n5+ptLpRC77T+a5JAxJcWnhVjrb+9CIV3vfvuvm80Sq49Ww9+I3xHc4Ndd1TAED3I3wCXeBbmuj1H56n7045SeOHJ7c+Wo53OTR+eLK+O+Ukvf7D8/TSbVOVlTYgpMXQu3tcaU8J1Kt79smp+o/zRmv2S8U60tQiy5LmLCnW5NGp+u6Uk4Le8/5vjFduVoq+/8JGffe59ZqclaozRiT7PTecdU8BAN2L2e5ABIItTeRbM3Puyx+FvBi66S0vTQm0WsAHu2o09id/a3fu3trPlf3QqqD3G5Dg0syvjNSdBR+2Lsk0d+lHWv+Tfw94TTjrngIAug/hE7CR0+lQWXVjwN2CQlkM3fd4fu7SY9tRhrK0k8vpkMvhiOp1LRfMzNa0p/5py71OGtJfCXFOfbTncOuxhqYW7ToYeFZ7V9Y9BQDYj8fugI2WF1do+tNvq6j82CPeQMHx+MXQlxe334azu8aVmtZ2qIElyRvh3u6RsGPdUwBA5Oj5BGyyvLgi7MflwRZD7+4tL7vLoyu26YOywx2GGlTUfq5gnbgFt56trfvqQ5rNv/vQUTW3eJWdOVj76iolSUmJcRqdNiDgHvLRsu4pAPR1hE/ABqXVjZq7tKRD8Aw1UPkWQ584cnCH8ZrdteWlncqqG3XfX4p11VDppY17dNT95Wu+oQZ2amz2aNmmvbrvktNVd9St6iNNmn3hqfJaliw/8T+a1j0FgL6Ox+6ADe5Zdmx8ZiRCXQw92oKnb6hB8b/GX3Zl+9En87N19slD9B/njVbZ45ep7PHLNDIl+GLzP3t9qzbtrtVz38vVn78/RUXltfrswBE1uTvujhRt654CQF9GzycQoc176wI+6g1H28XQo+kxejBthxo4ujQ//5iHX9uq0WkD9Wllg55evV2SdKixKeg1jc0e3fVScevPJ8S7dOe/j9WLhe3X83Q5HZo0KiVmagoAvR3hE4jQy0V7FOd0qCVAj5/DId17yThdPTlTbo9Xf16/Wwv/scPvub7F0GMhKAUaahCuSyYM053TxurU9CSdMSJZJ6X21y1/3BhwfKhvKMOyTXt1yokDVbznsJL6xenOfx8rSVq9tbLd+dG47ikA9GWETyBChWU1AYOnJM2cNFLPvVOqKxa9p6+MStGT35mojWW1endndYdzY2kxdDuGGpyYlKhfXnOWHv/bJ/pG9nCVVTeqaPdhhbrO/i3nn6yTTxwgt8erzRV1yl+8TrVtBpxG67qnANCXET6BCO08cCTo65/sb9D/rDnW01l26KhumJqlc8cM8Rs+pdhYDN2uoQbpSYmKdzm1ckulLhw/VLVH3frTB+UhXfvxvnpd/ut3/b4WC+ueAkBfRfgEIuD1WnJ7gvf+fVJZ3+7ngw1faMjAxIDnx8Ji6P6GGvzbaem69NKzdO+Gv0uSxg9P1oo7z9dv1+7UL1Z+Kkl6fOaZSoxzafa/xmpu21+vd3dUa+Vd5+sLt1der6XkE+JU/3mLpGPjOH/2rQm6+Ixhamxq0TPv7AqpfbmjUvSLmfR4AkA0YrY7EAGn09G6p3sgLceFU8uSguXKWFgM3d9Qg41lNYqLi9P4EcfGq045OVWHjjTp7JOHtJ4zZfQQfbDrUOvPXkv67nPr9b3fb1Bl3ec666QUrf3xv+mMEclyOKT7Lj1dU0an6pY/btT1zxXq7JOHBNy/XTr2mP0bZw7TS7dNJXgCQJQifAIRGpM+0Nb7xcJi6P6GGhxpalFdXZ3yRh8Lm2efPETPvVuq8SOS1T/BpaHJiRqdNkDr24RPn6LyWv2ooFjbKuuV0j9Bb/zofJ1y4kBdOXmkHl2xTe9/dkifVjXov5Z8pDhn4L+2LEm7qo/6fc3bhSWgAAD247E7EKG8rFRtrzrSpfUtj2fHYujd/cg+2FCD6upq5Y0eot/+s1STs1K1YOWxiUSTs1I16IR4VdZ9obJDX4bDnMzBOueUIXpnR7UOHWnS797ZpaevytFtfyxSvMuhxDiXincfbj2/7nO3dlUHH2PrGzPr2xmqsKymw25LeVmpUbczFAD0FYRPIEL5uZl6YV1ok2Q605XF0E2HLN9QA38B9NChQ5qUPUrjhyerxePVZwcb9cGuGp198rHwub60fa9nwxctmjI6Vf9x3mglJcZp7+HP9fM3tmnt9oM6fXjXeoDdHkv5i9/XhrJauZyOdl8KfLstba86ohfWlStvdKoWMDYUAIwifAIRmpAxSHmjU1VUXtuh9/PqZz7ocP6t/6/I733CXQy9rLpRdy8rUWFpjfGQNSZ9oN8tMw8dOqQBiXG6+bzRWv+v2fAf7DqkH/zbKUo+IV6/O27C0GcHj+jG32/w+x7l/9q/Peekwdq3+djancknxP3r0X3wmfab/tVbGqg32ne8qLxW059+m1nxAGAQYz4BGyyYmS1XqItTBhDOYui+LS2Lyo+tCRpqyFpeXBFRG33yslLl8vNo3+1269PKes3IGdE6sWh9aY3OGDFIp5w4sNPQ2NbRZo+WbNyj+y49XVNPGaJThw7Uf+dPDLj4fFuhDoHweC01e7y6q6DYttoAAIIjfAI2yEoboCfys9XV+BnOYui+LS2bPd4eC1n5uZkB33tD6SHFuZyt4bPuc7d2HmjQgfovtKu6Maz3eXTFNhWW1ui5G4/t376hrFZbKuoibv/xLElzl5aoLMz2AQDCx2N3wCa+x7Zzlx7b+SeUYBjuYuiRbmnpC1kTRw6O6BF8sKEGj67Yqvl/3dbu2KW/9L8YfGdSByTo218ZKUnqnxCn+y49vWsNbiPQeFWPZenuZSVactvUiN8DABAY4ROw0YycDE0cOTjgWEwf3/FwF0O3Y0tLu0LWgpnZmv702/JEvLt7YPsOf67JP/tH688nJiXqT9+f0jqeNBQFt56tTysb5PFauuKsDH1a2aBrnu04FtfjtVRYWqMtFXXMggcQE6J9Q5JACJ+AzbLSBmjJbVNbZ6FvKKvVjgMNrbPQx6YnaXJWStiz0O3a0tKukOUbanBXQXGX4+f5Y9P0zg7/24xKxxahP3ikSZKUGOfUMzdM0qbdtVr4j+1hvc/MSSP1pw/K9Z3fvh/0PJfToaUb9xA+AUSl3rKEHOET6CYTMga1+0sg0m+o/ra09Pn2VzL0wGXjNeXRNWr2eFuPP3P9JB1patGcJR+1O9+ukNV2qIHL6e3k7C/f2zfU4PLsETr5vhUhXbfgO9kakBin7/5uvcLt/C2rbtTjf/uk0/M8XksbymrDuzkAdLOeXN2kOzDhCDAk0kcj/ra09HmjZL9cToemjU9vPTZkQIK+Ni5dSzfu7XC+nSFrRk6GVs2+QDmZgyXJ7yz4tsdzR6Vo1ewLNCMnI6TtSSVp1tfH6IKxJ+r7L2xUY7Mn7DZuDmOSkm+RegCIBj29ukl3IHwCMcLflpY+TS1eLS/ep/xJXy5Qf8VZGdp3+HOt87OdpWRvyMpKG6AXbsqTJF2Vm6nxw5NbQ2W8y6Hxw5P13Skn6fUfntdh3/XOtie9eMIw/ejrY3XHi5u0u8b/1pmd+TyMwOr2WGzFCSAqRMPqJt2Bx+5ADAi2paVPwYbdWn7HuRqanKiq+iZ9Z9JIvVzUsdfTxxey7B6sft+lpys+Pr613Z3dP9j2pKcOHainrpyoxf/8TDuqjujEgYmSpGaPV3Wfu21tt0+8yxGTA/gB9C7RsrpJd6DnE4gBoTye/nhfvbbtb9DMr4zUhIxknTo0KWj4NBGyQrl/sDVDs0cOVv+EOP3o38dqw/3TWv/87/WT7G5qq7HpXdvWEwDsZOfqJtGGnk8gRgTa0rKtlzbs1k3njdbQ5H56b2e19td9EfDcaAlZwdYMfblob9AAbTeX06HJWSnG3g8A/Im21U3sRs8nECMCbWnZ1vLifRo+qJ+uzsvUko17Ap4XbSHLju1JA7n6mQ/009e3hnSux2spPzez8xMBoBv5Vjexg291k2hC+ARiRLDH0z4NTS3625ZKHW3yaNXHVQHPi7aQFen2pJIUaXZ1OR3KG50aVb0DAPqmYKubhCsal5AjfALdoDtmS/seT3fW+zksuZ9eLa5ot95nW9EasmbkZGjh1TlKcDk7/Yw+LqdDCS6nHrjsdMU7I/vrzOVwaMHM7IjuAQB2CLa6SVdE2xJyjPkEbGBq14lgW1omnxCnqScP0dknD9EDr24JeI9oDlmRbE+alpTY5d2WHJKeyI/uRZkB9A2hrG4Sru5a3aSrCJ9ABEzvOhFsS8sVPzpfySfE6/G/faJd1Y1+r4+FkNV2e9IlG/ZoY3lo25O23W3JY1khrYnXdrcl3/UA0JN8q5vYGUCjbQk5wifQRcuLK1qDjhT6rhORBp1AIeu8X7wV8JpYCVnBepAnj0rRlZNPCtqDHEnPKQBEi1BWNwlHtKxu4kP4RK9h8pGCb9eJcL6XeryWPLJ0V0GxJEUcQHtTyAq1B/mPH+zutAe5bc/p0o17tKEstJ5TAIgWwTbfaOuGqaN00RnDdN3v1gc8J9pWN5EIn4hhpsZZHi/QrhMFt56trfvqO13Wx65dJ3pLyOquHuQJGYPafe5oGu8EAMHk52bqhXXlnZ6XOiBBo4b0D3pOtK1uIhE+EYNMj7M8np27Tiy5bWrE7YnlkGWyBzlWagIAwTbfaGvhP3Zo4T92BHzd5XRo0qiUqOuAYKklxJTlxRWa/vTbKio/tmZZqL1ky4srbHl/364ToUxmCabtrhN2i5WQZde+xWUBJlcBQCyzY/ONaF3dhPCJmOHrJWv2eEMOfx6vpWaPV3cVFNsSQMPZdeJrp6Wr5KHpmpEzwu/r0bjrhEm9ed9iAIhUpJtvRPPqJoRPxIRo6SULddeJb04coV9ek/Ov0LvP7znRuOuEKbHQgwwAPS2SzTcWXp0TtaubMOYTMSFaxlmGsuvE9WeP0tyLTtP3X9io9aU1Qc+Ntl0nTPH1IB8f5AtuPVvb9terqcWrqydnyu3x6s/rd3c6pmnpxj1RN6YJAOzQ21Y3kQifiAG+XjKfglvP1ieVDfJ6Lc2cNFLNLV7996pPtbx4n3464wxdcuZwVTc06aHXPtba7Qdbr2vbS9aVoBLKrhOXnDlMQwYk6juL31fJ3s5746Jt1wlTgvUgz5w0Us+9U6orFr2nr4xK0ZPfmaiNZbV6d2e13/P7cg8ygL6ht6xu4kP4RNTz10s28ysZ+t+3d2nGr9/VNyaO0M+umKCLzhimv39cqUVv7dTN552sp67K0TmPr9EX7i/3OI+klyyUXSc+3levCSMG6crczJDCZ7TtOmFKsB7kT/Y36H/WHOvpLDt0VDdMzdK5Y4YEDJ9S3+1BBtC3xPLqJm0x5hNRz18v2bb9Dfr1mztVduiofvPWTjW1eFVztFkFG/ao7NBR/XLNDqUOSNDpw5LbXRdpL9mY9IFBX9996KiuefYDXTh+qB7+5hmd3i/adp0wobMe5E8q69v9fLDhCw0ZmBj0nr4eZADoS2IxeEqET8QAf71kbQOK15Jqjzbr08ove78OHmmSJA0ZmNDh2kh6yfKyUjsd9F1a3ahrnvlAl0wYpge/MT7gedG464QJvh7kQFqOC6aWJXX292tf7UEGgFhE+ERUC9RLdnxAOXbM2+GY088aaZH0kuXnZoY0Q3tXdaOueXa9Lp84Qj+57HS/50TjrhOmdNaDHK6+2IMMALGKMZ+IaqGMswxXJL1kwXaduPqZD9r9/NnBI5r883/4vU+07jphSqj7Foeir/YgA0CsoucTUS/aesl6864TpoTagxyKvtyDDACxiJ5PRL1o6yXz7ToR7p7kPtG864QpgXqQj+89lqRb/19RwPv09R5kAIhFhE9EvfzcTL2wrrz1Z38B5bxfvNXhWNa9b3Q4ZlcvmW/XiB8v+UheBd5jvi2X0yGXw6En8rOjdtcJkxbMzNb0p9+Wp8v7VtGDDACxiPCJqBdsnGU47Ogl8y3wW1hWo50HjsjttVr33XVIfmNULO06YRI9yObE6lqAAHonwidiQk/3kpVVNwbc2uz4FvlCaCzuOmGarwd47tJj26fSg2yPDl+S/rULypj0gcrLSuX3EUCPInwiJvRkL9ny4orWcCQFfsTuO+okHIWlN+5b3FOCfUlyeyxt29+g7VVH9MK6cuWNTtUC6gigBxA+ETN6opdseXFF2IHX47XkkaW7CorbtRuB9bZ9i3tCqF+SfMeLyms1/em3+ZIEwDjCJ2KKyV6y0upGzV1aEjB4Ftx6trbuq9dPX9/q93VLx4LyxJGD6V0KUW/Zt9g0f1+SOvv95EsSgJ5C+ETMMdVLds+yL3uRuspjWbp7WYmW3DY1ovv0VQTPznX2JakzfEkCYBrhEzGrO3vJNu+tU2FpTcT38XgtFZbWaEtFHY+L0S34kgQg1rDDEXoNO3vJXi7aozib7udyOrR04x5b7gW05fuSFGh8p8vp0MPfPEMlD03Xpgcu1JwLT/V7XtsvSQDQ3QifgB+FZTVqsXH7xw1ltbbcC2irsy9JMyeNlMdr6Ypfv6eH//qxvn/+aF092f8mC3xJAmAKj90BP3YeOGLr/XYcaLD1foDU+Zek/Yc/b51wtKu6UeOGJenm80arYEPHkMmXJACm0PMJHMfrteT22NPr6eP2WPLa1JMK+HT2JenDPYfb/bxp92FlpQ1QoM5SviQBMIHwCRzH6XQo3mXvLOt4l4OZ27AVX5IAxCrCJ+DHmPSBtt5vbHqSrfcDQvmSlJM5uN3PZ2UOVll1owLlS74kATCB8An4kZeVKpeNs90nZ6XYci+grc6+JI0YfILuv+x0nZw2QN+cOEI3npOl379XFvB8viQBMIHwCfiRn5sZ0vadofB4LeXn+p9hDESisy9Jf9m0V/3iXXp11rn66Ywz9Pv3yvRi4W6/5/IlCYApzHYH/JiQMUh5o1NVVF4bMIRe/cwHnd7H5XRo0qgUFphHt8jPzdQL68r9vtb29/P+V7d0ei++JAEwhZ5PIIAFM7PlckT26N3lcGjBzGybWgS05/uSFOkQEZfTobzRqXxJAmAE4RMIICttgJ7Iz1ZX/7fukPREfjb7ZaNb8SUJQKwhfAJBzMjJ0MKrc5Tgcobcu+RyOpTgcmrh1TmakZPRzS1EX8eXJACxpkvhc9GiRcrKylK/fv00ZcoUFRYWBj1/6dKlGjdunPr166czzzxTK1as6FJjgZ4wIydDq2ZfoEmjjk3GCBRCfcdzR6Vo1ewLCJ4whi9JAGJJ2OHzpZde0pw5czR//nxt2rRJEydO1EUXXaQDBw74Pf/999/XNddco5tvvlkffvihrrjiCl1xxRXasqXzAfBAtMhKG6Alt03V6z88T9+dcpLGD09uXWMx3uXQ+OHJ+u6Uk/T6D8/TS7dNpRcJxvElCUCsCHu2+1NPPaVbbrlFN910kyRp8eLFeuONN/T888/r3nvv7XD+//zP/+jiiy/W3LlzJUmPPPKIVq9erV//+tdavHhxhM0HzJqQMajdpAyv12JRbkQN35ekLRV1WrpxjzaU1WrHgQa5PZbiXQ6NTU/S5KwU5edmMrkIQI8JK3w2NzerqKhI8+bNaz3mdDo1bdo0rVu3zu8169at05w5c9odu+iii/Tqq68GfJ+mpiY1NTW1/lxfXy9Jcrvdcrvd4TS5V/PVgppEJtI6ejx2tiZ28fsYObtqeFp6f91/6WmtP/v7ktSb/z3xu2gP6hi5vlbDUD9nWOGzurpaHo9HQ4cObXd86NCh+uSTT/xeU1lZ6ff8ysrKgO/z2GOP6eGHH+5wfNWqVerfv384Te4TVq9e3dNN6BWooz2oY+SooT2ooz2oY+T6Sg2PHj0a0nlRucj8vHnz2vWW1tfXKzMzU9OnT1dycnIPtiy6uN1urV69WhdeeKHi4+N7ujkxizragzpGjhragzragzpGrq/V0PekujNhhc+0tDS5XC5VVVW1O15VVaVhw4b5vWbYsGFhnS9JiYmJSkxM7HA8Pj6+T/zLCxd1sQd1tAd1jBw1tAd1tAd1jFxfqWGonzGs2e4JCQmaNGmS1qxZ03rM6/VqzZo1mjp1qt9rpk6d2u586Vj3c6DzAQAA0HuF/dh9zpw5uvHGG5Wbm6u8vDwtXLhQjY2NrbPfb7jhBmVkZOixxx6TJN1555366le/qv/+7//WZZddpoKCAm3cuFHPPPOMvZ8EAKIcqyMAQBfC51VXXaWDBw/qwQcfVGVlpXJycrRy5crWSUW7d++W0/llh+o555yjF198Uffff7/uu+8+jR07Vq+++qomTJhg36cAgCjkW/KosKxGOw8caV3yaEz6QOVlpfbaJY8I2QCC6dKEo1mzZmnWrFl+X1u7dm2HY/n5+crPz+/KWwFAzCmrbtTdy0pUWFojl9Mhj9dqfc3tsbRtf4O2Vx3RC+vKlTc6VQtmxvb2ln01ZAPomqic7Q4AsWp5cYXmLi2RxzoWONsGz7Z8x4vKazX96bf1RH62Lj0j3Vg77dDXQjYAe3Rpb3cAQEfLiyt0V0Gxmj3egKHzeB6vpWaPV3cVFGvF5v3d3EL7LC+u0PSn31ZRea2k0EP28uIKY20EEJ0InwBgg9LqRs1dWqLQImdHlqT7X9liZ5O6TaQhmwAK9G2ETwCwwT3LvnzU3lWeLkdXc0IJ2Y9+60wVP3ihyh6/TOOHt98YxJI0d2mJyqobu7WdAKIX4RMAIrR5b50KS2tC7gUMxHf9tv2h7RLSEzoL2f926on6zqSR+o8/bNTkn/1Dn1Y1dDjHY1m6e1lJdzYTQBQjfAJAhF4u2qM4P0sLXZOXqfX3/bscx7307A2TtOA72QHv98qH0flYOpSQfdKQ/jrQ8IU27a7VwSNNfs/1eC0VltZoS0VddzYXQJQifAJAhArLatTiJ2S9sXm/BveP19STh7QeG3RCvC449US9GiRgbio/3B3NjFigkO3zZH62fjpjgkam9FfZ45fp3Xu+FvBcl9OhpRv3dEczAUQ5lloCgAjtPHDE7/H6z1v0z08PakZOht7/7JAk6dIzh6m20a11uw4FvN9nBzs+qo4GgUK2z8OvbVX5oaO6Ju8kzfj1e0Efz3u8ljaU1XZHMwFEOXo+ASACXq8ltydwyHq1uEKXTBimBNexv26vyMnQX0v2KdjcJLfXkjfC8aPdIVDI9mloalFjU4u8lqWDR5pU09gc9PwdB6IzZAPoXoRPAIiA0+lQvCvwo+g12w5IDulr49I1fFA/Tc5KDfrIXZLinY6o256ys5DdFW5PdIZsAN2Lx+4AEKEx6QO1bb//XrymFq/+vqVSV5w1QllD+mtXdaM+3hd8NvspJyZ1RzMj4gvZdgbQeFf0hWwA3Y+eTwCIUF5WqlxBQtSrxRX6+mnpujI3U6+GsMD6V0YNtrF19hmTPtDW+41Nj76QDaD7ET4BIEL5uZlBlx96/7NDOvy5W6ekDwxpd59vnZVhZ/Ns01nIDofL6dDkrBRb7gUgtvDYHQAiNCFjkPJGp6qovNZvCLUsacqjazq9jy/YnX7crkDRIj83Uy+sK7flXh6vpfzcTFvuBSC20PMJADZYMDNbruNXkw+TS9E9/tEXsoP1fj7/XpnO+8VbQe/jcjqUNzpVEzIG2d1EADGA8AkANshKG6An8rO7HB8dkn72rQl2Nqlb2BKyHQ4tmBl4hycAvRvhEwBsMiMnQwuvzlGCyxny2EiX06EEl1MLr87RpWcO7+YWRs6OkP1Efray0gbY2SwAMYTwCQA2mpGToVWzL9CkUccm0wQKob7juaNStGr2BZqRE52TjPyJNGTH0mcFYD8mHAGAzbLSBmjJbVO1paJOSzfu0YayWu040CC3x1K8y6Gx6UmanJWi/NzMmB33OCMnQxNHDtbdy0pUWFojl9Phd7KV73juqBT9YiY9ngAInwDQbSZkDGoXLr1eq1ctqt4XQjYA+xE+AcCQ3hQ82+rtIRuAvRjzCQCwFcETQDCETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYRPAAAAGEP4DMDrtXq6CQAAAL1OXE83IFpsqajT0o17VFhWo50HjsjtsRTvcmhM+kDlZaUqPzdTEzIG9XQzAQAAYlqfD59l1Y26e1mJCktr5HI65GnT4+n2WNq2v0Hbq47ohXXlyhudqgUzs5WVNqAHWwwAABC7+vRj9+XFFZr+9NsqKq+VpHbBsy3f8aLyWk1/+m0tL64w1kYAAIDepM/2fC4vrtBdBcUKZ2Snx2vJI0t3FRRLkmbkZHRL2wAAAHqrPtnzWVrdqLlLS4IGzyfzs/XM9ZP8vmZJmru0RGXVjd3SPgAAgN6qT4bPe5aVyGMF7/N8+LWt+vHSjwK+7rEs3b2sxO6mAQAA9Gp9Lnxu3lunwtKagOM7fRqaWlT/RUvA1z1eS4WlNdpSUWd3EwEAAHqtPhc+Xy7aozino9Pzgj1293E5HVq6cY9dTQMAAOj1+lz4LCyrUYtNC8h7vJY2lNXaci8AAIC+oM+Fz50Hjth6vx0HGmy9HwAAQG/Wp8Kn12vJ7bF320y3x2IrTgAAgBD1qfDpdDoU7+p8vGc44l0OOUMYQwoAAIA+Fj4laUz6QFvvNzY9ydb7AQAA9GZ9LnzmZaXKZVNPpcvp0OSsFFvuBQAA0Bf0ufCZn5vZ6RqfofJ4LeXnZtpyLwAAgL6gz4XPCRmDlDe6897PBJdTjc2egK+7nA7ljU7VhIxBdjcRAACg1+pz4VOSFszMlsvhP3y6nA6NSR+or4xK0Y6qwMsouRwOLZiZ3V1NBAAA6JX6ZPjMShugJ/Kz5S9+njY0SX+ddZ62Vx3Rn9aX+73eIemJ/GxlpQ3o1nYCAAD0NnE93YCeMiMnQ5I0d2mJPJbVOg506/56nf7gSr/XuJwOuRwOPZGf3Xo9AAAAQtcnez59ZuRkaNXsCzRp1LEZ64HGgfqO545K0arZFxA8AQAAuqjP9nz6ZKUN0JLbpmpLRZ2WbtyjDWW12nGgQW6PpXiXQ2PTkzQ5K0X5uZlMLgIAAIhQnw+fPhMyBrULl16vxc5FAAAANuvTj92DIXgCAADYj/AJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMCaupxsQCsuyJEn19fU93JLo4na7dfToUdXX1ys+Pr6nmxOzqKM9qGPkqKE9qKM9qGPk+loNfTnNl9sCiYnw2dDQIEnKzMzs4ZYAAAAgmIaGBg0aNCjg6w6rs3gaBbxer/bt26ekpCQ5HI6ebk7UqK+vV2Zmpvbs2aPk5OSebk7Moo72oI6Ro4b2oI72oI6R62s1tCxLDQ0NGjFihJzOwCM7Y6Ln0+l0auTIkT3djKiVnJzcJ36puxt1tAd1jBw1tAd1tAd1jFxfqmGwHk8fJhwBAADAGMInAAAAjCF8xrDExETNnz9fiYmJPd2UmEYd7UEdI0cN7UEd7UEdI0cN/YuJCUcAAADoHej5BAAAgDGETwAAABhD+AQAAIAxhE8AAAAYQ/gEAACAMYTPGFNTU6PrrrtOycnJGjx4sG6++WYdOXIkpGsty9Ill1wih8OhV199tXsbGuXCrWNNTY1++MMf6rTTTtMJJ5ygk046ST/60Y9UV1dnsNU9b9GiRcrKylK/fv00ZcoUFRYWBj1/6dKlGjdunPr166czzzxTK1asMNTS6BVODZ999lmdf/75SklJUUpKiqZNm9ZpzfuKcH8XfQoKCuRwOHTFFVd0bwNjRLh1PHz4sO644w4NHz5ciYmJOvXUU/v8f9fh1nDhwoWt/y/JzMzU7Nmz9cUXXxhqbZSwEFMuvvhia+LEidYHH3xgvfPOO9aYMWOsa665JqRrn3rqKeuSSy6xJFmvvPJK9zY0yoVbx82bN1vf/va3rddee83auXOntWbNGmvs2LHWzJkzDba6ZxUUFFgJCQnW888/b3388cfWLbfcYg0ePNiqqqrye/57771nuVwua8GCBdbWrVut+++/34qPj7c2b95suOXRI9waXnvttdaiRYusDz/80Nq2bZv1ve99zxo0aJC1d+9ewy2PLuHW0ae0tNTKyMiwzj//fGvGjBlmGhvFwq1jU1OTlZuba1166aXWu+++a5WWllpr1661iouLDbc8eoRbwz//+c9WYmKi9ec//9kqLS21/v73v1vDhw+3Zs+ebbjlPYvwGUO2bt1qSbI2bNjQeuxvf/ub5XA4rIqKiqDXfvjhh1ZGRoa1f//+Ph8+I6ljW0uWLLESEhIst9vdHc2MOnl5edYdd9zR+rPH47FGjBhhPfbYY37Pv/LKK63LLrus3bEpU6ZYt912W7e2M5qFW8PjtbS0WElJSdYLL7zQXU2MCV2pY0tLi3XOOedYv/vd76wbb7yR8GmFX8ff/va31sknn2w1NzebamLUC7eGd9xxh/X1r3+93bE5c+ZY5557bre2M9rw2D2GrFu3ToMHD1Zubm7rsWnTpsnpdGr9+vUBrzt69KiuvfZaLVq0SMOGDTPR1KjW1Toer66uTsnJyYqLi+uOZkaV5uZmFRUVadq0aa3HnE6npk2bpnXr1vm9Zt26de3Ol6SLLroo4Pm9XVdqeLyjR4/K7XYrNTW1u5oZ9bpax5/+9KdKT0/XzTffbKKZUa8rdXzttdc0depU3XHHHRo6dKgmTJigRx99VB6Px1Szo0pXanjOOeeoqKio9dH8rl27tGLFCl166aVG2hwtev//NXuRyspKpaentzsWFxen1NRUVVZWBrxu9uzZOuecczRjxozubmJM6God26qurtYjjzyiW2+9tTuaGHWqq6vl8Xg0dOjQdseHDh2qTz75xO81lZWVfs8Ptca9TVdqeLx77rlHI0aM6BDq+5Ku1PHdd9/Vc889p+LiYgMtjA1dqeOuXbv05ptv6rrrrtOKFSu0c+dO3X777XK73Zo/f76JZkeVrtTw2muvVXV1tc477zxZlqWWlhb953/+p+677z4TTY4a9HxGgXvvvVcOhyPon1D/53S81157TW+++aYWLlxob6OjUHfWsa36+npddtllGj9+vB566KHIGw6E4PHHH1dBQYFeeeUV9evXr6ebEzMaGhp0/fXX69lnn1VaWlpPNyemeb1epaen65lnntGkSZN01VVX6Sc/+YkWL17c002LGWvXrtWjjz6q3/zmN9q0aZP+8pe/6I033tAjjzzS000zip7PKPBf//Vf+t73vhf0nJNPPlnDhg3TgQMH2h1vaWlRTU1NwMfpb775pj777DMNHjy43fGZM2fq/PPP19q1ayNoeXTpzjr6NDQ06OKLL1ZSUpJeeeUVxcfHR9rsmJCWliaXy6Wqqqp2x6uqqgLWbNiwYWGd39t1pYY+Tz75pB5//HH94x//UHZ2dnc2M+qFW8fPPvtMZWVluvzyy1uPeb1eSceeeHz66ac65ZRTurfRUagrv4/Dhw9XfHy8XC5X67HTTz9dlZWVam5uVkJCQre2Odp0pYYPPPCArr/+en3/+9+XJJ155plqbGzUrbfeqp/85CdyOvtGn2Df+JRR7sQTT9S4ceOC/klISNDUqVN1+PBhFRUVtV775ptvyuv1asqUKX7vfe+996qkpETFxcWtfyTp6aef1u9//3sTH8+Y7qyjdKzHc/r06UpISNBrr73Wp3qfEhISNGnSJK1Zs6b1mNfr1Zo1azR16lS/10ydOrXd+ZK0evXqgOf3dl2poSQtWLBAjzzyiFauXNlunHJfFW4dx40bp82bN7f7O/Cb3/ymvva1r6m4uFiZmZkmmx81uvL7eO6552rnzp2t4V2Stm/fruHDh/e54Cl1rYZHjx7tEDB9Yd6yrO5rbLTp6RlPCM/FF19snXXWWdb69eutd9991xo7dmy7JYL27t1rnXbaadb69esD3kN9fLa7ZYVfx7q6OmvKlCnWmWeeae3cudPav39/65+Wlpae+hhGFRQUWImJidYf/vAHa+vWrdatt95qDR482KqsrLQsy7Kuv/5669577209/7333rPi4uKsJ5980tq2bZs1f/58lloKs4aPP/64lZCQYL388svtfucaGhp66iNEhXDreDxmux8Tbh13795tJSUlWbNmzbI+/fRT6/XXX7fS09Otn/3sZz31EXpcuDWcP3++lZSUZP3f//2ftWvXLmvVqlXWKaecYl155ZU99RF6BOEzxhw6dMi65pprrIEDB1rJycnWTTfd1O5/RKWlpZYk66233gp4D8Jn+HV86623LEl+/5SWlvbMh+gBv/rVr6yTTjrJSkhIsPLy8qwPPvig9bWvfvWr1o033tju/CVLllinnnqqlZCQYJ1xxhnWG2+8YbjF0SecGo4aNcrv79z8+fPNNzzKhPu72Bbh80vh1vH999+3pkyZYiUmJlonn3yy9fOf/7zPfAEPJJwaut1u66GHHrJOOeUUq1+/flZmZqZ1++23W7W1teYb3oMcltWX+nkBAADQkxjzCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAYwifAAAAMIbwCQAAAGMInwAAADCG8AkAAABjCJ8AAAAwhvAJAAAAY/4/EbmnwRC1Ae4AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# visualize dimensions 0 and 1 of the embedding matrix C for all characters\n", - "plt.figure(figsize=(8,8))\n", - "plt.scatter(C[:,0].data, C[:,1].data, s=200)\n", - "for i in range(C.shape[0]):\n", - " plt.text(C[i,0].item(), C[i,1].item(), itos[i], ha=\"center\", va=\"center\", color='white')\n", - "plt.grid('minor')" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "# training split, dev/validation split, test split\n", - "# 80%, 10%, 10%" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 3, 10])" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "context = [0] * block_size\n", - "C[torch.tensor([context])].shape" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "carlah.\n", - "ambril.\n", - "khkimyli.\n", - "thiy.\n", - "salaysge.\n", - "mahnen.\n", - "deliy.\n", - "chigeni.\n", - "nelania.\n", - "chaiir.\n", - "kaleig.\n", - "dham.\n", - "jore.\n", - "quinn.\n", - "srojlir.\n", - "jamii.\n", - "wazelog.\n", - "jaryxi.\n", - "jaxeeni.\n", - "sayley.\n" - ] - } - ], - "source": [ - "\n", - "\n", - "# sample from the model\n", - "g = torch.Generator().manual_seed(2147483647 + 10)\n", - "\n", - "for _ in range(20):\n", - " \n", - " out = []\n", - " context = [0] * block_size # initialize with all ...\n", - " while True:\n", - " emb = C[torch.tensor([context])] # (1,block_size,d)\n", - " h = torch.tanh(emb.view(1, -1) @ W1 + b1)\n", - " logits = h @ W2 + b2\n", - " probs = F.softmax(logits, dim=1)\n", - " ix = torch.multinomial(probs, num_samples=1, generator=g).item()\n", - " context = context[1:] + [ix]\n", - " out.append(ix)\n", - " if ix == 0:\n", - " break\n", - " \n", - " print(''.join(itos[i] for i in out))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/example/language_model/pytorch/RNN.ipynb b/example/language_model/pytorch/RNN.ipynb new file mode 100644 index 000000000..a568a6480 --- /dev/null +++ b/example/language_model/pytorch/RNN.ipynb @@ -0,0 +1,63 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Design criteria for modeling sequential data: \n", + "1. Handle variable-lenght sequence.\n", + "2. Track long-term depedencies.\n", + "3. Maintain information about order.\n", + "4. Share params accross the sequence.\n", + "\n", + "\n", + "RNN can fullfit these criteria and be used for sequence modeling.\n", + "\n", + "For instance, we can use it for:\n", + " - Many To One relation, where we take a sequence of tokens and then product a single reuslt, such as Sentiment Classification: where we take list of words\n", + " and predict the sentiment of the sentence.\n", + " - One to Many: Text generation: We input an image and the RNN would produce what the picture contains.\n", + " - Many to Many: Tanslation or Forecasting\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The CBOW model architecture tries to predict the current target word (the center word) based on the source context words. \n", + "\n", + "Considering a simple sentence, **“the quick brown fox jumps over the lazy dog”**, this can be pairs of `(context_window, target_word)` where if we consider a context window of size 2, we have examples like ([quick, fox], brown), ([the, brown], quick), ([the, dog], lazy) and so on. Thus the model tries to predict the target_word based on the context_window words." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/language_model/bigram.py b/example/language_model/pytorch/bigram.py similarity index 100% rename from example/language_model/bigram.py rename to example/language_model/pytorch/bigram.py diff --git a/example/language_model/pytorch/cbow.ipynb b/example/language_model/pytorch/cbow.ipynb new file mode 100644 index 000000000..cb4ac3c92 --- /dev/null +++ b/example/language_model/pytorch/cbow.ipynb @@ -0,0 +1,142 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "\n", + "CONTEXT_SIZE = 2 # 2 words to the left, 2 to the right\n", + "EMDEDDING_DIM = 100\n", + "\n", + "raw_text = \"\"\"We are about to study the idea of a computational process.\n", + "Computational processes are abstract beings that inhabit computers.\n", + "As they evolve, processes manipulate other abstract things called data.\n", + "The evolution of a process is directed by a pattern of rules called a program. \n", + "People create programs to direct processes. In effect,\n", + "we conjure the spirits of the computer with our spells.\"\"\".split()\n", + "\n", + "# By deriving a set from `raw_text`, we deduplicate the array\n", + "vocab = set(raw_text)\n", + "vocab_size = len(vocab)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Raw text: We are about to study the idea of a computational process. Computational processes are abstract beings that inhabit computers. As they evolve, processes manipulate other abstract things called data. The evolution of a process is directed by a pattern of rules called a program. People create programs to direct processes. In effect, we conjure the spirits of the computer with our spells.\n", + "\n", + "Context: ['People', 'create', 'to', 'direct']\n", + "\n", + "Prediction: programs\n" + ] + } + ], + "source": [ + "word_to_index = {word: ix for ix, word in enumerate(vocab)}\n", + "index_to_word = {ix: word for ix, word in enumerate(vocab)}\n", + "\n", + "data = []\n", + "\n", + "for i in range(CONTEXT_SIZE, len(raw_text) - CONTEXT_SIZE):\n", + " target = raw_text[i]\n", + " context = raw_text[i - CONTEXT_SIZE : i + CONTEXT_SIZE + 1]\n", + " data.append((context, target))\n", + "\n", + "\n", + "def make_context_vector(context, word_to_index):\n", + " idxs = [word_to_index[w] for w in context]\n", + " return torch.tensor(idxs, dtype=torch.long)\n", + "\n", + "\n", + "class CBOW(torch.nn.Module):\n", + " def __init__(self, vocab_size, embedding_dim):\n", + " super(CBOW, self).__init__()\n", + "\n", + " #out: 1 x emdedding_dim\n", + " self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n", + " self.linear1 = nn.Linear(embedding_dim, 128)\n", + " self.activation_function1 = nn.ReLU()\n", + " \n", + " #out: 1 x vocab_size\n", + " self.linear2 = nn.Linear(128, vocab_size)\n", + " self.activation_function2 = nn.LogSoftmax(dim = -1)\n", + " \n", + "\n", + " def forward(self, inputs):\n", + " embeds = sum(self.embeddings(inputs)).view(1,-1)\n", + " out = self.linear1(embeds)\n", + " out = self.activation_function1(out)\n", + " out = self.linear2(out)\n", + " out = self.activation_function2(out)\n", + " return out\n", + "\n", + " def get_word_emdedding(self, word):\n", + " word = torch.tensor([word_to_index[word]])\n", + " return self.embeddings(word).view(1,-1)\n", + "\n", + "\n", + "model = CBOW(vocab_size, EMDEDDING_DIM)\n", + "\n", + "loss_function = nn.NLLLoss()\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=0.001)\n", + "\n", + "#TRAINING\n", + "for epoch in range(50):\n", + " total_loss = 0\n", + "\n", + " for context, target in data:\n", + " context_vector = make_context_vector(context, word_to_index) \n", + "\n", + " log_probs = model(context_vector)\n", + "\n", + " total_loss += loss_function(log_probs, torch.tensor([word_to_index[target]]))\n", + "\n", + " #optimize at the end of each epoch\n", + " optimizer.zero_grad()\n", + " total_loss.backward()\n", + " optimizer.step()\n", + "\n", + "#TESTING\n", + "context = ['People','create','to', 'direct']\n", + "context_vector = make_context_vector(context, word_to_index)\n", + "a = model(context_vector)\n", + "\n", + "#Print result\n", + "print(f'Raw text: {\" \".join(raw_text)}\\n')\n", + "print(f'Context: {context}\\n')\n", + "print(f'Prediction: {index_to_word[torch.argmax(a[0]).item()]}')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/language_model/language-model.py b/example/language_model/pytorch/language-model.py similarity index 100% rename from example/language_model/language-model.py rename to example/language_model/pytorch/language-model.py diff --git a/example/language_model/names.txt b/example/language_model/pytorch/names.txt similarity index 100% rename from example/language_model/names.txt rename to example/language_model/pytorch/names.txt diff --git a/example/language_model/rnn.py b/example/language_model/pytorch/rnn.py similarity index 100% rename from example/language_model/rnn.py rename to example/language_model/pytorch/rnn.py diff --git a/gigatorch/embedding.py b/gigatorch/embedding.py new file mode 100644 index 000000000..73baa28f9 --- /dev/null +++ b/gigatorch/embedding.py @@ -0,0 +1,37 @@ +from gigatorch import Tensor +from gigatorch.weight_init import WightInitializer +import numpy as np + + +class Embedding: + def __init__(self, vocab_size: int, embed_size: int): + self.vocab_size, self.embed_size = vocab_size, embed_size + # What should be fan_in fan_out here? + self.weight = WightInitializer.xavier_normal( + 1, 2, vocab_size, embed_size) + + def __call__(self, idx: Tensor) -> Tensor: + return (self.vocab_counter == idx.unsqueeze(2)).expand(*idx.shape, self.vocab_size) @ self.weight + + +@staticmethod +def prepare_data(raw_text, context_size=2): + raw_text = raw_text.split() + vocab = set(raw_text) + + word_to_index = {word: ix for ix, word in enumerate(vocab)} + index_to_word = {ix: word for ix, word in enumerate(vocab)} + + data = [] + for i in range(context_size, len(raw_text) - context_size): + target = raw_text[i] + context = raw_text[i - context_size: i + context_size + 1] + data.append((target, context)) + + return data, word_to_index, index_to_word + + +@staticmethod +def make_context_vector(context, word_to_index): + indexes = [word_to_index[w] for w in context] + return Tensor(indexes, dtype=np.long) diff --git a/requirements.txt b/requirements.txt index 6f7c10d10..a920635aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ contourpy==1.2.0 cycler==0.12.1 debugpy==1.8.1 decorator==5.1.1 +exceptiongroup==1.2.0 executing==2.0.1 filelock==3.13.1 fonttools==4.49.0 @@ -47,6 +48,7 @@ setuptools-black==0.1.5 six==1.16.0 stack-data==0.6.3 sympy==1.12 +tomli==2.0.1 torch==2.2.1 torchvision==0.17.1 tornado==6.4 diff --git a/tests/embedding_test.py b/tests/embedding_test.py new file mode 100644 index 000000000..b5766b828 --- /dev/null +++ b/tests/embedding_test.py @@ -0,0 +1,15 @@ + +def test_word_embedding(): + raw_text = """We are about to study the idea of a computational process. + Computational processes are abstract beings that inhabit computers. + As they evolve, processes manipulate other abstract things called data. + The evolution of a process is directed by a pattern of rules called a program. + People create programs to direct processes. In effect, + we conjure the spirits of the computer with our spells.""".split() + + # By deriving a set from `raw_text`, we deduplicate the array + vocab = set(raw_text) + vocab_size = len(vocab) + + word_to_index = {word: ix for ix, word in enumerate(vocab)} + index_to_word = {ix: word for ix, word in enumerate(vocab)}