08.rainbow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configurations for Colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "IN_COLAB = \"google.colab\" in sys.modules\n",
    "\n",
    "if IN_COLAB:\n",
    "    !apt install python-opengl\n",
    "    !apt install ffmpeg\n",
    "    !apt install xvfb\n",
    "    !pip install PyVirtualDisplay==3.0\n",
    "    !pip install gymnasium==0.28.1\n",
    "    from pyvirtualdisplay import Display\n",
    "    \n",
    "    # Start virtual display\n",
    "    dis = Display(visible=0, size=(400, 400))\n",
    "    dis.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 08. Rainbow\n",
    "\n",
    "[M. Hessel et al., \"Rainbow: Combining Improvements in Deep Reinforcement Learning.\" arXiv preprint arXiv:1710.02298, 2017.](https://arxiv.org/pdf/1710.02298.pdf)\n",
    "\n",
    "We will integrate all the following seven components into a single integrated agent, which is called Rainbow!\n",
    "\n",
    "1. DQN\n",
    "2. Double DQN\n",
    "3. Prioritized Experience Replay\n",
    "4. Dueling Network\n",
    "5. Noisy Network\n",
    "6. Categorical DQN\n",
    "7. N-step Learning\n",
    "\n",
    "This method shows an impressive performance on the Atari 2600 benchmark, both in terms of data efficiency and final performance. \n",
    "\n",
    "![rainbow](https://user-images.githubusercontent.com/14961526/60591412-61748100-9dd9-11e9-84fb-076c7a61fbab.png)\n",
    "\n",
    "However, the integration is not so simple because some of components are not independent each other, so we will look into a number of points that people especailly feel confused.\n",
    "\n",
    "1. Noisy Network <-> Dueling Network\n",
    "2. Dueling Network <-> Categorical DQN\n",
    "3. Categorical DQN <-> Double DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jinwoo.park/miniforge3/envs/rainbow-is-all-you-need/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import os\n",
    "import random\n",
    "from collections import deque\n",
    "from typing import Deque, Dict, List, Tuple\n",
    "\n",
    "import gymnasium as gym\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from IPython.display import clear_output\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "\n",
    "# download segment tree module\n",
    "if IN_COLAB:\n",
    "    !wget https://raw.githubusercontent.com/curt-park/rainbow-is-all-you-need/master/segment_tree.py\n",
    "\n",
    "from segment_tree import MinSegmentTree, SumSegmentTree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Replay buffer\n",
    "\n",
    "Same as the basic N-step buffer. \n",
    "\n",
    "(Please see *01.dqn.ipynb*, *07.n_step_learning.ipynb* for detailed description about the basic (n-step) replay buffer.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    \"\"\"A simple numpy replay buffer.\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        obs_dim: int, \n",
    "        size: int, \n",
    "        batch_size: int = 32, \n",
    "        n_step: int = 1, \n",
    "        gamma: float = 0.99\n",
    "    ):\n",
    "        self.obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n",
    "        self.next_obs_buf = np.zeros([size, obs_dim], dtype=np.float32)\n",
    "        self.acts_buf = np.zeros([size], dtype=np.float32)\n",
    "        self.rews_buf = np.zeros([size], dtype=np.float32)\n",
    "        self.done_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.max_size, self.batch_size = size, batch_size\n",
    "        self.ptr, self.size, = 0, 0\n",
    "        \n",
    "        # for N-step Learning\n",
    "        self.n_step_buffer = deque(maxlen=n_step)\n",
    "        self.n_step = n_step\n",
    "        self.gamma = gamma\n",
    "\n",
    "    def store(\n",
    "        self, \n",
    "        obs: np.ndarray, \n",
    "        act: np.ndarray, \n",
    "        rew: float, \n",
    "        next_obs: np.ndarray, \n",
    "        done: bool,\n",
    "    ) -> Tuple[np.ndarray, np.ndarray, float, np.ndarray, bool]:\n",
    "        transition = (obs, act, rew, next_obs, done)\n",
    "        self.n_step_buffer.append(transition)\n",
    "\n",
    "        # single step transition is not ready\n",
    "        if len(self.n_step_buffer) < self.n_step:\n",
    "            return ()\n",
    "        \n",
    "        # make a n-step transition\n",
    "        rew, next_obs, done = self._get_n_step_info(\n",
    "            self.n_step_buffer, self.gamma\n",
    "        )\n",
    "        obs, act = self.n_step_buffer[0][:2]\n",
    "        \n",
    "        self.obs_buf[self.ptr] = obs\n",
    "        self.next_obs_buf[self.ptr] = next_obs\n",
    "        self.acts_buf[self.ptr] = act\n",
    "        self.rews_buf[self.ptr] = rew\n",
    "        self.done_buf[self.ptr] = done\n",
    "        self.ptr = (self.ptr + 1) % self.max_size\n",
    "        self.size = min(self.size + 1, self.max_size)\n",
    "        \n",
    "        return self.n_step_buffer[0]\n",
    "\n",
    "    def sample_batch(self) -> Dict[str, np.ndarray]:\n",
    "        idxs = np.random.choice(self.size, size=self.batch_size, replace=False)\n",
    "\n",
    "        return dict(\n",
    "            obs=self.obs_buf[idxs],\n",
    "            next_obs=self.next_obs_buf[idxs],\n",
    "            acts=self.acts_buf[idxs],\n",
    "            rews=self.rews_buf[idxs],\n",
    "            done=self.done_buf[idxs],\n",
    "            # for N-step Learning\n",
    "            indices=idxs,\n",
    "        )\n",
    "    \n",
    "    def sample_batch_from_idxs(\n",
    "        self, idxs: np.ndarray\n",
    "    ) -> Dict[str, np.ndarray]:\n",
    "        # for N-step Learning\n",
    "        return dict(\n",
    "            obs=self.obs_buf[idxs],\n",
    "            next_obs=self.next_obs_buf[idxs],\n",
    "            acts=self.acts_buf[idxs],\n",
    "            rews=self.rews_buf[idxs],\n",
    "            done=self.done_buf[idxs],\n",
    "        )\n",
    "    \n",
    "    def _get_n_step_info(\n",
    "        self, n_step_buffer: Deque, gamma: float\n",
    "    ) -> Tuple[np.int64, np.ndarray, bool]:\n",
    "        \"\"\"Return n step rew, next_obs, and done.\"\"\"\n",
    "        # info of the last transition\n",
    "        rew, next_obs, done = n_step_buffer[-1][-3:]\n",
    "\n",
    "        for transition in reversed(list(n_step_buffer)[:-1]):\n",
    "            r, n_o, d = transition[-3:]\n",
    "\n",
    "            rew = r + gamma * rew * (1 - d)\n",
    "            next_obs, done = (n_o, d) if d else (next_obs, done)\n",
    "\n",
    "        return rew, next_obs, done\n",
    "\n",
    "    def __len__(self) -> int:\n",
    "        return self.size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prioritized replay Buffer\n",
    "\n",
    "`store` method returns boolean in order to inform if a N-step transition has been generated.\n",
    "\n",
    "(Please see *02.per.ipynb* for detailed description about PER.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrioritizedReplayBuffer(ReplayBuffer):\n",
    "    \"\"\"Prioritized Replay buffer.\n",
    "    \n",
    "    Attributes:\n",
    "        max_priority (float): max priority\n",
    "        tree_ptr (int): next index of tree\n",
    "        alpha (float): alpha parameter for prioritized replay buffer\n",
    "        sum_tree (SumSegmentTree): sum tree for prior\n",
    "        min_tree (MinSegmentTree): min tree for min prior to get max weight\n",
    "        \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(\n",
    "        self, \n",
    "        obs_dim: int, \n",
    "        size: int, \n",
    "        batch_size: int = 32, \n",
    "        alpha: float = 0.6,\n",
    "        n_step: int = 1, \n",
    "        gamma: float = 0.99,\n",
    "    ):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        assert alpha >= 0\n",
    "        \n",
    "        super(PrioritizedReplayBuffer, self).__init__(\n",
    "            obs_dim, size, batch_size, n_step, gamma\n",
    "        )\n",
    "        self.max_priority, self.tree_ptr = 1.0, 0\n",
    "        self.alpha = alpha\n",
    "        \n",
    "        # capacity must be positive and a power of 2.\n",
    "        tree_capacity = 1\n",
    "        while tree_capacity < self.max_size:\n",
    "            tree_capacity *= 2\n",
    "\n",
    "        self.sum_tree = SumSegmentTree(tree_capacity)\n",
    "        self.min_tree = MinSegmentTree(tree_capacity)\n",
    "        \n",
    "    def store(\n",
    "        self, \n",
    "        obs: np.ndarray, \n",
    "        act: int, \n",
    "        rew: float, \n",
    "        next_obs: np.ndarray, \n",
    "        done: bool,\n",
    "    ) -> Tuple[np.ndarray, np.ndarray, float, np.ndarray, bool]:\n",
    "        \"\"\"Store experience and priority.\"\"\"\n",
    "        transition = super().store(obs, act, rew, next_obs, done)\n",
    "        \n",
    "        if transition:\n",
    "            self.sum_tree[self.tree_ptr] = self.max_priority ** self.alpha\n",
    "            self.min_tree[self.tree_ptr] = self.max_priority ** self.alpha\n",
    "            self.tree_ptr = (self.tree_ptr + 1) % self.max_size\n",
    "        \n",
    "        return transition\n",
    "\n",
    "    def sample_batch(self, beta: float = 0.4) -> Dict[str, np.ndarray]:\n",
    "        \"\"\"Sample a batch of experiences.\"\"\"\n",
    "        assert len(self) >= self.batch_size\n",
    "        assert beta > 0\n",
    "        \n",
    "        indices = self._sample_proportional()\n",
    "        \n",
    "        obs = self.obs_buf[indices]\n",
    "        next_obs = self.next_obs_buf[indices]\n",
    "        acts = self.acts_buf[indices]\n",
    "        rews = self.rews_buf[indices]\n",
    "        done = self.done_buf[indices]\n",
    "        weights = np.array([self._calculate_weight(i, beta) for i in indices])\n",
    "        \n",
    "        return dict(\n",
    "            obs=obs,\n",
    "            next_obs=next_obs,\n",
    "            acts=acts,\n",
    "            rews=rews,\n",
    "            done=done,\n",
    "            weights=weights,\n",
    "            indices=indices,\n",
    "        )\n",
    "        \n",
    "    def update_priorities(self, indices: List[int], priorities: np.ndarray):\n",
    "        \"\"\"Update priorities of sampled transitions.\"\"\"\n",
    "        assert len(indices) == len(priorities)\n",
    "\n",
    "        for idx, priority in zip(indices, priorities):\n",
    "            assert priority > 0\n",
    "            assert 0 <= idx < len(self)\n",
    "\n",
    "            self.sum_tree[idx] = priority ** self.alpha\n",
    "            self.min_tree[idx] = priority ** self.alpha\n",
    "\n",
    "            self.max_priority = max(self.max_priority, priority)\n",
    "            \n",
    "    def _sample_proportional(self) -> List[int]:\n",
    "        \"\"\"Sample indices based on proportions.\"\"\"\n",
    "        indices = []\n",
    "        p_total = self.sum_tree.sum(0, len(self) - 1)\n",
    "        segment = p_total / self.batch_size\n",
    "        \n",
    "        for i in range(self.batch_size):\n",
    "            a = segment * i\n",
    "            b = segment * (i + 1)\n",
    "            upperbound = random.uniform(a, b)\n",
    "            idx = self.sum_tree.retrieve(upperbound)\n",
    "            indices.append(idx)\n",
    "            \n",
    "        return indices\n",
    "    \n",
    "    def _calculate_weight(self, idx: int, beta: float):\n",
    "        \"\"\"Calculate the weight of the experience at idx.\"\"\"\n",
    "        # get max weight\n",
    "        p_min = self.min_tree.min() / self.sum_tree.sum()\n",
    "        max_weight = (p_min * len(self)) ** (-beta)\n",
    "        \n",
    "        # calculate weights\n",
    "        p_sample = self.sum_tree[idx] / self.sum_tree.sum()\n",
    "        weight = (p_sample * len(self)) ** (-beta)\n",
    "        weight = weight / max_weight\n",
    "        \n",
    "        return weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Noisy Layer\n",
    "\n",
    "Please see *05.noisy_net.ipynb* for detailed description.\n",
    "\n",
    "**References:**\n",
    "\n",
    "- https://github.com/higgsfield/RL-Adventure/blob/master/5.noisy%20dqn.ipynb\n",
    "- https://github.com/Kaixhin/Rainbow/blob/master/model.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NoisyLinear(nn.Module):\n",
    "    \"\"\"Noisy linear module for NoisyNet.\n",
    "    \n",
    "    \n",
    "        \n",
    "    Attributes:\n",
    "        in_features (int): input size of linear module\n",
    "        out_features (int): output size of linear module\n",
    "        std_init (float): initial std value\n",
    "        weight_mu (nn.Parameter): mean value weight parameter\n",
    "        weight_sigma (nn.Parameter): std value weight parameter\n",
    "        bias_mu (nn.Parameter): mean value bias parameter\n",
    "        bias_sigma (nn.Parameter): std value bias parameter\n",
    "        \n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        in_features: int, \n",
    "        out_features: int, \n",
    "        std_init: float = 0.5,\n",
    "    ):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        super(NoisyLinear, self).__init__()\n",
    "        \n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.std_init = std_init\n",
    "\n",
    "        self.weight_mu = nn.Parameter(torch.Tensor(out_features, in_features))\n",
    "        self.weight_sigma = nn.Parameter(\n",
    "            torch.Tensor(out_features, in_features)\n",
    "        )\n",
    "        self.register_buffer(\n",
    "            \"weight_epsilon\", torch.Tensor(out_features, in_features)\n",
    "        )\n",
    "\n",
    "        self.bias_mu = nn.Parameter(torch.Tensor(out_features))\n",
    "        self.bias_sigma = nn.Parameter(torch.Tensor(out_features))\n",
    "        self.register_buffer(\"bias_epsilon\", torch.Tensor(out_features))\n",
    "\n",
    "        self.reset_parameters()\n",
    "        self.reset_noise()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        \"\"\"Reset trainable network parameters (factorized gaussian noise).\"\"\"\n",
    "        mu_range = 1 / math.sqrt(self.in_features)\n",
    "        self.weight_mu.data.uniform_(-mu_range, mu_range)\n",
    "        self.weight_sigma.data.fill_(\n",
    "            self.std_init / math.sqrt(self.in_features)\n",
    "        )\n",
    "        self.bias_mu.data.uniform_(-mu_range, mu_range)\n",
    "        self.bias_sigma.data.fill_(\n",
    "            self.std_init / math.sqrt(self.out_features)\n",
    "        )\n",
    "\n",
    "    def reset_noise(self):\n",
    "        \"\"\"Make new noise.\"\"\"\n",
    "        epsilon_in = self.scale_noise(self.in_features)\n",
    "        epsilon_out = self.scale_noise(self.out_features)\n",
    "\n",
    "        # outer product\n",
    "        self.weight_epsilon.copy_(epsilon_out.ger(epsilon_in))\n",
    "        self.bias_epsilon.copy_(epsilon_out)\n",
    "\n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Forward method implementation.\n",
    "        \n",
    "        We don't use separate statements on train / eval mode.\n",
    "        It doesn't show remarkable difference of performance.\n",
    "        \"\"\"\n",
    "        return F.linear(\n",
    "            x,\n",
    "            self.weight_mu + self.weight_sigma * self.weight_epsilon,\n",
    "            self.bias_mu + self.bias_sigma * self.bias_epsilon,\n",
    "        )\n",
    "    \n",
    "    @staticmethod\n",
    "    def scale_noise(size: int) -> torch.Tensor:\n",
    "        \"\"\"Set scale to make noise (factorized gaussian noise).\"\"\"\n",
    "        x = torch.randn(size)\n",
    "\n",
    "        return x.sign().mul(x.abs().sqrt())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NoisyNet + DuelingNet + Categorical DQN\n",
    "\n",
    "#### NoisyNet + DuelingNet\n",
    "\n",
    "NoisyLinear is employed for the last two layers of advantage and value layers. The noise should be reset at evey update step.\n",
    "\n",
    "#### DuelingNet + Categorical DQN\n",
    "\n",
    "The dueling network architecture is adapted for use with return distributions. The network has a shared representation, which is then fed into a value stream with atom_size outputs, and into an advantage stream with atom_size × out_dim outputs. For each atom, the value and advantage streams are aggregated, as in dueling DQN, and then passed through a softmax layer to obtain the normalized parametric distributions used to estimate the returns’ distributions.\n",
    "\n",
    "```\n",
    "        advantage = self.advantage_layer(adv_hid).view(-1, self.out_dim, self.atom_size)\n",
    "        value = self.value_layer(val_hid).view(-1, 1, self.atom_size)\n",
    "        q_atoms = value + advantage - advantage.mean(dim=1, keepdim=True)\n",
    "        \n",
    "        dist = F.softmax(q_atoms, dim=-1)\n",
    "```\n",
    "\n",
    "(Please see *04.dueling.ipynb*, *05.noisy_net.ipynb*, *06.categorical_dqn.ipynb* for detailed description of each component's network architecture.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        in_dim: int, \n",
    "        out_dim: int, \n",
    "        atom_size: int, \n",
    "        support: torch.Tensor\n",
    "    ):\n",
    "        \"\"\"Initialization.\"\"\"\n",
    "        super(Network, self).__init__()\n",
    "        \n",
    "        self.support = support\n",
    "        self.out_dim = out_dim\n",
    "        self.atom_size = atom_size\n",
    "\n",
    "        # set common feature layer\n",
    "        self.feature_layer = nn.Sequential(\n",
    "            nn.Linear(in_dim, 128), \n",
    "            nn.ReLU(),\n",
    "        )\n",
    "        \n",
    "        # set advantage layer\n",
    "        self.advantage_hidden_layer = NoisyLinear(128, 128)\n",
    "        self.advantage_layer = NoisyLinear(128, out_dim * atom_size)\n",
    "\n",
    "        # set value layer\n",
    "        self.value_hidden_layer = NoisyLinear(128, 128)\n",
    "        self.value_layer = NoisyLinear(128, atom_size)\n",
    "\n",
    "    def forward(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Forward method implementation.\"\"\"\n",
    "        dist = self.dist(x)\n",
    "        q = torch.sum(dist * self.support, dim=2)\n",
    "        \n",
    "        return q\n",
    "    \n",
    "    def dist(self, x: torch.Tensor) -> torch.Tensor:\n",
    "        \"\"\"Get distribution for atoms.\"\"\"\n",
    "        feature = self.feature_layer(x)\n",
    "        adv_hid = F.relu(self.advantage_hidden_layer(feature))\n",
    "        val_hid = F.relu(self.value_hidden_layer(feature))\n",
    "        \n",
    "        advantage = self.advantage_layer(adv_hid).view(\n",
    "            -1, self.out_dim, self.atom_size\n",
    "        )\n",
    "        value = self.value_layer(val_hid).view(-1, 1, self.atom_size)\n",
    "        q_atoms = value + advantage - advantage.mean(dim=1, keepdim=True)\n",
    "        \n",
    "        dist = F.softmax(q_atoms, dim=-1)\n",
    "        dist = dist.clamp(min=1e-3)  # for avoiding nans\n",
    "        \n",
    "        return dist\n",
    "    \n",
    "    def reset_noise(self):\n",
    "        \"\"\"Reset all noisy layers.\"\"\"\n",
    "        self.advantage_hidden_layer.reset_noise()\n",
    "        self.advantage_layer.reset_noise()\n",
    "        self.value_hidden_layer.reset_noise()\n",
    "        self.value_layer.reset_noise()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rainbow Agent\n",
    "\n",
    "Here is a summary of DQNAgent class.\n",
    "\n",
    "| Method           | Note                                                 |\n",
    "| ---              | ---                                                  |\n",
    "|select_action     | select an action from the input state.               |\n",
    "|step              | take an action and return the response of the env.   |\n",
    "|compute_dqn_loss  | return dqn loss.                                     |\n",
    "|update_model      | update the model by gradient descent.                |\n",
    "|target_hard_update| hard update from the local model to the target model.|\n",
    "|train             | train the agent during num_frames.                   |\n",
    "|test              | test the agent (1 episode).                          |\n",
    "|plot              | plot the training progresses.                        |\n",
    "\n",
    "#### Categorical DQN + Double DQN\n",
    "\n",
    "The idea of Double Q-learning is to reduce overestimations by decomposing the max operation in the target into action selection and action evaluation. Here, we use `self.dqn` instead of `self.dqn_target` to obtain the target actions.\n",
    "\n",
    "```\n",
    "        # Categorical DQN + Double DQN\n",
    "        # target_dqn is used when we don't employ double DQN\n",
    "        next_action = self.dqn(next_state).argmax(1)\n",
    "        next_dist = self.dqn_target.dist(next_state)\n",
    "        next_dist = next_dist[range(self.batch_size), next_action]\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNAgent:\n",
    "    \"\"\"DQN Agent interacting with environment.\n",
    "    \n",
    "    Attribute:\n",
    "        env (gym.Env): openAI Gym environment\n",
    "        memory (PrioritizedReplayBuffer): replay memory to store transitions\n",
    "        batch_size (int): batch size for sampling\n",
    "        target_update (int): period for target model's hard update\n",
    "        gamma (float): discount factor\n",
    "        dqn (Network): model to train and select actions\n",
    "        dqn_target (Network): target model to update\n",
    "        optimizer (torch.optim): optimizer for training dqn\n",
    "        transition (list): transition information including \n",
    "                           state, action, reward, next_state, done\n",
    "        v_min (float): min value of support\n",
    "        v_max (float): max value of support\n",
    "        atom_size (int): the unit number of support\n",
    "        support (torch.Tensor): support for categorical dqn\n",
    "        use_n_step (bool): whether to use n_step memory\n",
    "        n_step (int): step number to calculate n-step td error\n",
    "        memory_n (ReplayBuffer): n-step replay buffer\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self, \n",
    "        env: gym.Env,\n",
    "        memory_size: int,\n",
    "        batch_size: int,\n",
    "        target_update: int,\n",
    "        seed: int,\n",
    "        gamma: float = 0.99,\n",
    "        # PER parameters\n",
    "        alpha: float = 0.2,\n",
    "        beta: float = 0.6,\n",
    "        prior_eps: float = 1e-6,\n",
    "        # Categorical DQN parameters\n",
    "        v_min: float = 0.0,\n",
    "        v_max: float = 200.0,\n",
    "        atom_size: int = 51,\n",
    "        # N-step Learning\n",
    "        n_step: int = 3,\n",
    "    ):\n",
    "        \"\"\"Initialization.\n",
    "        \n",
    "        Args:\n",
    "            env (gym.Env): openAI Gym environment\n",
    "            memory_size (int): length of memory\n",
    "            batch_size (int): batch size for sampling\n",
    "            target_update (int): period for target model's hard update\n",
    "            lr (float): learning rate\n",
    "            gamma (float): discount factor\n",
    "            alpha (float): determines how much prioritization is used\n",
    "            beta (float): determines how much importance sampling is used\n",
    "            prior_eps (float): guarantees every transition can be sampled\n",
    "            v_min (float): min value of support\n",
    "            v_max (float): max value of support\n",
    "            atom_size (int): the unit number of support\n",
    "            n_step (int): step number to calculate n-step td error\n",
    "        \"\"\"\n",
    "        obs_dim = env.observation_space.shape[0]\n",
    "        action_dim = env.action_space.n\n",
    "        \n",
    "        self.env = env\n",
    "        self.batch_size = batch_size\n",
    "        self.target_update = target_update\n",
    "        self.seed = seed\n",
    "        self.gamma = gamma\n",
    "        # NoisyNet: All attributes related to epsilon are removed\n",
    "        \n",
    "        # device: cpu / gpu\n",
    "        self.device = torch.device(\n",
    "            \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "        )\n",
    "        print(self.device)\n",
    "        \n",
    "        # PER\n",
    "        # memory for 1-step Learning\n",
    "        self.beta = beta\n",
    "        self.prior_eps = prior_eps\n",
    "        self.memory = PrioritizedReplayBuffer(\n",
    "            obs_dim, memory_size, batch_size, alpha=alpha, gamma=gamma\n",
    "        )\n",
    "        \n",
    "        # memory for N-step Learning\n",
    "        self.use_n_step = True if n_step > 1 else False\n",
    "        if self.use_n_step:\n",
    "            self.n_step = n_step\n",
    "            self.memory_n = ReplayBuffer(\n",
    "                obs_dim, memory_size, batch_size, n_step=n_step, gamma=gamma\n",
    "            )\n",
    "            \n",
    "        # Categorical DQN parameters\n",
    "        self.v_min = v_min\n",
    "        self.v_max = v_max\n",
    "        self.atom_size = atom_size\n",
    "        self.support = torch.linspace(\n",
    "            self.v_min, self.v_max, self.atom_size\n",
    "        ).to(self.device)\n",
    "\n",
    "        # networks: dqn, dqn_target\n",
    "        self.dqn = Network(\n",
    "            obs_dim, action_dim, self.atom_size, self.support\n",
    "        ).to(self.device)\n",
    "        self.dqn_target = Network(\n",
    "            obs_dim, action_dim, self.atom_size, self.support\n",
    "        ).to(self.device)\n",
    "        self.dqn_target.load_state_dict(self.dqn.state_dict())\n",
    "        self.dqn_target.eval()\n",
    "        \n",
    "        # optimizer\n",
    "        self.optimizer = optim.Adam(self.dqn.parameters())\n",
    "\n",
    "        # transition to store in memory\n",
    "        self.transition = list()\n",
    "        \n",
    "        # mode: train / test\n",
    "        self.is_test = False\n",
    "\n",
    "    def select_action(self, state: np.ndarray) -> np.ndarray:\n",
    "        \"\"\"Select an action from the input state.\"\"\"\n",
    "        # NoisyNet: no epsilon greedy action selection\n",
    "        selected_action = self.dqn(\n",
    "            torch.FloatTensor(state).to(self.device)\n",
    "        ).argmax()\n",
    "        selected_action = selected_action.detach().cpu().numpy()\n",
    "        \n",
    "        if not self.is_test:\n",
    "            self.transition = [state, selected_action]\n",
    "        \n",
    "        return selected_action\n",
    "\n",
    "    def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:\n",
    "        \"\"\"Take an action and return the response of the env.\"\"\"\n",
    "        next_state, reward, terminated, truncated, _ = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        \n",
    "        if not self.is_test:\n",
    "            self.transition += [reward, next_state, done]\n",
    "            \n",
    "            # N-step transition\n",
    "            if self.use_n_step:\n",
    "                one_step_transition = self.memory_n.store(*self.transition)\n",
    "            # 1-step transition\n",
    "            else:\n",
    "                one_step_transition = self.transition\n",
    "\n",
    "            # add a single step transition\n",
    "            if one_step_transition:\n",
    "                self.memory.store(*one_step_transition)\n",
    "    \n",
    "        return next_state, reward, done\n",
    "\n",
    "    def update_model(self) -> torch.Tensor:\n",
    "        \"\"\"Update the model by gradient descent.\"\"\"\n",
    "        # PER needs beta to calculate weights\n",
    "        samples = self.memory.sample_batch(self.beta)\n",
    "        weights = torch.FloatTensor(\n",
    "            samples[\"weights\"].reshape(-1, 1)\n",
    "        ).to(self.device)\n",
    "        indices = samples[\"indices\"]\n",
    "        \n",
    "        # 1-step Learning loss\n",
    "        elementwise_loss = self._compute_dqn_loss(samples, self.gamma)\n",
    "        \n",
    "        # PER: importance sampling before average\n",
    "        loss = torch.mean(elementwise_loss * weights)\n",
    "        \n",
    "        # N-step Learning loss\n",
    "        # we are gonna combine 1-step loss and n-step loss so as to\n",
    "        # prevent high-variance. The original rainbow employs n-step loss only.\n",
    "        if self.use_n_step:\n",
    "            gamma = self.gamma ** self.n_step\n",
    "            samples = self.memory_n.sample_batch_from_idxs(indices)\n",
    "            elementwise_loss_n_loss = self._compute_dqn_loss(samples, gamma)\n",
    "            elementwise_loss += elementwise_loss_n_loss\n",
    "            \n",
    "            # PER: importance sampling before average\n",
    "            loss = torch.mean(elementwise_loss * weights)\n",
    "\n",
    "        self.optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        clip_grad_norm_(self.dqn.parameters(), 10.0)\n",
    "        self.optimizer.step()\n",
    "        \n",
    "        # PER: update priorities\n",
    "        loss_for_prior = elementwise_loss.detach().cpu().numpy()\n",
    "        new_priorities = loss_for_prior + self.prior_eps\n",
    "        self.memory.update_priorities(indices, new_priorities)\n",
    "        \n",
    "        # NoisyNet: reset noise\n",
    "        self.dqn.reset_noise()\n",
    "        self.dqn_target.reset_noise()\n",
    "\n",
    "        return loss.item()\n",
    "        \n",
    "    def train(self, num_frames: int, plotting_interval: int = 200):\n",
    "        \"\"\"Train the agent.\"\"\"\n",
    "        self.is_test = False\n",
    "        \n",
    "        state, _ = self.env.reset(seed=self.seed)\n",
    "        update_cnt = 0\n",
    "        losses = []\n",
    "        scores = []\n",
    "        score = 0\n",
    "\n",
    "        for frame_idx in range(1, num_frames + 1):\n",
    "            action = self.select_action(state)\n",
    "            next_state, reward, done = self.step(action)\n",
    "\n",
    "            state = next_state\n",
    "            score += reward\n",
    "            \n",
    "            # NoisyNet: removed decrease of epsilon\n",
    "            \n",
    "            # PER: increase beta\n",
    "            fraction = min(frame_idx / num_frames, 1.0)\n",
    "            self.beta = self.beta + fraction * (1.0 - self.beta)\n",
    "\n",
    "            # if episode ends\n",
    "            if done:\n",
    "                state, _ = self.env.reset(seed=self.seed)\n",
    "                scores.append(score)\n",
    "                score = 0\n",
    "\n",
    "            # if training is ready\n",
    "            if len(self.memory) >= self.batch_size:\n",
    "                loss = self.update_model()\n",
    "                losses.append(loss)\n",
    "                update_cnt += 1\n",
    "                \n",
    "                # if hard update is needed\n",
    "                if update_cnt % self.target_update == 0:\n",
    "                    self._target_hard_update()\n",
    "\n",
    "            # plotting\n",
    "            if frame_idx % plotting_interval == 0:\n",
    "                self._plot(frame_idx, scores, losses)\n",
    "                \n",
    "        self.env.close()\n",
    "                \n",
    "    def test(self, video_folder: str) -> None:\n",
    "        \"\"\"Test the agent.\"\"\"\n",
    "        self.is_test = True\n",
    "        \n",
    "        # for recording a video\n",
    "        naive_env = self.env\n",
    "        self.env = gym.wrappers.RecordVideo(self.env, video_folder=video_folder)\n",
    "        \n",
    "        state, _ = self.env.reset(seed=self.seed)\n",
    "        done = False\n",
    "        score = 0\n",
    "        \n",
    "        while not done:\n",
    "            action = self.select_action(state)\n",
    "            next_state, reward, done = self.step(action)\n",
    "\n",
    "            state = next_state\n",
    "            score += reward\n",
    "        \n",
    "        print(\"score: \", score)\n",
    "        self.env.close()\n",
    "        \n",
    "        # reset\n",
    "        self.env = naive_env\n",
    "\n",
    "    def _compute_dqn_loss(self, samples: Dict[str, np.ndarray], gamma: float) -> torch.Tensor:\n",
    "        \"\"\"Return categorical dqn loss.\"\"\"\n",
    "        device = self.device  # for shortening the following lines\n",
    "        state = torch.FloatTensor(samples[\"obs\"]).to(device)\n",
    "        next_state = torch.FloatTensor(samples[\"next_obs\"]).to(device)\n",
    "        action = torch.LongTensor(samples[\"acts\"]).to(device)\n",
    "        reward = torch.FloatTensor(samples[\"rews\"].reshape(-1, 1)).to(device)\n",
    "        done = torch.FloatTensor(samples[\"done\"].reshape(-1, 1)).to(device)\n",
    "        \n",
    "        # Categorical DQN algorithm\n",
    "        delta_z = float(self.v_max - self.v_min) / (self.atom_size - 1)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            # Double DQN\n",
    "            next_action = self.dqn(next_state).argmax(1)\n",
    "            next_dist = self.dqn_target.dist(next_state)\n",
    "            next_dist = next_dist[range(self.batch_size), next_action]\n",
    "\n",
    "            t_z = reward + (1 - done) * gamma * self.support\n",
    "            t_z = t_z.clamp(min=self.v_min, max=self.v_max)\n",
    "            b = (t_z - self.v_min) / delta_z\n",
    "            l = b.floor().long()\n",
    "            u = b.ceil().long()\n",
    "\n",
    "            offset = (\n",
    "                torch.linspace(\n",
    "                    0, (self.batch_size - 1) * self.atom_size, self.batch_size\n",
    "                ).long()\n",
    "                .unsqueeze(1)\n",
    "                .expand(self.batch_size, self.atom_size)\n",
    "                .to(self.device)\n",
    "            )\n",
    "\n",
    "            proj_dist = torch.zeros(next_dist.size(), device=self.device)\n",
    "            proj_dist.view(-1).index_add_(\n",
    "                0, (l + offset).view(-1), (next_dist * (u.float() - b)).view(-1)\n",
    "            )\n",
    "            proj_dist.view(-1).index_add_(\n",
    "                0, (u + offset).view(-1), (next_dist * (b - l.float())).view(-1)\n",
    "            )\n",
    "\n",
    "        dist = self.dqn.dist(state)\n",
    "        log_p = torch.log(dist[range(self.batch_size), action])\n",
    "        elementwise_loss = -(proj_dist * log_p).sum(1)\n",
    "\n",
    "        return elementwise_loss\n",
    "\n",
    "    def _target_hard_update(self):\n",
    "        \"\"\"Hard update: target <- local.\"\"\"\n",
    "        self.dqn_target.load_state_dict(self.dqn.state_dict())\n",
    "                \n",
    "    def _plot(\n",
    "        self, \n",
    "        frame_idx: int, \n",
    "        scores: List[float], \n",
    "        losses: List[float],\n",
    "    ):\n",
    "        \"\"\"Plot the training progresses.\"\"\"\n",
    "        clear_output(True)\n",
    "        plt.figure(figsize=(20, 5))\n",
    "        plt.subplot(131)\n",
    "        plt.title('frame %s. score: %s' % (frame_idx, np.mean(scores[-10:])))\n",
    "        plt.plot(scores)\n",
    "        plt.subplot(132)\n",
    "        plt.title('loss')\n",
    "        plt.plot(losses)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Environment\n",
    "\n",
    "You can see the [code](https://github.com/Farama-Foundation/Gymnasium/blob/main/gymnasium/envs/classic_control/cartpole.py) and [configurations](https://github.com/Farama-Foundation/Gymnasium/blob/main/gymnasium/envs/classic_control/cartpole.py#L91) of CartPole-v1 from Farama Gymnasium's repository."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# environment\n",
    "env = gym.make(\"CartPole-v1\", max_episode_steps=200, render_mode=\"rgb_array\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set random seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 777\n",
    "\n",
    "def seed_torch(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    if torch.backends.cudnn.enabled:\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "seed_torch(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "# parameters\n",
    "num_frames = 10000\n",
    "memory_size = 10000\n",
    "batch_size = 128\n",
    "target_update = 100\n",
    "\n",
    "# train\n",
    "agent = DQNAgent(env, memory_size, batch_size, target_update, seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv8AAAE/CAYAAADR+6wLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABZv0lEQVR4nO3deXxU1f3/8dcn+woIJOwQkIiCCiqK+75rtYv+qm1duqmtbbXrF6tWq7W1+2artWqtS7Uudd+1dRcRFRBZZAsCYQkgkD2ZmfP7494ZJskkJCSzZOb9fDzyyMy9d+49dwhzP3Pu53yOOecQEREREZH0l5XsBoiIiIiISGIo+BcRERERyRAK/kVEREREMoSCfxERERGRDKHgX0REREQkQyj4FxERERHJEAr+05iZTTKz982s1sy+k+z2iIiIpDIzqzKz45PdDpF4UvCf3n4EvOycK3XO/SnZjWnPzG41syVmFjKzC2Os/66ZrTezbWZ2h5nlR60bbGaPmFm9ma0ysy+0e+1xZrbYzBrM7H9mNi5qnZnZL81ss//zKzOzuJ5sgpjZwWb2gpltMbMaM3vQzEZEre/y3M2swn+/Gvz3r9OLYDq/jyIiIulKwX96Gwd82NlKM8tOYFtimQd8E3iv/QozOwmYCRwHVAATgJ9GbfIXoAUYBnwRuNnMpvivHQr8B7gaGAzMAf4d9dqLgE8DU4F9gdOBi/vsrHrBzHJ6uYvdgFvx3rNxQC3wj6j1Ozv3+4D3gSHAlcBDZlbWybFS9n0UERGR2BT8pykz+y9wDHCTmdWZ2R5mdqeZ3WxmT5tZPXCMmZ3mpwZtN7PVZnZt1D4qzMyZ2Zf9dZ+Y2SVmdqCZzTezrWZ2U7vjfsXMFvnbPhfd496ec+4vzrmXgKYYqy8AbnfOfeic+wS4HrjQP0Yx8DngaudcnXPudeBx4Dz/tZ8FPnTOPeicawKuBaaa2Z5R+/6tc26Nc24t8Nvwvrvxvk40s1f8uxGbzOzfUeumRPW6bzCzH/vL883sD2ZW7f/8IXwXw8yONrM1ZvZ/ZrYe+IeZZZnZTDNb7veoP2Bmg7vTPufcM/55b3fONQA3AYe1e19jnruZ7QHsD1zjnGt0zj0MfOC/17Hs8vsoIpLKdvK5PdTMnvSvgVvM7DUzy/LX/Z+ZrTUv3XaJmR2X3DMR6UjBf5pyzh0LvAZ8yzlX4pz7yF/1BeAGoBR4HagHzgcGAacB3zCzT7fb3QygEvg88Ae8HuHjgSnA/zOzowD81/0YL/gu849/3y6ewhS8OwNh84BhZjYE2AMIRp1TeP2UWK91ztUDyztb3+61O3M98DxeD/to4M8AZlYKvAg8C4wEJgIv+a+5EjgYmIbXS34QcFXUPofj3aEYh9eb/h28HvWj/H19gnenA/9Y861dmlMXjqTt3Z+uzn0KsMI5V9vJ+vZ68z6KiKSyrj63vw+swbvODcO77jkzmwR8CzjQOVcKnARUJbTVIt2g4D/zPOace8M5F3LONTnnXnbOfeA/n48XrB/V7jXX+9s+j/dl4T7n3Ea/t/c1YD9/u4uBXzjnFjnnAsDPgWld9f53oQTYFvU8/Lg0xrrw+tJOXruz9duAkm7mq7fiBekj/ffkdX/56cB659xv/eW1zrm3/XVfBK7z37MavPSl86L2GcLrbW92zjXivY9X+j3qzXh3Ls4KpwQ55/Z1zv1rZw01s32BnwA/jFrc1bnv7H1rrzfvo4hIKuvqc7sVGAGMc861Oudec845IAjkA5PNLNc5V+WcW56U1ot0QcF/5lkd/cTMZvgDPGvMbBtwCTC03Ws2RD1ujPG8xH88Dvijfyt0K7AFMGDULrSzDhgQ9Tz8uDbGuvD6cI91T9cPAOr8D++d+RHeOc02sw/N7Cv+8jF4dxdiGQmsinq+yl8WVuOnJ4WNAx6Jeh8X4V1UhnWjfYCXngQ8A1zmnHstalVX576z96293ryPIiKprKvP7V8Dy4DnzWyFmc0EcM4tAy7H67DZaGb3m1n0Z71ISlDwn3naB2b/wsuXH+OcGwjcghfc7orVwMXOuUFRP4XOuTd3YV8f4t1qDZsKbHDObQY+AnLMrLLd+g9jvdYfI7B7Z+vbvbZLzrn1zrmvO+dG4vXQ/9UPtFf7x4ilGi+gDxvrL4vstt32q4FT2r2PBf6dlp3y77S8iHfH5u52q7s69w+BCX4KU6z17e3y+ygikuI6/dz27+x+3zk3AfgU8L1wbr9z7l/OucP91zrgl4lttsjOKfiXUmCLc67JzA7CGxOwq24BroiqujPQzM7ubGMzyzOzArwvG7lmVhAeNAXcBXzVzCab2W54uZZ3QiSH/z/AdWZWbGaHAWcC4UD3EWBvM/ucv/+fAPOdc4uj9v09Mxvl98p8P7zvnTGzs81stP/0E7wP9yDwJDDczC73B4qVmtkMf7v7gKvMrMy8SkQ/Ae7p4jC3ADeE06X8153ZzfaNAv4L/MU5d0uMTTo9d38MxVzgGv/f4jN4VXwe7uRwu/w+ioikuE4/t83sdL/4gwHb8a4BQfPm1jnWHxjchHdnPJik9ot0SsG/fBMviK7F+3B7YFd35Jx7BK+X434z2w4sAE7p4iXP4304HopXnrIRb4AqzrlngV8B/8O73boKuKZduwuBjXgf0t9wzn3ov7YGr0LNDXgB+gzgnKjX/g14Aq+SzQLgKX8ZAH46zxc7afOBwNtmVod3x+Qy59xKf5DsCXi9QOuBpXjVlgB+hldudL5/zPf8ZZ35o7/v5/1/l1n+OXSnfV/DK4t6jXlVnur8tnbr3PHep+l479uNwFn++4mZHdHDfYmI9FddfW5X4t1drQPeAv7qnHsZL9//RmAT3nWgHG8wsEhKMaXnioiIiIhkBvX8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGUPAvIiIiIpIhFPyLiIiIiGSInGQ3AGDo0KGuoqIi2c0QEUlJ77777ibnXFmy25FMuk6IiMTW02tESgT/FRUVzJkzJ9nNEBFJSWa2KtltSDZdJ0REYuvpNUJpPyIiIiIiGULBv4iIiIhIhlDwLyIiIiKSIRT8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGUPAvIiIiIpIhdhr8m9kYM/ufmS0ysw/N7DJ/+WAze8HMlvq/d4t6zRVmtszMlpjZSfE8ARERERER6Z7u9PwHgO875/YCDgYuNbPJwEzgJedcJfCS/xx/3TnAFOBk4K9mlh2PxouIiIiISPftdIZf59w6YJ3/uNbMFgGjgDOBo/3N/gm8DPyfv/x+51wzsNLMlgEHAW/1deNFZOcaW4I8/cE6WoKhPt+3AcfsWc6wAQWdbtMaDPH0B+toaAn2+fH7kwMrBjOxvCTZzcg4by3fzMbaJs6cNirZTRERSQk7Df6jmVkFsB/wNjDM/2KAc26dmZX7m40CZkW9bI2/rP2+LgIuAhg7dmyPGy4i3fPch+v5/oPz4rb/8w8Zx3Vn7t3p+rdXbOGy++fG7fj9xY2f3UfBfxI89O4aZq3YrOBfRMTX7eDfzEqAh4HLnXPbzazTTWMscx0WOHcrcCvA9OnTO6wXkb6xrbEVgGcuO4LdivL6dN+fu/lNapsCXW5T2+Qd/+6vHkRleWmfHr8/GVDYo74WERGRuOjW1cjMcvEC/3udc//xF28wsxF+r/8IYKO/fA0wJurlo4HqvmqwiPRMON1m3JAiivL6NgAtysumqbXrdJ6mgLd+1KBChg/sPD1IRERE4q871X4MuB1Y5Jz7XdSqx4EL/McXAI9FLT/HzPLNbDxQCczuuyaLSE80tng98wU5fT/uviB358F/c2sosq2IiIgkV3e6AQ8DzgM+MLO5/rIfAzcCD5jZV4GPgbMBnHMfmtkDwEK8SkGXOucye6SfSBI1tgYpzM0mK6vTVL1dVpCbRVNr1wOJw18OFPyLiIgkX3eq/bxO7Dx+gOM6ec0NwA29aJeI9JGGliBFefEJvAtys6lr7jrnvyngfTnIz9GcgiIiIsmmq7FImmtsCVIYp+A/Pyd7pz3/4bQfBf8iIiLJp6uxSJqLZ89/YTcH/OZkGTnZ+riRxOu8MJ2ISGbS1VgkzTW0Bins4yo/YQU5WTsP/luDyvcXERFJEQr+RdJcY0uAwtz4/FfvVrWfQIiCOB1fREREekZXZJE056X9xKnnv5vVfvLjUGZUREREek7Bv0iai+eA34LcbJoCQZzrfJLu5kCIfPX8i4iIpARdkUXSXGNrkKI45dwX5GbjHLQEO+/9b24NxmWCMZHu6urLqYhIplHwL5Lm4l3nH+gy9aepVT3/mcTMJpnZ3Kif7WZ2ebttjjazbVHb/CRu7YnXjkVE+qn4JAKLSMrw0n7il/MPXl7/wMLcmNs0B9Tzn0mcc0uAaQBmlg2sBR6JselrzrnTE9g0ERFBPf8iaS0QDNESDMWv5z8n3PPfecWfplZV+8lgxwHLnXOrkt0QERHx6IosksYa/KA8uWk/qvaTwc4B7utk3SFmNs/MnjGzKYlslIhIJlPwL5LGGlu84D9ek2xFp/10RnX+M5OZ5QFnAA/GWP0eMM45NxX4M/BoJ/u4yMzmmNmcmpqauLVVRCST6IosksYaWhLV899V2o9m+M1QpwDvOec2tF/hnNvunKvzHz8N5JrZ0Bjb3eqcm+6cm15WVhb/FouIZAAF/yJprKElACQg+A/sLO1HHzUZ6Fw6Sfkxs+FmZv7jg/CuRZvj1RAV+hQR2UHVfkTSWDjtJxHVfjrjpf2o5z+TmFkRcAJwcdSySwCcc7cAZwHfMLMA0Aic4+JUjN9U61NEpA0F/yJprDFhA35jB//OOX+GXwX/mcQ51wAMabfslqjHNwE3JbpdIiKitB+RtBbO+S+M4wy/0Hnw3+ynAyntR0REJDXoiiySxhrjPeA3J5z2Ezvnv9lfrrQfERGR1KDgXySN7aj2E6+c/657/psC4VKj+qgRERFJBboii6SxcLWfwiRN8hX+UqBJviSZ4jOUWESkf1LwL5LGGuOc85+dZeRlZ0V6+NsL5/yr51+SxVC5HxGRaLoii6SxhtYgOVlGXhwH3ObnZnWe9uMvL1DPv4iISEpQ8C+SxhpbgnFL+QkryM3uIvj3q/2o519ERCQl7PSKbGZ3mNlGM1sQtezfZjbX/6kys7n+8goza4xad0unOxaRuGtsCcat0k9YQW5W59V+IgN+1fMvIiKSCrpTAuROvMlY7govcM59PvzYzH4LbIvafrlzbloftU9EeqGhNRi3Sj9hBTnd6PlXnX8REZGUsNOowDn3qplVxFpnZgb8P+DYPm6XiPSBxpZA3Ab7hnWd9qOefxERkVTS2+64I4ANzrmlUcvGm9n7ZvaKmR3Ry/2LSC80JCDtpzA3u4u0H7/ajwb8ShI5VOtTRCSst/kA5wL3RT1fB4x1zm02swOAR81sinNue/sXmtlFwEUAY8eO7WUzRCSWhpYgpQXxTfvJz82irjkQc12kzr8G/EqSmCp9ioi0sctXZDPLAT4L/Du8zDnX7Jzb7D9+F1gO7BHr9c65W51z051z08vKyna1GSLShcaWYELSfsLzCbSnUp8iIiKppTfdcccDi51za8ILzKzMzLL9xxOASmBF75ooIruqoTWQgGo/2ZH0nvbCy9XzLyIikhq6U+rzPuAtYJKZrTGzr/qrzqFtyg/AkcB8M5sHPARc4pzb0pcNFpHu8+r8x7vaT+eTfDWH035U7UdERCQldKfaz7mdLL8wxrKHgYd73ywR6QuJGPDbVbWf5kCI/JwsTInXIiIiKSG+XYIikjTOORpbkzvJV1NrUGU+Januf2c1AK3BELnZugMlIqJPQpE01RwI4RwUJqLUZyCIcx3LKTa1hpTyIymhs3EpIiKZRldlkTTV4FfgKYpzz3t+bjbOQUuwY3DVHFDPv6SGWF9ORUQykYJ/kTTV0OLV3i+K94BfP7hvaukY/De1hihQpR9JAQr9RUQ8uiqLpKlw7f14p/2Eg/umQMdBv02BIPmq8S8iIpIyFPyLpKlw2k/cJ/nyg/tYFX+a1fMvKUJZPyIiHl2VRdJUJOc/AaU+gZgVf5qU859xzGySmc2N+tluZpe328bM7E9mtszM5pvZ/nFvmIJ/ERFApT5F0lZjq5fzH/dqP3l+2k+Mnv+m1hBDitXHkEmcc0uAaQD+jO9rgUfabXYK3gzwlcAM4Gb/t4iIxJmuyiJpqtEfgBv3Ab9dpf0EguSr5z+THQcsd86tarf8TOAu55kFDDKzEfFsiFPXv4gIoOBfJG3tqPYT/1KfAE0x6qg3t4YiXw4kI50D3Bdj+ShgddTzNf6yNszsIjObY2ZzampqetUQ5fyLiHgU/IukqcbWxFb7CVcXitbUGiRfA34zkpnlAWcAD8ZaHWNZh/DcOXerc266c256WVlZXzdRRCQj6aoskqYSPeC3OUapz+aAev4z2CnAe865DTHWrQHGRD0fDVTHszHq+BcR8Sj4F0lT4eA/3sH3jmo/6vmXNs4ldsoPwOPA+X7Vn4OBbc65dYlrmohI5lK1H5E01dgSoCA3i6ysWBkWfaewk1KfgWCIQMip5z8DmVkRcAJwcdSySwCcc7cATwOnAsuABuDL8W6TU9K/iAig4F8kbTW0BONe6QeiZvht1/Pf7A8A1iRfmcc51wAMabfslqjHDrg0oW1K5MFERFKYrsoiaaqxJRj32X0hutRn257/8JeB/Bx9zIiIiKQKXZVF0pTX8x//4D8ry8jLzqKp3YDfpkjPv9J+REREUoWCf5E01diamOAfID83q0Opz2a/51/Bv6QCpfyLiHgU/IukqcaWYNxr/IcV5GZ3KPUZTgNS2o+IiEjq0FVZJE01tAYSMuAXvEG9HXL+A+r5l9ThNORXRARQ8C+SthoS2PNfmJvdsdpPuOdf1X4kFSj2FxEBFPyLpK3GliBFCep1L4gR/Id7/vNV519ERCRlKPgXSVOJ7PkvyMnukPYT7vlXnX9JBer4FxHx6KoskqYSOeA3P7djqc9m5fxLClG1HxERz06DfzO7w8w2mtmCqGXXmtlaM5vr/5wate4KM1tmZkvM7KR4NVykK9c9sZBrH/8w2c1ImkAwREswRFFuogb8Znco9alJvkRERFJPd67KdwInx1j+e+fcNP/naQAzmwycA0zxX/NXM1O3nyTci4s28PyH65PdjKRp8APvRNX590p9tkv70SRfkkJU7UdExLPT4N859yqwpZv7OxO43znX7JxbCSwDDupF+0R6rCUQYs0nDVRva6K2qTXZzUmKJr8XPnHVfrI6DvjVJF+SQpT2IyLi6c39+G+Z2Xw/LWg3f9koYHXUNmv8ZR2Y2UVmNsfM5tTU1PSiGSJtrf6kgZB/oV9eU5/cxiRJQ0vie/47Bv+a5EtERCTV7OpV+WZgd2AasA74rb/cYmwbs7/FOXerc266c256WVnZLjZDpKOqTTsC/qUbapPYkuRJTvDfPu0nSHaWkZut4F+STx3/IiKeXboqO+c2OOeCzrkQ8Hd2pPasAcZEbToaqO5dE0V6ZqUf/GdnGcs21iW5NcnR2BoAoDBRM/zmeNV+XFRuRVNriAL1+ouIiKSUXboym9mIqKefAcKVgB4HzjGzfDMbD1QCs3vXRJGeWbmpnoGFuVSWl7A0g4L/uau3smDtNmBHz39hgvLt83OzcY42g36bWoPkK99fUoRT0r+ICAA77RY0s/uAo4GhZrYGuAY42sym4d1JrQIuBnDOfWhmDwALgQBwqXMuGGO3InFTtbmeiqHFjB1cxPsff5Ls5iTMDx+cR11zgJd/eHRS0n7Am9gr8jignn9JHYr9RUQ8Ow3+nXPnxlh8exfb3wDc0JtGifRG1aYGDqzYjQllJTwxr5qGlgBFCUp/SZam1iArNtUTDDkenLOGknzvfBM2w68/i29TIMhAciNtUs+/iIhIalG3nKSVptYg1dsaqRhaTGV5CQDLN6Z/xZ9lG+sIhhz5OVnc/PJytvslThPV8x9OL4qu+NPUGlKlHxERkRSjK7OklY+3NOAcjB9aTOUwL/hfujH9K/4sWe+d4w9PmsTarY3cO+tjgITO8Au0qfjTHAiqxr+IiEiKUfAvaSVc6adiSDHjhhSTm20ZMeh3yYZa8nKyuPDQCqaOHsgSv8RpwtN+onr+m9XzLyIiknJ0ZZa0Egn+hxaTm53F+KHFLN2Q/sH/4vW1TCwrISc7i8uOrwQgJ8vIS1DwXZATI+1HPf8Zy8wGmdlDZrbYzBaZ2SHt1h9tZtvMbK7/85N4tymkEb8iIkA3BvyK9CdVm+oZUpzHwEJv0GlleSkfVm9Lcqvib8n67Ry2+1AAjplUzt6jBrB6S2PCjh8e2NvYruc/fEdAMs4fgWedc2eZWR5QFGOb15xzpyeqQYr9RUQ8Cv4lrazc5JX5DJtYXsIzC9bR1Jq+vdBbG1rYsL2ZScNLATAz/vD5aQmd4GxH2k9Unf9AkPyc9HzPpXNmNgA4ErgQwDnXArQks02gnn8RkTB1y0laqdpcT8WQHcF/5bASQg5W1KRvxZ/F/mDfcPAPMLG8lJP3HtHZS/pcYaS2f3S1n6B6/jPTBKAG+IeZvW9mt5lZcYztDjGzeWb2jJlNiXejQor9RUQABf+SRhpaAmzY3sz4oTsyDCrLvYA4nSv+LF63HYA9hw9IWhsKYpT6bA6E0vZui3QpB9gfuNk5tx9QD8xst817wDjn3FTgz8CjsXZkZheZ2Rwzm1NTU9OrRmmGXxERj4J/SRtVmxoA2qT9VAwtIjvLEpoCk2hLNtQysDCXYQPyk9aGWKU+m1qDqvaTmdYAa5xzb/vPH8L7MhDhnNvunKvzHz8N5JrZ0PY7cs7d6pyb7pybXlZW1qtGqedfRMSjK7OkjXCln/FRwX9+TjbjhhSldcWfxetr2XN4KWaWtDa0L/XpnKOpVT3/mcg5tx5YbWaT/EXHAQujtzGz4eb/wZrZQXjXos3xbJdy/kVEPAr+pV9paAnw5PzqmOuqNu+o8R+tsrwkbdN+QiHHR37wn0w7Sn16Pf8tQe+3gv+M9W3gXjObD0wDfm5ml5jZJf76s4AFZjYP+BNwjotzXo6CfxERj6r9SL9y76yPueHpRewxrJQ9hrUNeFduqqe8NJ/i/LZ/1pXlpby4aCMtgVDC6t4nytqtjdS3BJmUxHx/gKwsIy87K1LqM/wlQGk/mck5NxeY3m7xLVHrbwJuSmybEnk0EZHUpSuz9CuzVniZAR9t6NiTX9WuzGfY7uXFBEOOj7c0xL19iRar0k+yFORmRdJ+mv3f+er5lxShnn8REY+Cf+k3giHH7KotADFz+Ks21zN+SMfgf8LQEgCW16Rf3v+S9V6ln9QI/rMjpT6bA37aj3r+JUVowK+IiEdXZuk3Fq3bTm1TAKBD9Z6a2mY21bVQOaykw+smlHlfCNKx1v/i9bWM3q2QkvzkZ/AV5GZH0n2a1PMvKUY9/yIiHgX/0m+8vdLr9d971IAOA3gX+rXup4wc2OF1pQW5lJfmp2nPf/IH+4a1SftRz7+kGNX5FxHx6Mos/cbbKzYzdnARR1SWsXJTPa3BHTXlP6zeBsDkEbEHvk4oK2ZFmgX/zYEgKzbVp0TKD4R7/sMDfoORZSKpQLG/iIhHwb/0CyE/33/G+MFUlpfQGnSs2rwjjWdh9XZG71bIwKLcmK/fvayE5TX1adX7t3xjPcGQS3qln7CCnOi0H1X7kdSinH8REY+uzNIvfLSxlq0NrcyYMITKcq+nO3rQ78Lq7Z32+gNMKCthW2MrW+pb4t7WRFnsD/ZNlbSf/NwsGiJpP+r5l9SinH8REY+Cf+kXZi33SnzOGD+YieUlmMFSf9BvfXOAlZvrY+b7h+3uD/pdnkaDfpesryUvO6vNjMbJNHnEAD5Ys5UP1myL9Pwr+JdUoeBfRMSj4F/6hbdXbmHUoELGDC6iMC+b0bsVRoL/xeu34xxMGdl5z//uZV4VoHTK+1+8vpbdy0vIzU6N/8aXHjuRISX5XPnoB9S3eFWZlPYjqUKxv4iIR1dmSXnOOWav3MKMCYMjyyrLS1nqT/S1sNpLf5ncRfA/clAh+TlZaVXxJ5Uq/QAMKMjl6tMnM3/NNv75ZhWgnn9JHer5FxHxKPiXlLdsYx2b61s4ePyQyLLK8hJWbKonEAzxYfV2divKZcTAgk73kZ1ljB9anDa1/rc1tLJ+e1PKVPoJ+9S+Iziicigf+l/I1PMvqSKoEb8iIkA3gn8zu8PMNprZgqhlvzazxWY238weMbNB/vIKM2s0s7n+zy1xbLtkiFl+ff/onv+J5SW0BEKs/qSRD6u3M3nkAMysy/14FX/So+d/cQrN7BvNzLjuzL3J84N+9fyLiIiklu50y90JnNxu2QvA3s65fYGPgCui1i13zk3zfy7pm2ZKJntuwXpG71bI2MFFkWWVw7ygd9G67SzZUNvlYN+wCWXFrP6kMVKJpj9b4qc8pVLaT9j4ocX84MQ9GDu4SD3/kjLU7y8i4tnpldk59yqwpd2y551zAf/pLGB0HNomwspN9by+bBOfnz6mTc/+xHJvAO8zC9bTEgh1WeYzbPeyEoIhx8ebG+LW3kRZvL6WAQU5DB/QeapTMl105O688sOjycrq+m6MiIiIJFZfdMt9BXgm6vl4M3vfzF4xsyP6YP+Swe6dtYqcLOPzB41ps7wkP4eRAwt4YeF6oOtKP2ET2pX73FTXzAV3zOadqi0dtv39Cx/xi2cW9bb5ALy0aAPn3f42LYHQzjfupsXrtrPn8J2nOiVTKrdNMpC6/kVEgF4G/2Z2JRAA7vUXrQPGOuf2A74H/MvMYkZlZnaRmc0xszk1NTW9aYakqabWIA++u4aT9h5OeWnHHu7KYaU0tYbIz+lerfvwNis2eXn/P396Ea98VMMPHpxHU+uOVKB3V33CH19ayj9er6KuORBzX921rbGVmf/5gNeWbmLWis292leYc46PNtSx54jUS/kRSVVO0b+ICNCL4N/MLgBOB77onFdDzTnX7Jzb7D9+F1gO7BHr9c65W51z051z08vKyna1GZLGnphXzbbGVr40Y1zM9ZV+6s+eIwaQ041a96UFuQwbkM/yjfW8uXwT/3lvLUfuUcaqzQ389eXlAASCIa585AMKc7NpCYZ49aOOX0zXfNL9tKHfPr+EzXXN5OVk8cLCDd1+XVfWfNJIXXMg5Qb7ioiISOrbpeDfzE4G/g84wznXELW8zMyy/ccTgEpgRV80VDLPPW9/zMTyEg6OqvITrXKYF/x3J+UnbMLQEhav387Vjy5g7OAibj3vAM6cNpJbXl7O8po67nyzisXra/nN2VMZVJTbIWB/cn41h//yf9z22s7/rOet3srds1Zx/iEVHL1HGS8u2oDrg1rjS9an7mBfkVSlMv8iIp7ulPq8D3gLmGRma8zsq8BNQCnwQruSnkcC881sHvAQcIlzrmNCtchOfLBmG/NWb+WLM8Z2mjs+sdwLfrsz2Dds9/JiPqzezvKaeq47cwoFudlcedpe5Odm8f0H5vG7Fz7iuD3LOXWf4Ry7Zzn/XbyR1uCOXP0736gCdqQMhX28uYHP/+0tzr7lTf7z3hoaWgJc+egHlJXk8/0T9+CEycNYt60pUv++N8KVfvYYpuBfpLsU/IuIeLpT7edc59wI51yuc260c+5259xE59yY9iU9nXMPO+emOOemOuf2d849Ef9TkHR079urKMzN5rP7d15Iar8xg7juzCl8er9R3d7vhKHe3YLT9hnB0ZPKASgvLeBHJ+/J3NVbCTnHtWdMwcw4cfIwtjW2MqfqE8CrrT9n1Sd89/g92GNYKd/613ssr6nj9aWbOOMvr7N4fS2b6lr43gPz2P/6F1iwdjvXfGoKpQW5HLtnOVkGz+9C6k/Vpnp+8fQiPqlv8dtRy6hBhZQW5PZ4XyIiIpLZcpLdAJH2mlqDPDl/HaftO4KBhZ0HuFlZxvmHVPRo30dNKuPwxUP5yacmt1n+hYPGMm/1Vg6ZMIQx/nwCR1SWRXL1D9l9CPfMWkV+ThbnHzKOz+4/ijP/8gbn3jqLTXXNTCwv4e/nT2fs4CLeWr6Ze2d/zKDCXE7dZzgAQ0ryOWDcbrywcAPfO2HHMJj3P/6EymGllOR3/l/x+icX8tLijTyzYD1/P386S9ZvV8qPiIiI7BLNwCMp56VFG6lrDvCZHvTod9fuZSXc87UZDGtXHz87y/jN2VP53AE77jQU5+dw+MShvLBoPbVNrTzy3lpO33ckuxXnMWZwEbd86QC2NrZy/F7D+M83D2PckGLMjEMnDuUvX9ifGz6zT5uUpRMmD2PRuu2s3uINk3ls7lo+89c3+cqd73RaBnTB2m28tHgjn9lvFE2tQT7z1zdYXlOvwb4iPaSsHxERj4J/STmPzV1LeWk+B08YkuymcMLkYaze0sivnl1CfUuQLx08NrLuoPGDefeq4/nbeQd02XO/Y1/eXYAXF21g3uqt/Oih+VQMKWL2yi1c+8SHMQcD/+mlpQwoyOGnZ07hiW8fzh7DSgmGHJN7MMhZJNHMbJCZPWRmi81skZkd0m69mdmfzGyZmc03s/2T1VYRkUyjtB9JKdsaWnl5SQ3nHTKO7BSYHfa4vcoxg7tnrWLKyAFMGzOozfqe5N2PH1rMxPISHn5vDbe8spyy0nwe/sah3Pb6Sm5+eTl7Di9tk8a0sHo7zy/cwHeP34MBBbkMKMjl/osO5uUlGzl+r2F9dIYicfFH4Fnn3FlmlgcUtVt/Cl41uEpgBnCz/ztu+qLSlohIOlDPv6SUZxasoyUY4sxpI5PdFMAbDBwO+M87eFyvZ609YfIwFqzdTm1TgL+fP50hJfn88MRJHL9XOT99YiGPz6smFPKClD//dyml+TlceFhF5PUFudmcvPeIbs1rIJIM/sSORwK3AzjnWpxzW9ttdiZwl/PMAgaZ2YjEtlREJDMpgpCU8tjcasYPLWafUQOT3ZSIsw4YzahBhZzRB19IPj1tFOWl+fz+89PYyy9RmpVl/P7z06gsL+E7973Pkb/+H794ehHPLFjPlw+r6HLQs0gKmgDUAP8ws/fN7DYzaz8F9yhgddTzNf6yuFG/v4iIR8G/pIz125qYtXIzZ04b2ese9r70xRnjeGPmsRTl9T5LbtLwUt7+8XGcNGV4m+WlBbk8/q3DuekL+zFmtyL+9uoKSvJz+Mrh43t9TJEEywH2B252zu0H1AMz220T6z94h/jczC4yszlmNqempuNs2z2hrB8REY9y/iVlPDm/GufgjKmpkfITL519scnLyeL0fUdy+r4jWVFTRyDkGFSUl+DWifTaGmCNc+5t//lDdAz+1wBjop6PBqrb78g5dytwK8D06dMVvouI9AH1/EvKeGxuNfuOHsiEspJkNyXpJpSVaAZf6Zecc+uB1WY2yV90HLCw3WaPA+f7VX8OBrY559bFuWXx3b2ISD+h4F9SQkNLgA/WblMVG5H08G3gXjObD0wDfm5ml5jZJf76p4EVwDLg78A3492g215bGe9DiIj0C0r7kZRQvbUJgHFD2lcEFJH+xjk3F5jebvEtUesdcGki2zRn1SeJPJyISMpSz7+khOqtjQCMHFSY5JaIiIiIpC8F/5ISFPyLiIiIxJ+Cf0kJa7c2kmUwrDQ/2U0RERERSVsK/iUlrN3ayPABBZq5VkRERCSOFGlJSqje2qiUHxEREZE4U/AvKaF6a5OCfxEREZE4U/AvSRcKOdZtU8+/iIiISLwp+Jek21TXTGvQMWpQQbKbIiIiIpLWFPxL0q1VmU8RSYAFa7cluwkiIkmn4F+SLjy7r4J/EYmnVz6qSXYTRESSTsG/JJ0m+BIRERFJDAX/knRrtzZSkp/DgIKcZDdFRNLY8o11yW6CiEjSKfiXpKve2sioQYWYWbKbIiJpbHN9S7KbICKSdDsN/s3sDjPbaGYLopYNNrMXzGyp/3u3qHVXmNkyM1tiZifFq+GSPqq3NTJSlX5EJM4mlpckuwkiIknXnZ7/O4GT2y2bCbzknKsEXvKfY2aTgXOAKf5r/mpm2X3WWklLmuBLRBIhP0c3u0VEdvpJ6Jx7FdjSbvGZwD/9x/8EPh21/H7nXLNzbiWwDDiob5oq6aixJciW+hYF/yISd82BULKbICKSdLvaDTLMObcOwP9d7i8fBayO2m6Nv6wDM7vIzOaY2ZyaGpVfy1TV27xKP6MU/ItInN3++koO/cVLyW6GiEhS9fU90FgjNl2sDZ1ztzrnpjvnppeVlfVxM6S/WPuJynyKSOJUb2tKdhNERJJqV4P/DWY2AsD/vdFfvgYYE7XdaKB615sn6W5HjX8N+BURERGJt10N/h8HLvAfXwA8FrX8HDPLN7PxQCUwu3dNlHRWvbWRLINhAxT8i4iIiMTbTmdVMrP7gKOBoWa2BrgGuBF4wMy+CnwMnA3gnPvQzB4AFgIB4FLnXDBObZc0sHZrE8MGFJCbrSocIiIiIvG20+DfOXduJ6uO62T7G4AbetMoyRzVWxuV7y8iIiKSIOpulaTyJvhS8C+STsysysw+MLO5ZjYnxvqjzWybv36umf0kGe0UEclEO+35F4mXUMixbmsTJ++tfH+RNHSMc25TF+tfc86dnrDWiIgIoJ5/SaJN9c20BEOMVs+/iIiISEIo+Jek2bCtGVClH5E05IDnzexdM7uok20OMbN5ZvaMmU1JZONERDKZ0n4kaRpbvUJQxfn6MxRJM4c556rNrBx4wcwWO+dejVr/HjDOOVdnZqcCj+KVhm7D/+JwEcDYsWP7rHHOOcxizUkpIpL+1PMvSdMSCAGQl6M/Q5F04pyr9n9vBB4BDmq3frtzrs5//DSQa2ZDY+wnLjPBu5jzzouIZAZFXZI0zQGv5z9fwb9I2jCzYjMrDT8GTgQWtNtmuPld72Z2EN61aHOi2hhU9C8iGUz5FpI0zX7Pf35OdpJbIiJ9aBjwiB/b5wD/cs49a2aXADjnbgHOAr5hZgGgETjHufhG5AW5WTS1ep85IQX/IpLBFPxL0oR7/pX2I5I+nHMrgKkxlt8S9fgm4KZEtuudK49nn2ufByAUSuSRRURSi6IuSZqWSM+//gxFJL5KC3Ijj5X2IyKZTFGXJE2zgn8RSQKl/YhIJlPUJUnT3KpqPyKSeKGQgn8RyVyKuiRpWoIa8CsiidcSCBEIhojzGGMRkZSk4F+Sprk1iBnkZmuyHRFJnF8/t4SJVz7D7a+vTHZTREQSTsG/JE1zIER+TpZm2hSRhBgxsACAxetrAXjo3TXJbI6ISFIo+JekaQ6EyMvWn6CIJEa4m+GDtdsAzfQrIplJkZckTXMgRH6u8v1FJDHa32VcsqE2SS0REUkeBf+SNM2BoMp8ikjCrN3amOwmiIgknSIvSZrmQEhlPkVEREQSSJGXJE1LIKQynyKSMFNGDkh2E0REkk7BvyRNuNqPiEgiXHZcZbKbICKSdIq8JGmaW4NK+xGRuDpl7+GRx3sMK01iS0REUkNOshsgmaslGKIkX3+CIhI/fz53PxpbgwBoShERkV70/JvZJDObG/Wz3cwuN7NrzWxt1PJT+7LBkj6aW5XzLyLxlZOdRWlBLgBGx+j/O/e9z8btTYlulohI0uxy8O+cW+Kcm+acmwYcADQAj/irfx9e55x7ug/aKWmoORAkP1dpPyKSGLF6/h+fV81BP38p8Y0REUmSvoq8jgOWO+dW9dH+JAM0B0Lka4ZfEUkQzegrItJ3wf85wH1Rz79lZvPN7A4z262PjiFppiUQUs+/iCRMqIvof1tjawJbIiKSPL2OvMwsDzgDeNBfdDOwOzANWAf8tpPXXWRmc8xsTk1NTW+bIf1Qs+r8i0gChQf+xnLVowsS2BIRkeTpi27XU4D3nHMbAJxzG5xzQedcCPg7cFCsFznnbnXOTXfOTS8rK+uDZkh/0xxQqU8RSZyRgwo7Xbd43fYEtkREJHn6IvI6l6iUHzMbEbXuM4C6U6QD55w/w6+CfxFJjIGFuSy87qSY65ZurEtwa0REkqNXRdbNrAg4Abg4avGvzGwa4ICqdutEAAiEHCGHgn+RNGRmVUAtEAQCzrnp7dYb8EfgVLxKcRc6595LSNtilPsUEckkvQr+nXMNwJB2y87rVYskIzQHQgDK+RdJX8c45zZ1su4UoNL/mYE3VmxGIhrlUMkfEcls6naVpGjxg3/l/ItkpDOBu5xnFjCoXcpo3BTmZnP2AaMTcSgRkZSkyEuSojngVd1Q2o9IWnLA82b2rpldFGP9KGB11PM1/rK4MzN+ffZUHv7GoYk4nIhIylHklSI21zVz0A0v8sGabcluSkI0t/ppP6rzL5KODnPO7Y+X3nOpmR3Zbn2sxPsO+TjxLAl9wDhNQSMimUmRV4pY80kjG2ubWbKhNtlNSYhwzn9etnL+RdKNc67a/70ReISOJZ/XAGOino8GqmPsRyWhRUT6mIL/FNHQ4qXB1DcHktySxGiJDPjVn6BIOjGzYjMrDT8GTqRjyefHgfPNczCwzTm3LsFNFRHJSIq8UkRjqxf017dkRvAfyflX2o9IuhkGvG5m84DZwFPOuWfN7BIzu8Tf5mlgBbAMbzLIbyajoRcfOSEZhxURSapelfqUvpNpPf870n4U/IukE+fcCmBqjOW3RD12wKWJbFcsV5y6F397dUWbZTW1zWyqa2avEQOS1CoRkfhS5JUidgT/wSS3JDEiaT+5yvkXkdRx3G9f5pQ/vpbsZoiIxI2C/xTRmHE9/yr1KSKpZ3tTZnwGi0jmUuSVIsK5/pmT868BvyIiIiKJpsgrRYR7/usyJO2nWTP8ikgK6MvZfh99fy1f+PusPtufiEg8KPJKEeGc/4aMSfsJ9/wr519EkueaM6YAMLAwt8vt3v/4E96p2tJmWWswxK+fW0xtUysAl/97Lm8u3xyfhoqI9BEF/ymiIdLznyHBf6tKfYpI8pXk53DewePIzuo46fCdb6xk1govmP/MX9/k7FvearP+iXnV/OV/y/nls4sT0lYRkb6gUp8potHP9Q9/CUh3KvUpIqkiyyDkXIfl1z6xEIBnLz8isqxi5lMcv9cwvnH0BJ5ZsB6AD9Zuz5hiDSLS/yn4TxGZVudfM/yKSKowM0KhjsF/2Ml/aFv688VFG3hx0YbI83mrtzLlmufi1j4Rkb6kyCtFNLZmWNpPIEReThZmHW+1i4gkUpYZ7Tv+K2Y+1at9rt3ayHsffwLAqx/V8LdXlvdqfyIifUU9/yki3PPfHAgRCIbISfN0mOZAkPw0P0cR6R+yDILOUb21sU/255zjsBv/C8DsHx/H+XfMBuDio3bvk/2LiPSGoq8UEZ3rX58Bef8tgZAG+4pISsjOMhpaghzqB+y9tbymLvJ42ca6LrYUEUk8RV8pojFqcq9MyPtvDoRU5lNEUkJfp1se/7tXI4+/cNvbfbpvEZHeUvCfIhpaghTnZfuPMyX415+fiCRfINj5YN++9O6qTxJyHBGRrijnP0U0tgQpK82nfnNDRszy29wa1Oy+IpISshL0UfS5m98EoCA3i0XXncz4K55mzOBCXvvRsfzjjZVs2N7MzFP2TExjRCRjKfpKAc456lsClJXmA5kxy29LUD3/IpIqElt1rKk1xPgrngZg9ZZGQiHHT59YyC2qCCQiCaDoKwU0B0KEHJHgPxPKfTa3KudfRFJDVzX+E+H4378SeexiTDYmItKXFPyngEa/uk9ZiRf812dEzr/SfkQkNfx7zuqkHn9FTX3k8ZG//h+vL93EY3PXRpZNu+557nh9ZTKaJiJpqFfRl5lVmdkHZjbXzOb4ywab2QtmttT/vVvfNDV9NfgTfA0NB/8ZkPOvtB8RkY5Wb2nkS7e/zWX3z2VLfQsAWxtaue7JhUlumYiki76Ivo5xzk1zzk33n88EXnLOVQIv+c+lC+Eyn+G0n4wo9dmqOv8iIl0559a3Osw0XLWpnteW1gBeilAwySlLItL/xCP6OhP4p//4n8Cn43CMtBKe4GtISQYF/4EQeZrhV0SkUx9t6DhB2NG/eZnzbvdmDP7Bg/PZ/cdPJ7pZItLP9Tb6csDzZvaumV3kLxvmnFsH4P8uj/VCM7vIzOaY2ZyamppeNqN/Cwf/xXnZFOdlZ84MvxrwKyLSbbe9tiLy+O5Zq3j4vTUAvLtqS5tZhUVEutLbOv+HOeeqzawceMHMFnf3hc65W4FbAaZPn57R9y3DA34L87Ipys/JkJ7/oNJ+RER64GdPLYo8vvrRBZHHn7v5LQCqbjwt4W0Skf6nV9GXc67a/70ReAQ4CNhgZiMA/N8be9vIdBfu+S/Ky6EkPycjev41w69IejOzbDN738yejLHuaDPb5heLmGtmP0lGG9PNd/89N9lNEJF+YJejLzMrNrPS8GPgRGAB8Dhwgb/ZBcBjvW1kumvwB/wW5WVTnJ+dIT3/IZX6FElvlwGLulj/ml8sYppz7rpENSqdPfL+WgLBULKbISIprjfR1zDgdTObB8wGnnLOPQvcCJxgZkuBE/zn0oXG1qi0n7yctJ/kKxAMEQw55fyLpCkzGw2cBtyW7LZkmolXPsO1j39IIBiiqTXInKotyW6SiKSYXc75d86tAKbGWL4ZOK43jco0O9J+sinJz2FjbVOSWxRfLX7PlNJ+RNLWH4AfAaVdbHOI33lUDfzAOfdhIhrWG6/+8Bi2NrYwYmAhB97wYof1y244hYlXPpOElrV155tV3PlmVeT5c5cfiRkMLs6LzCcjIpmrtwN+pQ80+D39BTnZFOfnUL8pvXP+m1u94F9pPyLpx8xOBzY65941s6M72ew9YJxzrs7MTgUeBSpj7Osi4CKAsWPHxqW9PTF2SBFjKQLgli8dwH8Xb+CBOWsi63NStHzxSX94FYDSghw+uPakJLdGRJItNT+pMkxDS5CivGyysozivOy0T/vZ0fOvtB+RNHQYcIaZVQH3A8ea2T3RGzjntjvn6vzHTwO5Zja0/Y6cc7c656Y756aXlZUloOltnXXAaGZfGftG9sl7D+eaT01h6uiBAOxeVgzAxPISJpaXJKyNPVHblN7XFhHpHgX/KaCh1Qv+AYrzcyJ3AtJVuOdfaT8i6cc5d4VzbrRzrgI4B/ivc+5L0duY2XAzM//xQXjXos0Jb2wXxg4u4jdnT6W8tKDTbYrzc/jd56cBkJ1lALz4vaN48XtHRWZsB+8LwWs/Oiau7RUR6S5FXymgsSVIYTj49yf5CqXxlO3NAS+tSWk/IpnDzC4xs0v8p2cBC/yc/z8B5zjnUupD72tHjO/WdhOGFnPxURO49bzpbZZX+r3/V522F09953DGDC7iqtP2Ysb4wQCcf8i4vm1wN1XMfIrT//xa5PnVjy7gv4s3JKUtIpIcyvlPAQ0tAYpyvX+K4nzvd2NrMPI43TQH1PMvkgmccy8DL/uPb4lafhNwU3Ja1T3DBuzo8c8y6Kw/xsy44pS9Oiy/+UsHMG/1Vo7cY0e60teOmIBz8PbKLeS1Gx/wzGVHkJudxfG/e6VvTqALC9ZuZ/rPXuSVHx7N3bNWcfesVaz8xal8sHYb+44eFPfji0hyKfpKAQ3RPf9+wJ/Otf4jwX+ucv5FJDVZ1OP5157E/GtP7NHrBxbmtgn8wxwdv0WYwV4jBnQYK3DHhdM7bNtXNtU1M+Wa5yLP7561ijNueoNnF6yP2zFFJDWkZ9dyP9PYEp3z7/2uaw5QnsxGxVE47Uc9/yKSar57/B78/sWPGDmoMLKsJE53YZ/89uEsr6njxMnDY66fNma3uBw3lp885lVavfftVeRkGeu2NfKlg8fhD80QkTSi4D8FNLQEGVSUC0BxXk5kWboK9/wr519EUkFBbhZNfiGCbx87kVP2Gc4ew7qaomDXFfmf8UX5Oew9aiB7jxrYZn3VjacRDDmybEdltPaGDchnw/bmyPMDK3bjnapP+qR9ry3dxGtLNwGwz+hBTBszCIBZKzazqa6Z0/cd2SfHEZHkUfSVAhpbgxT6F4RwD1M6l/tsUc6/iKSQQYV5kcdZWRa3wB/gnAPH8H8n78k3j969022yswwzIz8nmw+uPZFRgwq57XwvBeiurxzEAxcf0mb7rx0xIS5tvXfWKn793GJCIcc5t87iW/96Py7HEZHEUs9/CvAG/HrpPkWZlPOvOv8ikmFysrP4RheBf3ulBbm8MfNYwLsrEHb9mVO42k/VKYjT+KkH3/UmMDt2z3RNQhXJTOp6TQHRA35L/Jz/+nRO+2lVzr+IpI7bLvB61f/yhf2T3JLuO2nvHeMEDp/YYX60PvW5m9+K6/5FJLHU859kzrnIDL+QGdV+dszwq+BfRJJv71ED2/Sq9wflpQU8e/kRLFq3PTLBWNgXZ4zl3rc/jtuxtzW28kl9C8MHFkTuOry+dBPlA/LjmjIlIn1DwX+StQRDBEMuEvyHB4Olc/C/Y4Zfpf2IiOyqPYcPYM/hA9osq7rxNN5cviluwf/LSzZy4T/eaXM8gC/d/nab5yKSuhT8J1mjn94TDvqL/S8B9c1pnPajaj8iIn3qzZnHRoopzBg/JG7HiQ78AZZuqGWgX61ORPoHRV9J1hAJ/r2gPyc7i4LcLOpb0rjn36/zr+BfRKRvjBxUSMXQYsCrFnTlqXslZKDuCb9/lRN//2rcjyP91/KaOipmPsU7VVuS3RTxKfpKsnDwHx7wC16t/3RO+2kJhMjNtg55qiIi0je+fuQEzj5gdEKOtbWhNSHHkf7pjWXevBGPz61OckskTGk/SdY+7Qe8Qb/pHPw3B0LK9xcRibOT9x7Ob8+eSsXQIkYOKuSQX/w32U2SDBTu5nO4pLZDdlDPf5I1+Ok9RdE9//k51KV1zn9QKT8iInFmZnzugNEcMG4wIwYWMvOUPQE4ojJ+pUEPuP4Fqrc28uyCdXE7hvSN+uYAf35pKYFOZpLuM+aF/66L2H9TXTO/eHoRwZC+ICSCIrAka2iNlfaTHflSkI5aAiGV+RQRSbCzDhhNSX4OPzppT5b87GRu+sJ+fPSzU/jPNw/ts2Nsrm/h0Bv/yyX3vIfzo713qrawvUmpQXe/VcUNTy3s8euaA0EqZj7FH19cyuL126mY+RTvrup9/vyX//EOv33hIx5LUDpOZ2F9bVMrX7rtbf726gpeXVqTkLZkOkVgSdbYbsAvZEraj/70REQSaWhJPgt+ehL7jB5Ifk42p+87krycLKaM9MqFHrp731YJOuWPr/HRhlrOvuUtvvbPOX2671SyrbG1Wz3WVz/2IX9/bWWX21xwx2zueH3HNqGQY+0njQD8/sWPuH/2agCe/mB9t9rWEgjxSX1LzHWz/QG4LcEQlVc+zZ1vdGzbxu1NMQfrVm2q79Y5B0Mu0uXfvuf/reWbqW8OcPYtb7F4fa2/Tcd9Ouc6PQfZNYrAkixS7Sd3R85/SX5Oms/wG1Laj4hIisjPyWblL07l3q/N6NP9Ll5fG6kENHvlFipmPgXAgrXbImVJU51zjrrmAKf/+TXO8+cyiNbYEmTqT5/n+ie9Hv2Vm+p3us+KmU9x3ROx7wC88lEN1z25kN88t4TmQJBfPLOIY3/7SmT9nW9WAXD76yup2lRPcyDIso11PPDOaqb/7AUqZj7FZfe/z/2zvXkevvWv99jv+hcAaA2GOPJX/+PZBW2/ODgHrUHHtTHa9NeXl0eOu72plbVbG7n55eUc/ZuX+c3zXhtrapsj79PfXlnOx5sbuHvWKn793GJ2//HTXP3Yhx32u2F7E+f+fRbf/ffcSOAfLRAM8evnFrOtsZX731nNfte/wEcbOm4H8El9C7W6s9QjGvCbZI1+ek902k9RXnZa9/y3BDXgV0QklZiflz1sQD4btjfH7TjhLwDQPyYEu2/2an78yAedrm/0U3fvfLOKmrpmnpq/jr+fP50TJg8DvC8PVz66gMkj2k7GdscbKynJz+Z7J04CYN7qrdwza1Vk/U3/W8bg4rwu7xTMrtrCjx6a32H5Y3OreWxuNUdPKuf5hRsA+MXTi/j3nNVsbWjlknvebfPev//xJ5HHLYEdnXMfVm+LfNl4av465n68lbVbGyPb3vzycm72vxwATB0ziHmrt/KLZxbHbG/0cZr896194F/fHGRh9XaW1dTxl/8t5y//W86ew71Zo5dtrIs5g/R+179AQW4Wi68/JeZxpSN1vyZZ+zr/EB7wm77Bf3MgqLQfEZEU9PaPj092E+KqJRDi+w/MiwSxS9bXdjke4YWFbXvJK2Y+xf2zP2bD9iZWb2kgumL1U/O9Qc6vfrQjb72uOcC/3v6Yqx5d0GHff/rvMipmPkXFzKc48y9v8OC7a9qs/90LH3V5LrEC/2gH/+KlyOO/vbqiTUnWJ+fvyPOPPu4eVz1D9dZGFqzdRk1t2y+B0YF/LPNWb+1y/eL1tVxwx2ycc5hfAyjULs3n2/e9z6l/eo3v3Pd+m9cBfLylgW/c827ki0O0ptb+cScplmDI8eCc1Qkd7LzLPf9mNga4CxgOhIBbnXN/NLNrga8D4b/+Hzvnnu5tQ9NVOL2nMDc6+M+moSXo/Qex9KuFr7QfEZHU9edz92PUboV89q9vJvzYSzfU8o83q/jZmXuT1QdzwcxdvZWm1iD7jR3E7a+vZPeyEh5+bw1b6pv5x5cP4qQ/vMrg4jweuPhg7pn1MVecuif5Odm8u2oLIwcVxrwGz/zPjjsBN31hvw7r7561irv9XvyvHT5+l9sez07Ab/3r/U7XfeHvs6ja3BCX477yUQ3jr3iafUcPBGDNJ11/oYh2o39H4dP71bB7WQktgRCTRw7Yyat2WLW5noffW8t3j6+kanMDn/7LGzz57cMZM7ioZyfRx+56q4qfPrGQptYg5x1SkZBj9ibtJwB83zn3npmVAu+a2Qv+ut87537T++alv8aWAAW5WW0+5IrzcwiGHM2BEAW56Zce0xwIUVqgjDMRkVT0qakjk3bsr981h6rNDXzt8PFMKCvp9f4+/Zc3APjBiXvwm+c/Yrw/C3J0YL2lvoXjf+eNTQinuYQNLcnrcv9dBdEAt73e9QDfVBSvwD/a/DXbdvm1F9/9buTxr8/aN/K4tqmV0oJc3qnawr7+oPaH313D9x+cx+PfOowzbvL+Fv7f9NE8MGc12xpbeXxeNZceM5EVNXWMG1KclMlHw4OZt9QnbtzCLne/OufWOefe8x/XAouAUX3VsEzR0BJsM8EXeAN+Ib7f+pOpRZN8iYikvN9/fiqn7D28Te/2hKHFDCnuOiDurlteWc4PHpxHxcynuPXV5dQ2tRLwUx+y+uCud2tU/frwXfbwgNx3qj7ZaRoLwKY6VZlJZT+MSn165aOaSHWpM/7sBfrff3AeQCTwB2+Ac64f5AeCjg+rt3Hsb1/h9ztJs4qb8DwICZwErU9yL8ysAtgPCA+F/5aZzTezO8xst744RrpqbAm2yfeHHbP9NqTpRF/NgSD5uUr7ERFJZZ/ZbzQ3f+kATt93ZKQS0LABBZwxrW/uDNz4zGIe8vPNf/70Yva59vlIGsjO8tl3JhAMcdFdXZcXPexGzXicTr71r/d53J+zYMmG2jZf/trLzvJikBcXbeC0P70OwK2vreiw3fpt3tgO8FLSNm5virm/+uYAFTOf4t/vfLzL7e9qErS+1uvcCzMrAR4GLnfObTezm4Hr8eZzuB74LfCVGK+7CLgIYOzYsb1tRr/VECP4L8n3nqdrz39zIERetoJ/kXRmZtnAHGCtc+70dusM+CNwKtAAXBi+kyypKdwR79gxWDOeZlf1bhKryT95jpZ4z1wrKeem/y2LPJ760+djbnPEr/4XefzB2h3pRy2BEKs217N6SyOvfLSxTaWlqhtP44Tfv4oZ/Opz+3LwhCFtxgps8L8U3PLKCj5/YM9i2mSM7OxV8G9muXiB/73Ouf8AOOc2RK3/O/BkrNc6524FbgWYPn16xs7n3NAapLBd2k+xn/aTrrP8tgRC6vkXSX+X4aWDxhqRdwpQ6f/MAG72f0uKCqfhJLAgCXXNAZ6aX81B44dEcvVj+cljCzhg3G6MG1KMc44F1ds7BP7RJSklMzTswnxJR/365ZjLw/MyOLcj1Si6XGr47805xxk3vc5JU4Zz6TETe3TsRAbCvan2Y8DtwCLn3O+ilo9wzq3zn34G6FjfSiIaWwIU5cZO+0nnnn/l/IukLzMbDZwG3AB8L8YmZwJ3OW86z1lmNqjdtUNSzKhBhQAcPamMPcpLuSPGbLB9be9rnos8jg60lm6o5YTfv8o/LjyQx+dV88j7a7nrrVWxdiHSJ2L9vf9v8UYK87JpaAnwlTu9FLPwYOn5a7Z1O/hPRlHH3vT8HwacB3xgZnP9ZT8GzjWzaXhfYqqAi3txjLTX0BJk+IDcNsvCA37r0zjnX6U+RdLaH4AfAR1n5PGMAlZHPV/jL1Pwn6LGDC5izlXHM7goj6wsY8FPT+KJedVc8Z8POPegMdw3e/XOd9ILe139LIuuPxmAL9/5TpvfIsmws7+/L/x9Fm8u3xx5/sOTJuGc44jKMvJzsxhSnE9ZaT5/eHGpt0ECk/53Ofh3zr1O7FQl1fTvgcaWYJvZfcGr8w9Qn4ZpP6GQozXoNMmXSJoys9OBjc65d83s6M42i7Gsw5VPY8NSy9CS/MjjkvwczjlwDKfuM4KBhbn84rP7ep/voRCTrnoW8Hrrf/bkwj4pdxmeSbexJdij2vAiyRId+AP8+rklAPzm+dhVhf7032V8er9RfVLidmcUgSVZrAG/xXnhnv/0C/7DeXFK+xFJW4cBZ5hZFXA/cKyZ3dNumzXAmKjno4HqdtvgnLvVOTfdOTe9rKwsXu2VXWRmDCzccec6K8s6fLYX5ffdnC4VM5/ia3f1/97+kQMLmOpPctUbV522V7e2O+fAMbx7VWJmbj6zk0pQV58+mRnjByekDf3Z3J3MktxXFPwnWUNLoEOd/+L89A3+m/0puJX2I5KenHNXOOdGO+cqgHOA/zrnvtRus8eB881zMLBN+f7p48uHVUQef/Po3ftsXgCAN5Zt3vlGKeyBiw/hzSuO45FvHtat7TubZGzUoEKGDSiIPD95ynCqbjyNK0/1vhCEZxb+xtG7c+Pn9mVIST4f/eyUyPZXnz6Zbxy9OwCXHVfJ7/7f1F06n/Z+e/ZU7vnqDM4/ZByv/vCYyPKvHFbB3847gB+fuifzfnIiAHd/9SAuO66SBy4+hEuP2b1Pjt/fJSr/X9OsJllja8e0n7ycLPJysqipbU5Sq+KnOeDdulXaj0hmMbNLAJxzt+Clh54KLMMr9fnlJDZN+tg1n5rCNZ+aAkBBbjbvXn0CzjleWLiBi6JmZ01XQ0vyYk4OdsbUkRzk935nZRmv/98xLFpXy9c7mY/gmEll3HHhgZgZc6q2cNYtbwHw/RP24PSpIxk1qJDzDh7HZcdXRlKyvn7kBL5+5AQArjp9cpv95eVkccbUkTw+r5qhJXlsrvNijKK8bD67/2gmjxzAnW9UccNn9mFO1RbWb2/CzLjmsQV88+iJlA/I57L75wLwjwsP5OhJZbz38VbKS/P59zurOXnv4eRkZ3F45VAOrxwKwIOXHEJLIISZMagoj4uO9IL88ADuIyq9O3oHjR/MD0/ak6bWIMs21vGv2R/zr7e9mvn3fHUGg4vzOPVPr+3Cv0b/kqjqtAr+k6glEKI16DpU+wE4dlI5/3lvLd87cVKb26r9XXMgnPaj4F8k3TnnXgZe9h/fErXcAZcmp1WSDGbGiVOGJ7sZu2xIcR6b672AfuTAAl743lFMiapGFLbHsBKe+s4RVF75TJvlL37vSCaWtx3/Pnq3IkbvVkTVjadRMfMpAN664lj+/upK7nhjJb/83L6Y3xU8vWJHysy3j6uMPL7+03v36DzOnOYF/1NHD+LYPcupbQpw/iEVAOw5fAA3fm5fAGZMGBJ5zRlTd6TyhIP/oyeVYWYcMM6bx/UHJ02KebwDK3qW6lOQm83eowZyw6f35oiJQzlh8jBy/HmBot8nr72lLF5f26P9p7pElXhXBJZEjX4N2vY9/wDfPm4itc0B7nyjKsGtiq9I8B/jC4+IiKS3e77a/6Zz+Orh43lj5rGR52ZGcX4OR+3h9VpffNSEyLp9Rw8iNzuLpTfsSLGpuvG0DoF/Z0YMLOTq0/di8fUnUx6V1tNXjttrmBdEDy2mtCCX6z+9d8wYpDMXHlrB7RdMj3wpiRcz45R9RkQC/1i+fsQEqm48jae/c0Sbv6sHLj4krm2Lp/c/3pqQ46jnP4kaWr1veO1z/gGmjBzICZOHcfvrK/jK4RWUFqRH73847Ucz/IqIZJ5wOkh/cv4h4yjIzWb5z0/l2/e9x9eP8IL9P3x+Gk8vWMcXZ4zjU/uO5PQ/v87Rk7wvBLnZWUwZOYCKIZ1PThbtgYsPiRT/MDMKYnSQXXzkhIQNCO3MtWdMSerxwxZed1Ikdpo80ptH8L2rT6CxNRiZkyLsRydPYq8RA/jyP97hiMqhvLZ0EwAvfu8ojv/dK4lt+E7UNrUm5DgK/pMoPPtc+2o/Yd85tpJPLdzAP9+s4lvHVsbcpr9pifT8K/gXEclED3/jED53s5e/vteIASxatz3JLYIZ4wczoayE+2Z7eeZnHTCaYQPyufz4Pcj1O6uys4y/fvGAyGt2K87jizPGAbD3qIEs+OlJkXl6AJ76zhHdPv5B3aiEc8Wp3avuk86u/dRkXlq8MWan6eBOBpZ/82hvsq3wOINrHlvAP99axbghRZFtXvzekRz/u1fJz8mKZCgkR2JG/CoCS6LGnQT/+4weyHF7lnPb6yvTZrZf5fyLiGS2A8btCHR/8dl9ktiSHf598SFcd+YUCv0e9+8cW8kPT9ozEvh3R0kfljWV2C48bDx3dyN1LDwW4clvH95h3U/P3JuqG09r8287sbyUqhtPY8nPTuGtK47t8JpECSVooi9FYAnw9orNLFi7rcPyHT3/nX9gfOe4SrY2tHJHH0ySkgoU/IuIyMPfOITHLj2MaWMGUXXjaZ2WtOwrT0f1wlfdeBrvXnU85aX53P3Vg3jnSq8Gfm52Fo9/6zAuPLSC0bsVdrYr6Qce/sahVN14GnuP6no+hdvOn86Dl7QdIzBiYCETy3dMtPWbs6fy2f1GRe4sTChrm8r188/03RfYRAX/+poaZ6u3NHDBP2ZTmJvNf79/NLtF3ZYKj+ruarDN1DGDOHnKcH73wkcEQo7Lj6skK8u7LRQIhqje2sSYwYVxH3zTV5pbw6U+NeBXRCRTRff+A532sFeWl7B0Y90uHeOsA0Zz/F7DGDu4KJIXHjakJJ/ZV3ac+KpyWGnK5LVL/B0/eVjM5c9dfiS/em4xXz1sPOUDCjjrgNE453DOK9P6xrJNfPG2t7n8+Eq+MGMsKzfV8ffXOnbS/vGcaVz16AJqm7qXvTFucNHON+oDCv7jyDnHTx5bgGHUNgW48ZnF/PKsfSPrd5b2E/bHc6dx9aML+NNLS1m0bjszT9mTJ+et4/53Pmbdtib2GFbCF2eM4zP7j2JAig8M3jHDr3r+RUTE01mH5/dPnMQl9+za3AC/ObtvJq6SzJOdZVxxStsxFmYWmYTrsIlDI2MIAK48bXIk+P/CjLFccuTuPDG/mjOmjuTMaaMi2wWCIe59+2OuefzDmMf97gl79PGZxKbgP46e+3A9/1tSw1Wn7UVNXTN/e2UFZ00fzYEVg3HOMXfNVgCKu0j7Aa+X/Jef25cpIwdy3ZMLeWHhBgCOqBzKhYdW8PQH67jm8Q+54elFnQb/44YUcc6BYzh935E9KuvV18Iz/KrnX0RE2nvs0sOo2lzP4vW1nDB5GPuP3a3L7ScNK+X6T+/Nex9/wtaGVm55ZXmCWioSWzgN6NJjJnZYl5OdxQWHVsQM/h+79DAGFcU3/S3SjoQcJQPVNQe49vGF7DViABceWkFLMMST89Zx5SMf8Mg3D+O6Jxby7zmrOXHyMMYM3nluoZlxwaEVTB45gHeqtnDaPiMY55cQu/io3flgzTYen7eWev9uQjTn4J2qLfzwoflc/+RCPrv/aL44YyyVw7pXd7ivtAZDLNngTciRp55/ERFpp3xAPlPHDOLMqGWXHLV7p0H9c989EthRLeflJRs7nfjprq8cRFNrx2ukSF947UfHdHoHa2ei7yIkQr8O/pdtrGPlpvpkNyOmZxesZ0NtE3/90v7kZGeRk53FT8+YwtfumsNRv36ZTXXNfPvYiXz3+D16lK9/YMXgmDPm7TN6IPuM7nxgi3OO2Su3cO/bH3Pv26u4880qDho/mLP2H91mHEI8OOeYt2YrD8xZQ01tMxOGFrNbcWqnJ4mISOI4Oo+aZp6yZyT4f2PmsRx243873fbZy49k/bYmqjZ3jA2O9CflEomHMT3I1z9+r3JeXLQxjq3pWr8O/h+fV82fXlqa7GZ06ryDx7W5ZXn85GGcsvdwXl5Sw1++sD+n7TsiYW0xM2ZMGMKMCUPYVDeZh95dw32zP+ZHD89PyPGzDI7ds5wvzhjHkXuUkZ3VPwYoi4hI/P31iwfw91dXUF4ae1bbV394DPUtAUYNKqSsNJ+a2mae93v92xs+sIDhA/t+dlyRvnLbBQcC8IMH5yUlE8JcgsoKdWX69Oluzpw5PX7dhu1N1NQ2x6FFvZedZew5vLRDr35rMER9cyBheV1dCYUcSzfW0RqM/4QW5aX5cZmqXCQTmNm7zrnpyW5HMu3qdULST1NrkGDIUay6+iJAz68R/fp/zrABBQzrZwFlbnZWSgT+4JWrmjQ8sXn/IiIivVGQq4IRIr2hUZciIiIiIhlCwb+IiIiISIZQ8C8iIiIikiEU/IuIiIiIZAgF/yIiIiIiGULBv4iIiIhIhlDwLyIiIiKSIRT8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGMOdcstuAmdUAq3bx5UOBTX3YnP4m088f9B7o/NP//Mc558qS3Yhk0nWixzLtnDPtfCHzzlnn27keXSNSIvjvDTOb45ybnux2JEumnz/oPdD5Z/b5y85l4t9Ipp1zpp0vZN4563z7jtJ+REREREQyhIJ/EREREZEMkQ7B/63JbkCSZfr5g94Dnb9I1zLxbyTTzjnTzhcy75x1vn2k3+f8i4iIiIhI96RDz7+IiIiIiHRDvw7+zexkM1tiZsvMbGay2xNvZjbGzP5nZovM7EMzu8xfPtjMXjCzpf7v3ZLd1ngys2wze9/MnvSfZ8z5m9kgM3vIzBb7fweHZNj5f9f/219gZveZWUEmnb/0XLpcJ3bl89/MrvDPe4mZnRS1/AAz+8Bf9yczs2ScU3f05PM+Tc63R5/x/f2ce/qZ3h/P18zuMLONZrYgalmfnaOZ5ZvZv/3lb5tZxc7a1G+DfzPLBv4CnAJMBs41s8nJbVXcBYDvO+f2Ag4GLvXPeSbwknOuEnjJf57OLgMWRT3PpPP/I/Csc25PYCre+5AR529mo4DvANOdc3sD2cA5ZMj5S8+l2XWiR5///rpzgCnAycBf/fcD4GbgIqDS/zk5kSfSQ936vE+j8+32Z3x/P+eefqb34/O9k47t6ctz/CrwiXNuIvB74Jc7a1C/Df6Bg4BlzrkVzrkW4H7gzCS3Ka6cc+ucc+/5j2vxPhRG4Z33P/3N/gl8OikNTAAzGw2cBtwWtTgjzt/MBgBHArcDOOdanHNbyZDz9+UAhWaWAxQB1WTW+UvPpM11Yhc+/88E7nfONTvnVgLLgIPMbAQwwDn3lvMG/d1Fiv6f6eHnfTqcb08/4/v9OdOzz/R+eb7OuVeBLe0W9+U5Ru/rIeC4nd356M/B/yhgddTzNf6yjODf1tkPeBsY5pxbB94FAihPYtPi7Q/Aj4BQ1LJMOf8JQA3wD/82+G1mVkyGnL9zbi3wG+BjYB2wzTn3PBly/rJL0vI60c3P/87OfZT/uP3yVPQHuv95nw7n29PP+H59zrvwmd6vz7edvjzHyGuccwFgGzCkq4P35+A/1reajChdZGYlwMPA5c657cluT6KY2enARufcu8luS5LkAPsDNzvn9gPqyaAUFz8n8kxgPDASKDazLyW3VZLi0u460YPP/87OvV+8J7vwed+vz9fX08/4fn3Ou/CZ3q/Pt5t25Rx7fP79OfhfA4yJej4a73ZRWjOzXLwP/nudc//xF2/wbwnh/96YrPbF2WHAGWZWhXf7/lgzu4fMOf81wBrn3Nv+84fwLhSZcv7HAyudczXOuVbgP8ChZM75S8+l1XWih5//nZ37Gv9x++Wppqef9/39fKHnn/H9/Zx7+pne3883Wl+eY+Q1fvrUQDqmGbXRn4P/d4BKMxtvZnl4AyQeT3Kb4srP4bodWOSc+13UqseBC/zHFwCPJbptieCcu8I5N9o5V4H37/1f59yXyJzzXw+sNrNJ/qLjgIVkyPnj3Ro+2MyK/P8Lx+HlPWfK+UvPpc11Yhc+/x8HzvErgYzHGyA4208xqDWzg/19nk8K/p/Zhc/7fn2+sEuf8f39nHv6md7fzzdaX55j9L7Owvu/0vWdD+dcv/0BTgU+ApYDVya7PQk438PxbuXMB+b6P6fi5Xa9BCz1fw9OdlsT8F4cDTzpP86Y8wemAXP8v4FHgd0y7Px/CiwGFgB3A/mZdP766flPulwnduXzH7jSP+8lwClRy6f7/4eWAzfhT/iZqj/d/bxPh/Pt6Wd8fz/nnn6m98fzBe7DG9PQitdL/9W+PEegAHgQb3DwbGDCztqkGX5FRERERDJEf077ERERERGRHlDwLyIiIiKSIRT8i4iIiIhkCAX/IiIiIiIZQsG/iIiIiEiGUPAvIiIiIpIhFPyLiIiIiGQIBf8iIiIiIhni/wN9htA1xoQY3QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "agent.train(num_frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test\n",
    "\n",
    "Run the trained agent (1 episode)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Building video /Users/jinwoo.park/Repositories/rainbow-is-all-you-need/videos/rainbow/rl-video-episode-0.mp4.\n",
      "Moviepy - Writing video /Users/jinwoo.park/Repositories/rainbow-is-all-you-need/videos/rainbow/rl-video-episode-0.mp4\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                               "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Moviepy - Done !\n",
      "Moviepy - video ready /Users/jinwoo.park/Repositories/rainbow-is-all-you-need/videos/rainbow/rl-video-episode-0.mp4\n",
      "score:  200.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "video_folder=\"videos/rainbow\"\n",
    "agent.test(video_folder=video_folder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Render"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <video width=\"320\" height=\"240\" alt=\"test\" controls>\n",
       "        <source src=\"data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAAHN5tZGF0AAACsAYF//+s3EXpvebZSLeWLNgg2SPu73gyNjQgLSBjb3JlIDE2MSByMzAzME0gOGJkNmQyOCAtIEguMjY0L01QRUctNCBBVkMgY29kZWMgLSBDb3B5bGVmdCAyMDAzLTIwMjAgLSBodHRwOi8vd3d3LnZpZGVvbGFuLm9yZy94MjY0Lmh0bWwgLSBvcHRpb25zOiBjYWJhYz0xIHJlZj0zIGRlYmxvY2s9MTowOjAgYW5hbHlzZT0weDM6MHgxMTMgbWU9aGV4IHN1Ym1lPTcgcHN5PTEgcHN5X3JkPTEuMDA6MC4wMCBtaXhlZF9yZWY9MSBtZV9yYW5nZT0xNiBjaHJvbWFfbWU9MSB0cmVsbGlzPTEgOHg4ZGN0PTEgY3FtPTAgZGVhZHpvbmU9MjEsMTEgZmFzdF9wc2tpcD0xIGNocm9tYV9xcF9vZmZzZXQ9LTIgdGhyZWFkcz0xMiBsb29rYWhlYWRfdGhyZWFkcz0yIHNsaWNlZF90aHJlYWRzPTAgbnI9MCBkZWNpbWF0ZT0xIGludGVybGFjZWQ9MCBibHVyYXlfY29tcGF0PTAgY29uc3RyYWluZWRfaW50cmE9MCBiZnJhbWVzPTMgYl9weXJhbWlkPTIgYl9hZGFwdD0xIGJfYmlhcz0wIGRpcmVjdD0xIHdlaWdodGI9MSBvcGVuX2dvcD0wIHdlaWdodHA9MiBrZXlpbnQ9MjUwIGtleWludF9taW49MjUgc2NlbmVjdXQ9NDAgaW50cmFfcmVmcmVzaD0wIHJjX2xvb2thaGVhZD00MCByYz1jcmYgbWJ0cmVlPTEgY3JmPTIzLjAgcWNvbXA9MC42MCBxcG1pbj0wIHFwbWF4PTY5IHFwc3RlcD00IGlwX3JhdGlvPTEuNDAgYXE9MToxLjAwAIAAAAG/ZYiEACf//vWxfApqyfOKDOgyLuGXJMmutiLibQDAFQ+wAAADAAATKpYnsyAlGpfAAAAQkANQH0GUHmImKsVAleIOBmriALkwL5pacv8ySayPhF43gb1plLBUL8CkRb1l2I8Dg58iWLxiO+miciASd2yvRbE4vMCvnJZXDcGubLgAnifLvrJzcMb0AvICCzDRe31/j4vhNAXDuz+ZDHOKuA99qAK72UvpyncE1w/9VE4E21ODwBg4bnuwcRZUIi7AvMmVUJ6b9hZ3v2DkvpV4Ak/fSTZZ5vIvSNJpnh4ifPOQsHz0AlfLzCzIuiUGPyxmLlDveEzscBOCbJ8VKVbjDCk81DHgEv+azP/NQPxeuZSwLyhRbceBerRk5bD+uaE90XCzRNOuwIybOMXF31efm17KqUhiunQtffSCGhqe/va8AF1CT3OAEcvC/PHoM/bYe0W2DWwqR+cgZxccZIzjuQP6ljItM/KfGQa0MU2ScDWRZUpoXoxiOfkX4KJgAEztUGjEWYmbytDRaH1R2uoWjQkHkaIqqanjV5Hj3zv53mhgeI1Tm2s466unAc/cDoo8JhlGohjgAAADAAADADAhAAAAT0GaJGxC//6MsAAAChew++jpLv4xVLLXyBkAuCHuTRUxkAbOqI4jVZZ57UCo2RqVHkC/uixkVwpThEskyW7BhYs7OU+XXDWFZL+cG1oxU6EAAAAiQZ5CeIR/AAADAzlzoXHKxzAP+vq28FwAcXs3LTrvHh4/owAAABMBnmF0R/8AAAMAQ1hyDRbQTTGwAAAAHQGeY2pH/wAABR801bGTC8/rwBUNX82ZDkA6O0qBAAAAQEGaaEmoQWiZTAhX//44QAAAJZxDexdZvZTABxGnBgSxbanpicxNClxQjbXb5n7eQUmvdXnQnKJcMUOrRtustX8AAAAhQZ6GRREsI/8AAAMDJTHNo7eWcSO9SznU2illN5A7UX+BAAAAEAGepXRH/wAAAwACOuqUtlEAAAAbAZ6nakf/AAAE+zTV3tuIaoJIG+jw6CRwkZOSAAAAZUGarEmoQWyZTAhP//3xAAADAPJM0gg5eAG5alsel20qwgeYfQCbG6VcN4BoGFvfPpY9EQINKEv6+6hW7dlxQKUXO3MiIveNtHxArgyOMhvrxlSIX2Inzho0c1J9Tjr+UlXjOi/gAAAAHUGeykUVLCP/AAADAzkwebbiuZVjqXVL0PjyanxxAAAADgGe6XRH/wAAAwAAAwGpAAAAGQGe62pH/wAADTSfl8cDTcVrTiSflMtfxQQAAAAuQZrwSahBbJlMCE///fEAAAMAX2UNIAcqtpYf1+qQKpoCmKegIIgrVH9p0SDxgQAAACRBnw5FFSwj/wAAAwM5MHm24rmVlp6knFIDYSn7+FEiAz2U58EAAAAfAZ8tdEf/AAANMtcLTOxBgjKB+DjzBKcRBQCWMnjbcQAAABwBny9qR/8AAAUfMJZ7iS0zXP3swSfB94AXQrZgAAAAPEGbNEmoQWyZTAhn//6eEAAARUUotAHC2wPBrQIOFqiQ+vS/A/8kDOAGoUiwkoc/u6yeeAi2LSpPzpxFlAAAACNBn1JFFSwj/wAAFrxDzbcWF7Xyymcj6KlX55X1jZSCsbTEBQAAAB4Bn3F0R/8AAA0OjhMdkqgVL4kTXY6sdkIjyFN5SoAAAAAcAZ9zakf/AAAjvwQnEslpmysEdURW9HmTypSOSAAAABdBm3hJqEFsmUwIZ//+nhAAAAMAAAMDPwAAABBBn5ZFFSwj/wAAAwAAAwEHAAAADgGftXRH/wAAAwAAAwGpAAAADgGft2pH/wAAAwAAAwGpAAAAF0GbvEmoQWyZTAhn//6eEAAAAwAAAwM+AAAAEEGf2kUVLCP/AAADAAADAQcAAAAOAZ/5dEf/AAADAAADAakAAAAOAZ/7akf/AAADAAADAakAAAAXQZvgSahBbJlMCGf//p4QAAADAAADAz8AAAAeQZ4eRRUsI/8AABazK7UFuRwqSyLhNEd9Kjg2woRsAAAAFwGePXRH/wAAI8MXaKmlU9T7wnbHENmAAAAAFwGeP2pH/wAAI78EJxLJaZq4XbkkUIOBAAAAF0GaJEmoQWyZTAhn//6eEAAAAwAAAwM+AAAAI0GeQkUVLCP/AAADAzdw+fSDNcGcyYNiP7MRsAW10yGA5u25AAAAGQGeYXRH/wAABR/Qwc8auO5s4hIlPix3fgwAAAAdAZ5jakf/AAADAeZ/1Wxfuw3fSfDCP+U2oWi47cEAAAAXQZpoSahBbJlMCGf//p4QAAADAAADAz8AAAAQQZ6GRRUsI/8AAAMAAAMBBwAAAA4BnqV0R/8AAAMAAAMBqQAAAA4BnqdqR/8AAAMAAAMBqQAAACFBmqxJqEFsmUwIZ//+nhAAAAMAAsfVIQQII4U6TCph6rYAAAAQQZ7KRRUsI/8AAAMAAAMBBwAAAA4Bnul0R/8AAAMAAAMBqQAAAA4BnutqR/8AAAMAAAMBqQAAABdBmvBJqEFsmUwIX//+jLAAAAMAAAMDQwAAAB5Bnw5FFSwj/wAAFrMrtQW5HCpLIuE0R30qODbChG0AAAAXAZ8tdEf/AAAjwxdoqaVT1PvCdscQ2YEAAAAXAZ8vakf/AAAjvwQnEslpmrhduSRQg4AAAABUQZs0SahBbJlMCF///oywAAAKCtq9R+ugBxc4OHiR4lQ9IHtR+oPCmtf2K0qzhKrNvY2OBY/Hq7rcoGkRLE85kVHiQYKgDB0a7f1JfKvc1+YbvwHsAAAAHUGfUkUVLCP/AAADATWBSGNfAgwBz0qSnDBN/HgxAAAAGwGfcXRH/wAAAwHlih4h09hDuxrSPKtEVTj+DAAAABABn3NqR/8AAAMAtfaAygRcAAAAGkGbeEmoQWyZTAhf//6MsAAAAwFlBnRvoB/hAAAAK0GflkUVLCP/AAAWsyu1Bbkc50ObZgv914XI/7RDm7aG+C24IwQWQyXOckAAAAAcAZ+1dEf/AAAjwxdoqaVT1PyDVNJhAEKwMHOSTQAAAB0Bn7dqR/8AACO/BCcSyWmauF24luk52QXQqeQdswAAABZBm7xJqEFsmUwIV//+OEAAAAMAAAyoAAAAIUGf2kUVLCP/AAAWwJs/Yf2GQchFaN4zaxATTofmUMPskwAAABwBn/l0R/8AACPDF2ippVPU+8IJjrAYLdbrsEBAAAAAHQGf+2pH/wAAI78EJxLJaZq4XbApiqbaLg+VM/gxAAAAF0Gb4EmoQWyZTAhP//3xAAADAAADAB6RAAAAIUGeHkUVLCP/AAAWwJs/Yf2GQchFaN3CpH6a/Dw+nUUkmAAAABwBnj10R/8AACPDF2ippVPU+8IJjrAYLdbrsEBAAAAAHQGeP2pH/wAAI78EJxLJaZq4XbApiqbaLg+VM/gxAAAANkGaJEmoQWyZTAhn//6eEAAARVH19XC+EEAEZwlx77PlY+/rKHhvwwqMGKy2sSSA6UM+w7Fj5gAAACNBnkJFFSwj/wAAFrUrN2tasvjmI2xPBSxYaxvR6hsNIaD23QAAABwBnmF0R/8AACPDF2ippVPU+8IJjrAYLdbrsEBAAAAAGgGeY2pH/wAAAwC6ZhLPcSWm3umfHpIG8C2zAAAAF0GaaEmoQWyZTAhf//6MsAAAAwAAAwNDAAAAEEGehkUVLCP/AAADAAADAQcAAAAOAZ6ldEf/AAADAAADAakAAAAOAZ6nakf/AAADAAADAakAAABMQZqsSahBbJlMCF///oywAAAJv7haLQEtxyAAcOamLch4P93597zcK2K73XmU4mPkqszIsaE5CzLqyhA8OLRFlwp3OwSH3iJUrVBlgAAAAB9BnspFFSwj/wAAAwMlMc2jt5ZxI71LOdTaKVtHTEBBAAAADgGe6XRH/wAAAwAAAwGpAAAAGwGe62pH/wAABPs01d7biGqCSBvo8OgkcJGTkgAAABZBmvBJqEFsmUwIV//+OEAAAAMAAAypAAAAIEGfDkUVLCP/AAADAyW77JiDfHzOYxoYiK0k4Z38OIgJAAAAGwGfLXRH/wAAAwHbsiYf2amkATxT9v/HgVE5IQAAABsBny9qR/8AAAT7NNXe24hqgkgb6PDoJHCRk5IAAAAXQZs0SahBbJlMCE///fEAAAMAAAMAHpAAAAAgQZ9SRRUsI/8AAAMDJbvsmIN8fM5jGhiIrSThnfw4iAkAAAAbAZ9xdEf/AAADAduyJh/ZqaQBPFP2/8eBUTkgAAAAGwGfc2pH/wAABPs01d7biGqCSBvo8OgkcJGTkgAAAHlBm3hJqEFsmUwIT//98QAAAwBc+UuG3zfJxtCAC58brTMFC1gpsoFUG+XZ9YT8XHTguJN/McahE+t+JiPWfcU4jZm2qvT/m0oF1XjkmnMAWvAZg82LYIpCbcSTI/CPJiu2g04xLV6yG6hg7id9VqRSn6kgQ6DvtMvhAAAAIUGflkUVLCP/AAADAyRT6YuwN+t0ztfGsDARq2MlD3fwYAAAABsBn7V0R/8AAAMB2yYcc0wrNIowRk3XW85LTkkAAAAYAZ+3akf/AAADAeZ+lsnylpmrhWUySBgxAAAAJUGbvEmoQWyZTAhn//6eEAAAAwO0gN9XC96Xmx7fDyyBS44MY+AAAAAZQZ/aRRUsI/8AAAMBNX9c43ml64ooPOUr4QAAACEBn/l0R/8AAAUakmGTym4E4VRvICdlfCZAxg3tqlb7lswAAAAOAZ/7akf/AAADAAADAakAAAAXQZvgSahBbJlMCGf//p4QAAADAAADAz8AAAAQQZ4eRRUsI/8AAAMAAAMBBwAAAA4Bnj10R/8AAAMAAAMBqQAAAA4Bnj9qR/8AAAMAAAMBqQAAABdBmiRJqEFsmUwIZ//+nhAAAAMAAAMDPgAAABBBnkJFFSwj/wAAAwAAAwEHAAAADgGeYXRH/wAAAwAAAwGpAAAADgGeY2pH/wAAAwAAAwGpAAAAF0GaaEmoQWyZTAhn//6eEAAAAwAAAwM/AAAAEEGehkUVLCP/AAADAAADAQcAAAAOAZ6ldEf/AAADAAADAakAAAAOAZ6nakf/AAADAAADAakAAAAXQZqsSahBbJlMCF///oywAAADAAADA0IAAAAQQZ7KRRUsI/8AAAMAAAMBBwAAAA4Bnul0R/8AAAMAAAMBqQAAAA4BnutqR/8AAAMAAAMBqQAAABdBmvBJqEFsmUwIX//+jLAAAAMAAAMDQwAAADFBnw5FFSwj/wAAAwM2g/YGzlKo2nnbIAFraMQtlT7kRiJ/luWqu6LbX9LZyF1X27ZhAAAAGgGfLXRH/wAABR/Qwc62jcNX2BFaCJxmV2zBAAAAHAGfL2pH/wAAAwHmf9VOUbCGGzlaK4C4oo/Z9swAAAAWQZs0SahBbJlMCFf//jhAAAADAAAMqAAAACJBn1JFFSwj/wAAAwM5u+yWFlll0YEM0vhUL/3xZraNL2zBAAAAGgGfcXRH/wAABR/Qwc62jcNX2BFaCJxmV2zAAAAAHAGfc2pH/wAAAwHmf9VOUbBQHgBs9tGxgzz/tmAAAAAXQZt4SahBbJlMCE///fEAAAMAAAMAHpEAAAAgQZ+WRRUsI/8AAAMDObvslhZZZdOlDN+Iwna1u8nPbbgAAAAaAZ+1dEf/AAAFH9DBzraNw1fYEVoInGZXbMEAAAAbAZ+3akf/AAADAeZ/1Ul/E3HV9ncYh4anJjtxAAAAO0GbvEmoQWyZTAhX//44QAABDaKPl1Ub0MwwyVf9Uj9ADctdrzznQNDbDVID/eWTlanVCtlDD6Vx8KQ8AAAAIUGf2kUVLCP/AAAWvEPNtxYX2Q96IRUC8ElkVhRXKcYTkwAAABoBn/l0R/8AAAUf0MHOto3DV9gRWgicZldswAAAAB0Bn/tqR/8AACO/BCcSyWmauFZTHyo7fi+tjQSSYQAAABpBm+BJqEFsmUwIT//98QAAAwANg6EfRGATMQAAABJBnh5FFSwj/wAAAwB0AHIAh4AAAAAQAZ49dEf/AAADALoLfsCrgAAAAA4Bnj9qR/8AAAMAAAMBqQAAAEFBmiRJqEFsmUwIZ//+nhAAAAm3v42SVj/M7Q5QgU9j8Wayeb1WhtGVZyiNqOS5WEOIAaan8TmtQMrwk0JyGZ9BYAAAADhBnkJFFSwj/wAAFrNHtw25OEABqxNbADUtNQpecba8Gh2FBs35E40O/afIHavgFjxd8UhVqjaPBwAAAB8BnmF0R/8AACPDF2ippcMOtYMF5GGKYtIXCtkDoWzAAAAAGAGeY2pH/wAAI78EJxLJaZq4VnMEXInHTQAAABdBmmhJqEFsmUwIZ//+nhAAAAMAAAMDPwAAACFBnoZFFSwj/wAAAwMlfNK7fAcoSK0vHSQX8jKm5aFXJMEAAAAaAZ6ldEf/AAAE+9DCGG1xHpuevXktHmaWEgMAAAAcAZ6nakf/AAADAduAEvjgabitdUKXCTtmVuDkgAAAABdBmqxJqEFsmUwIZ//+nhAAAAMAAAMDPgAAACBBnspFFSwj/wAAAwMlu+yYg3x8zmMaGIitJOGd/DiICQAAABoBnul0R/8AAAT70MIYbXEem569eS0eZpYSAgAAABwBnutqR/8AAAMB24AS+OBpuK11QpcJO2ZW4OSAAAAAF0Ga8EmoQWyZTAhn//6eEAAAAwAAAwM/AAAAIEGfDkUVLCP/AAADAyW77JiDfHzOYxoYiK0k4Z38OIgJAAAAGgGfLXRH/wAABPvQwhhtcR6bnr15LR5mlhIDAAAAHAGfL2pH/wAAAwHbgBL44Gm4rXVClwk7Zlbg5IAAAAAXQZs0SahBbJlMCGf//p4QAAADAAADAz4AAAAQQZ9SRRUsI/8AAAMAAAMBBwAAAA4Bn3F0R/8AAAMAAAMBqQAAAA4Bn3NqR/8AAAMAAAMBqQAAABdBm3hJqEFsmUwIZ//+nhAAAAMAAAMDPwAAABBBn5ZFFSwj/wAAAwAAAwEHAAAADgGftXRH/wAAAwAAAwGpAAAADgGft2pH/wAAAwAAAwGpAAAAF0GbvEmoQWyZTAhn//6eEAAAAwAAAwM+AAAAEEGf2kUVLCP/AAADAAADAQcAAAAOAZ/5dEf/AAADAAADAakAAAAOAZ/7akf/AAADAAADAakAAAAXQZvgSahBbJlMCF///oywAAADAAADA0MAAAAQQZ4eRRUsI/8AAAMAAAMBBwAAAA4Bnj10R/8AAAMAAAMBqQAAAA4Bnj9qR/8AAAMAAAMBqQAAABdBmiRJqEFsmUwIX//+jLAAAAMAAAMDQgAAABBBnkJFFSwj/wAAAwAAAwEHAAAADgGeYXRH/wAAAwAAAwGpAAAADgGeY2pH/wAAAwAAAwGpAAAAIEGaaEmoQWyZTAhf//6MsAAAAwFtxUEAcNx0+50KEl4RAAAAEEGehkUVLCP/AAADAAADAQcAAAAOAZ6ldEf/AAADAAADAakAAAAOAZ6nakf/AAADAAADAakAAABPQZqsSahBbJlMCFf//jhAAAAmr/jFn2RACsUm7+D6raqbedh+7x7t/zbmBDpprodqZhvnQqYrx01tU9bAI+FfuK8ePUm7hdeehtNlpqJJQAAAAB5BnspFFSwj/wAAAwM4Uji0Xjm85Bvts+Ph9V3flt0AAAAaAZ7pdEf/AAAFH9DBzraNw1fYEVoInGZXbMAAAAAQAZ7rakf/AAADALX2gMoEXAAAABlBmvBJqEFsmUwIT//98QAAAwANLQ+s9gIXAAAAOUGfDkUVLCP/AAAWsyu1BbkZ455yFe3OllGj/3c1UgBscUeNGl/kzogOrrooA1wigIvVO3mQkU6sGQAAAB0Bny10R/8AACPDF2ippVPBIBAH4+bR/MPEKWBCAwAAAB0Bny9qR/8AACO/BCcSyWmauFZTHyo7fi+tjQSSYAAAADJBmzRJqEFsmUwIZ//+nhAAAEVR9gTQCN/VFWLqNYZ0oj49AV2gEayktGxsBi2xtrQ0IAAAACFBn1JFFSwj/wAAFrUrN2tasvjmI2xQqPCkMWC/CpBz228AAAAeAZ9xdEf/AAAjwxdoqaVTwSALcv7aeV/3jdChng5IAAAAGwGfc2pH/wAAAwC15rl8cDTdEGzOMBFVLpbbgAAAABdBm3hJqEFsmUwIX//+jLAAAAMAAAMDQwAAACFBn5ZFFSwj/wAAAwMjrnllo765NLBKkadG5/J+iP6aNbMAAAAaAZ+1dEf/AAAE+9DCGG1xHpuevXktHmaWEgMAAAAbAZ+3akf/AAADAdt/1Xe24hqhRFvovP42CetnAAAAF0GbvEmoQWyZTAhf//6MsAAAAwAAAwNCAAAAIEGf2kUVLCP/AAADAyOulDfvrW1KitZJHndwj5KWtpEBAAAAGgGf+XRH/wAABPvQwhhtcR6bnr15LR5mlhICAAAAGwGf+2pH/wAAAwHbf9V3tuIaoURb6Lz+NgnrZwAAABdBm+BJqEFsmUwIX//+jLAAAAMAAAMDQwAAADtBnh5FFSwj/wAAAwM2aGt4WtCkPQKiIgCI7jWPT6wV6WANPWGDin+029OaZZMzIEGj2R7FF/qtnx1QQAAAABYBnj10R/8AAAUf0MHOto3DV288KiYOAAAAIAGeP2pH/wAAAwHmf9VOUbBQHgCA7oA2m54DgjpOJNtvAAAAFkGaJEmoQWyZTAhX//44QAAAAwAADKgAAAAQQZ5CRRUsI/8AAAMAAAMBBwAAAA4BnmF0R/8AAAMAAAMBqQAAAA4BnmNqR/8AAAMAAAMBqQAAABdBmmhJqEFsmUwIR//94QAAAwAAAwAxYQAAABBBnoZFFSwj/wAAAwAAAwEHAAAADgGepXRH/wAAAwAAAwGpAAAADgGep2pH/wAAAwAAAwGpAAAAFkGaqUmoQWyZTAj//IQAAAMAAAMAwIAAAAyjbW9vdgAAAGxtdmhkAAAAAAAAAAAAAAAAAAAD6AAAD8gAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAC810cmFrAAAAXHRraGQAAAADAAAAAAAAAAAAAAABAAAAAAAAD8gAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAABAAAAAAlgAAAGQAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAA/IAAACAAABAAAAAAtFbWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAyAAAAygBVxAAAAAAALWhkbHIAAAAAAAAAAHZpZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAAK8G1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAAACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAACrBzdGJsAAAAsHN0c2QAAAAAAAAAAQAAAKBhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAlgBkABIAAAASAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAANmF2Y0MBZAAf/+EAGWdkAB+s2UCYM+XhAAADAAEAAAMAZA8YMZYBAAZo6+PLIsD9+PgAAAAAFGJ0cnQAAAAAAAA5GQAAORkAAAAYc3R0cwAAAAAAAAABAAAAygAAAQAAAAAUc3RzcwAAAAAAAAABAAAAAQAABmBjdHRzAAAAAAAAAMoAAAABAAACAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAAFAAAAAAEAAAIAAAAAAQAAAAAAAAABAAABAAAAAAEAAAUAAAAAAQAAAgAAAAABAAAAAAAAAAEAAAEAAAAAAQAABQAAAAABAAACAAAAAAEAAAAAAAAAAQAAAQAAAAABAAACAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAygAAAAEAAAM8c3RzegAAAAAAAAAAAAAAygAABHcAAABTAAAAJgAAABcAAAAhAAAARAAAACUAAAAUAAAAHwAAAGkAAAAhAAAAEgAAAB0AAAAyAAAAKAAAACMAAAAgAAAAQAAAACcAAAAiAAAAIAAAABsAAAAUAAAAEgAAABIAAAAbAAAAFAAAABIAAAASAAAAGwAAACIAAAAbAAAAGwAAABsAAAAnAAAAHQAAACEAAAAbAAAAFAAAABIAAAASAAAAJQAAABQAAAASAAAAEgAAABsAAAAiAAAAGwAAABsAAABYAAAAIQAAAB8AAAAUAAAAHgAAAC8AAAAgAAAAIQAAABoAAAAlAAAAIAAAACEAAAAbAAAAJQAAACAAAAAhAAAAOgAAACcAAAAgAAAAHgAAABsAAAAUAAAAEgAAABIAAABQAAAAIwAAABIAAAAfAAAAGgAAACQAAAAfAAAAHwAAABsAAAAkAAAAHwAAAB8AAAB9AAAAJQAAAB8AAAAcAAAAKQAAAB0AAAAlAAAAEgAAABsAAAAUAAAAEgAAABIAAAAbAAAAFAAAABIAAAASAAAAGwAAABQAAAASAAAAEgAAABsAAAAUAAAAEgAAABIAAAAbAAAANQAAAB4AAAAgAAAAGgAAACYAAAAeAAAAIAAAABsAAAAkAAAAHgAAAB8AAAA/AAAAJQAAAB4AAAAhAAAAHgAAABYAAAAUAAAAEgAAAEUAAAA8AAAAIwAAABwAAAAbAAAAJQAAAB4AAAAgAAAAGwAAACQAAAAeAAAAIAAAABsAAAAkAAAAHgAAACAAAAAbAAAAFAAAABIAAAASAAAAGwAAABQAAAASAAAAEgAAABsAAAAUAAAAEgAAABIAAAAbAAAAFAAAABIAAAASAAAAGwAAABQAAAASAAAAEgAAACQAAAAUAAAAEgAAABIAAABTAAAAIgAAAB4AAAAUAAAAHQAAAD0AAAAhAAAAIQAAADYAAAAlAAAAIgAAAB8AAAAbAAAAJQAAAB4AAAAfAAAAGwAAACQAAAAeAAAAHwAAABsAAAA/AAAAGgAAACQAAAAaAAAAFAAAABIAAAASAAAAGwAAABQAAAASAAAAEgAAABoAAAAUc3RjbwAAAAAAAAABAAAAMAAAAGJ1ZHRhAAAAWm1ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAG1kaXJhcHBsAAAAAAAAAAAAAAAALWlsc3QAAAAlqXRvbwAAAB1kYXRhAAAAAQAAAABMYXZmNTguNzYuMTAw\" type=\"video/mp4\"/>\n",
       "        </video>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Played: videos/rainbow/rl-video-episode-0.mp4\n"
     ]
    }
   ],
   "source": [
    "import base64\n",
    "import glob\n",
    "import io\n",
    "import os\n",
    "\n",
    "from IPython.display import HTML, display\n",
    "\n",
    "\n",
    "def ipython_show_video(path: str) -> None:\n",
    "    \"\"\"Show a video at `path` within IPython Notebook.\"\"\"\n",
    "    if not os.path.isfile(path):\n",
    "        raise NameError(\"Cannot access: {}\".format(path))\n",
    "\n",
    "    video = io.open(path, \"r+b\").read()\n",
    "    encoded = base64.b64encode(video)\n",
    "\n",
    "    display(HTML(\n",
    "        data=\"\"\"\n",
    "        <video width=\"320\" height=\"240\" alt=\"test\" controls>\n",
    "        <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\"/>\n",
    "        </video>\n",
    "        \"\"\".format(encoded.decode(\"ascii\"))\n",
    "    ))\n",
    "\n",
    "\n",
    "def show_latest_video(video_folder: str) -> str:\n",
    "    \"\"\"Show the most recently recorded video from video folder.\"\"\"\n",
    "    list_of_files = glob.glob(os.path.join(video_folder, \"*.mp4\"))\n",
    "    latest_file = max(list_of_files, key=os.path.getctime)\n",
    "    ipython_show_video(latest_file)\n",
    "    return latest_file\n",
    "\n",
    "\n",
    "latest_file = show_latest_video(video_folder=video_folder)\n",
    "print(\"Played:\", latest_file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rainbow-is-all-you-need",
   "language": "python",
   "name": "rainbow-is-all-you-need"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}