microsoft/AI-For-Beginners

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import matplotlib.pyplot as plt\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"CartPole-v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<>:74: DeprecationWarning: invalid escape sequence \\g\n",
      "<>:93: DeprecationWarning: invalid escape sequence \\d\n"
     ]
    }
   ],
   "source": [
    "class REINFORCE:\n",
    "  def __init__(self, env, path=None):\n",
    "    self.env=env #import env\n",
    "    self.state_shape=env.observation_space.shape # the state space\n",
    "    self.action_shape=env.action_space.n # the action space\n",
    "    self.gamma=0.99 # decay rate of past observations\n",
    "    self.alpha=1e-4 # learning rate in the policy gradient\n",
    "    self.learning_rate=0.01 # learning rate in deep learning\n",
    "    \n",
    "    if not path:\n",
    "      self.model=self._create_model() #build model\n",
    "    else:\n",
    "      self.model=self.load_model(path) #import model\n",
    "\n",
    "    # record observations\n",
    "    self.states=[]\n",
    "    self.gradients=[] \n",
    "    self.rewards=[]\n",
    "    self.probs=[]\n",
    "    self.discounted_rewards=[]\n",
    "    self.total_rewards=[]\n",
    "\n",
    "  def hot_encode_action(self, action):\n",
    "    '''encoding the actions into a binary list'''\n",
    "\n",
    "    action_encoded=np.zeros(self.action_shape, np.float32)\n",
    "    action_encoded[action]=1\n",
    "\n",
    "    return action_encoded\n",
    "  \n",
    "  def remember(self, state, action, action_prob, reward):\n",
    "    '''stores observations'''\n",
    "    encoded_action=self.hot_encode_action(action)\n",
    "    self.gradients.append(encoded_action-action_prob)\n",
    "    self.states.append(state)\n",
    "    self.rewards.append(reward)\n",
    "    self.probs.append(action_prob)\n",
    "\n",
    "  def _create_model(self):\n",
    "    ''' builds the model using keras'''\n",
    "    model=keras.Sequential()\n",
    "\n",
    "    # input shape is of observations\n",
    "    model.add(keras.layers.Dense(24, input_shape=self.state_shape, activation=\"relu\"))\n",
    "    # add a relu layer \n",
    "    model.add(keras.layers.Dense(12, activation=\"relu\"))\n",
    "\n",
    "    # output shape is according to the number of action\n",
    "    # The softmax function outputs a probability distribution over the actions\n",
    "    model.add(keras.layers.Dense(self.action_shape, activation=\"softmax\")) \n",
    "    model.compile(loss=\"categorical_crossentropy\",\n",
    "            optimizer=keras.optimizers.Adam(lr=self.learning_rate))\n",
    "        \n",
    "    return model\n",
    "\n",
    "  def get_action(self, state):\n",
    "    '''samples the next action based on the policy probabilty distribution \n",
    "      of the actions'''\n",
    "\n",
    "    # transform state\n",
    "    state=state.reshape([1, state.shape[0]])\n",
    "    # get action probably\n",
    "    action_probability_distribution=self.model.predict(state).flatten()\n",
    "    # norm action probability distribution\n",
    "    action_probability_distribution/=np.sum(action_probability_distribution)\n",
    "    \n",
    "    # sample action\n",
    "    action=np.random.choice(self.action_shape,1,\n",
    "                            p=action_probability_distribution)[0]\n",
    "\n",
    "    return action, action_probability_distribution\n",
    "\n",
    "  def get_discounted_rewards(self, rewards): \n",
    "    '''Use gamma to calculate the total reward discounting for rewards\n",
    "    Following - \\gamma ^ t * Gt'''\n",
    "    \n",
    "    discounted_rewards=[]\n",
    "    cumulative_total_return=0\n",
    "    # iterate the rewards backwards and and calc the total return \n",
    "    for reward in rewards[::-1]:      \n",
    "      cumulative_total_return=(cumulative_total_return*self.gamma)+reward\n",
    "      discounted_rewards.insert(0, cumulative_total_return)\n",
    "\n",
    "    # normalize discounted rewards\n",
    "    mean_rewards=np.mean(discounted_rewards)\n",
    "    std_rewards=np.std(discounted_rewards)\n",
    "    norm_discounted_rewards=(discounted_rewards-\n",
    "                          mean_rewards)/(std_rewards+1e-7) # avoiding zero div\n",
    "    \n",
    "    return norm_discounted_rewards\n",
    "\n",
    "  def update_policy(self):\n",
    "    '''Updates the policy network using the NN model.\n",
    "    This function is used after the MC sampling is done - following\n",
    "    \\delta \\theta = \\alpha * gradient + log pi'''\n",
    "      \n",
    "    # get X\n",
    "    states=np.vstack(self.states)\n",
    "\n",
    "    # get Y\n",
    "    gradients=np.vstack(self.gradients)\n",
    "    rewards=np.vstack(self.rewards)\n",
    "    discounted_rewards=self.get_discounted_rewards(rewards)\n",
    "    gradients*=discounted_rewards\n",
    "    gradients=self.alpha*np.vstack([gradients])+self.probs\n",
    "\n",
    "    history=self.model.train_on_batch(states, gradients)\n",
    "    \n",
    "    self.states, self.probs, self.gradients, self.rewards=[], [], [], []\n",
    "\n",
    "    return history\n",
    "\n",
    "  def train(self, episodes, rollout_n=1, render_n=50):\n",
    "    '''train the model\n",
    "        episodes - number of training iterations \n",
    "        rollout_n- number of episodes between policy update\n",
    "        render_n - number of episodes between env rendering ''' \n",
    "    \n",
    "    env=self.env\n",
    "    total_rewards=np.zeros(episodes)\n",
    "\n",
    "    for episode in range(episodes):\n",
    "      # each episode is a new game env\n",
    "      state=env.reset()\n",
    "      done=False          \n",
    "      episode_reward=0 #record episode reward\n",
    "      \n",
    "      while not done:\n",
    "        # play an action and record the game state & reward per episode\n",
    "        action, prob=self.get_action(state)\n",
    "        next_state, reward, done, _=env.step(action)\n",
    "        self.remember(state, action, prob, reward)\n",
    "        state=next_state\n",
    "        episode_reward+=reward\n",
    "\n",
    "        #if episode%render_n==0: ## render env to visualize.\n",
    "          #env.render()\n",
    "        if done:\n",
    "          # update policy \n",
    "          if episode%rollout_n==0:\n",
    "            history=self.update_policy()\n",
    "\n",
    "      total_rewards[episode]=episode_reward\n",
    "      if episode%10==0:\n",
    "        print(f\"{episode} -> {episode_reward}\")\n",
    "      \n",
    "    self.total_rewards=total_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\winapp\\Miniconda3\\envs\\py38\\lib\\site-packages\\keras\\optimizer_v2\\adam.py:105: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -> 39.0\n",
      "10 -> 50.0\n",
      "20 -> 18.0\n",
      "30 -> 15.0\n",
      "40 -> 16.0\n",
      "50 -> 17.0\n",
      "60 -> 20.0\n",
      "70 -> 11.0\n",
      "80 -> 20.0\n",
      "90 -> 40.0\n",
      "100 -> 36.0\n",
      "110 -> 18.0\n",
      "120 -> 79.0\n",
      "130 -> 43.0\n",
      "140 -> 41.0\n",
      "150 -> 37.0\n",
      "160 -> 95.0\n",
      "170 -> 72.0\n",
      "180 -> 138.0\n",
      "190 -> 102.0\n"
     ]
    }
   ],
   "source": [
    "r = REINFORCE(env)\n",
    "\n",
    "r.train(200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1f7a517f550>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABEU0lEQVR4nO29eZhlVXX3/1l3rLo1z13dVd3VM9000EAzi0FmiRHURDEGyfsaia+SmBijqDHRJMRMDkkcIkZ/EgcQFAQnEFtAEOimu2l6nqeq7prn8Y7798cZ6tyqW9U1T70+z1NP3bvvOffse+rW96zz3WuvLcYYFEVRlIWFb7Y7oCiKokw9Ku6KoigLEBV3RVGUBYiKu6IoygJExV1RFGUBEpjtDgCUlpaampqa2e6GoijKvGL79u0txpiyTK/NCXGvqalh27Zts90NRVGUeYWInBzptTHbMiLiF5HXROSn9vNiEXlGRA7bv4s8235CRI6IyEERuWVy3VcURVHGy3g89w8D+z3P7wM2G2NWA5vt54jIeuBO4HzgVuCrIuKfmu4qiqIoY2FM4i4iVcDvAv/jab4deNB+/CBwh6f9YWNM1BhzHDgCXD4lvVUURVHGxFgj9y8BHwNSnrYKY0w9gP273G5fAtR6tquz29IQkXtEZJuIbGtubh5vvxVFUZRROKu4i8hbgCZjzPYxvqdkaBtWwMYY84AxZpMxZlNZWcbBXkVRFGWCjCVb5hrgrSJyG5AF5IvId4FGEak0xtSLSCXQZG9fB1R79q8CzkxlpxVFUZTROWvkboz5hDGmyhhTgzVQ+mtjzB8BTwJ325vdDTxhP34SuFNEwiKyHFgNbJ3yniuKoigjMpkZqv8M3CQih4Gb7OcYY/YCjwD7gKeADxljkpPtqKIoykKhayDOj7bXMZ0l18c1ickY8xzwnP24FbhhhO3uB+6fZN8URVEWJL/c28hHH32dmtIcLl1WdPYdJoDWllEURZlh+mIJAH57pGXajqHiriiKMsMMxC2n+sXDKu6KoigLhoG4NWVox6l2eqOJaTmGiruiKMoM40TuiZRhy/HWaTmGiruiKMoMMxBPkRX0EQ74eGGarJk5UfJXURTlXGIgkSQ3HOTm8ytYlJ81LcdQcVcURZlhBuJJsoI+/ultF0zbMdSWURRFmWGi8RRZwemthK7iriiKMsM4kft0ouKuKIoywwwkkmQFNHJXFEWZ9zR1D/D9LacAJ1tGxV1RFGXe87Nd9Xzy8d209cbUllEURVkoxJPWrNSegQQD8SRhjdwVRVHmP/GkVd63N5awbBn13BVFUeY/yZQl7n2xBNGE2jKKoigLgoQt7j3RpA6oKoqiLBSSKctz74sm5saAqohkichWEXldRPaKyGft9s+IyGkR2Wn/3ObZ5xMickREDorILdP5ARRFUeYDTuTe2R8nkTLT7rmPpbZMFLjeGNMjIkHgRRH5hf3aF40x/+7dWETWYy2kfT6wGPiViKzRdVQVRTmXSdgDqq29MYDZt2WMRY/9NGj/jLaq6+3Aw8aYqDHmOHAEuHzSPVUURZnHOAOqba64z4EBVRHxi8hOoAl4xhizxX7pXhHZJSLfEhFnldclQK1n9zq7beh73iMi20RkW3Nz88Q/gaIoyjwgYXvurT1RgLmR526MSRpjNgJVwOUisgH4GrAS2AjUA5+3N5dMb5HhPR8wxmwyxmwqKyubQNcVRVHmD07k7tgy4cAciNwdjDEdwHPArcaYRlv0U8A3GLRe6oBqz25VwJnJd1VRFGX+4kxiapsrnruIlIlIof04G7gROCAilZ7N3gbssR8/CdwpImERWQ6sBrZOaa8VRVHmGW7k3jMz4j6WbJlK4EER8WNdDB4xxvxURL4jIhuxLJcTwJ8CGGP2isgjwD4gAXxIM2UURTnXSQwdUJ1mW+as4m6M2QVcnKH9rlH2uR+4f3JdUxRFWTg4k5hidgGxWbdlFEVRlMnjeO4OKu6KoigLAMdzd5gTee6KoijK5EgME3eN3BVFUeY9jufuoPXcFUVRFgBDPfew2jKKoijzH6/nLjLHZqgqiqIoE8PruYcDPkQyVWqZOlTcFUVRZgCv5z7dg6mg4q4oijIjJJIGnx2sT/dgKqi4K4qizAiJlCE/OwhMf447qLgriqLMCMmUocAVd43cFUVRFgSJVMoV9+leqANU3BVFUWaERNKQn2VH7tOcBgkq7oqiKDNCQm0ZRVGUhUcyZcjPtqqs64CqoijKAiGRTJEV9BPy+zRyVxRFWSgkUoaAT8jNCpA9F8RdRLJEZKuIvC4ie0Xks3Z7sYg8IyKH7d9Fnn0+ISJHROSgiNwynR9AURRlPpBIGfw+H//0tg388TU10368sUTuUeB6Y8xFwEbgVhG5ErgP2GyMWQ1stp8jIuuBO4HzgVuBr9rrryqKopyzJFOGoF+4dUMl5y3Kn/bjnVXcjUWP/TRo/xjgduBBu/1B4A778e3Aw8aYqDHmOHAEuHwqO60oijKfMMaQTBn8vuktFuZlTJ67iPhFZCfQBDxjjNkCVBhj6gHs3+X25kuAWs/udXbb0Pe8R0S2ici25ubmSXwERVGUuY1T7jcw18TdGJM0xmwEqoDLRWTDKJtn6r0Z1mDMA8aYTcaYTWVlZWPqrKIoynzEKffr981cDsu4jmSM6QCew/LSG0WkEsD+3WRvVgdUe3arAs5MtqOKoijzlcRcjNxFpExECu3H2cCNwAHgSeBue7O7gSfsx08Cd4pIWESWA6uBrVPcb0VRlHlD0l5iL+CfOXEPjGGbSuBBO+PFBzxijPmpiLwMPCIi7wNOAX8AYIzZKyKPAPuABPAhY0xyerqvKIoy90nYC3XMZOR+VnE3xuwCLs7Q3grcMMI+9wP3T7p3iqIoC4A577kriqIo42dOeu6KoijK5JgNz13FXVEUZZpxPPc5N4lJURRFmTiDtox67oqiKAuGRNIZUNXIXVEUZcHglB8IqueuKIqycFDPXVEUZQGinruiKMoCRD13RVGUBYhb8lc9d0VRlIXDbNSWUXFXFEWZZhxbRj13RVGUBcRg4TCN3BVFURYM6rkriqIsQNRzVxRFWYCo564oirIAcWwZ/1yyZUSkWkSeFZH9IrJXRD5st39GRE6LyE775zbPPp8QkSMiclBEbpnOD6AoijLXmY3FOsayhmoC+CtjzA4RyQO2i8gz9mtfNMb8u3djEVkP3AmcDywGfiUia3QdVUVRzlWSc9FzN8bUG2N22I+7gf3AklF2uR142BgTNcYcB44Al09FZxVFUeYj8bnuuYtIDdZi2VvspntFZJeIfEtEiuy2JUCtZ7c6MlwMROQeEdkmItuam5vH33NFUZR5wpz03B1EJBf4EfAXxpgu4GvASmAjUA983tk0w+5mWIMxDxhjNhljNpWVlY2334qiKPOGObtAtogEsYT9e8aYxwCMMY3GmKQxJgV8g0HrpQ6o9uxeBZyZui4riqLML5JzsZ67iAjwTWC/MeYLnvZKz2ZvA/bYj58E7hSRsIgsB1YDW6euy4qiKPOLuZotcw1wF7BbRHbabZ8E3i0iG7EslxPAnwIYY/aKyCPAPqxMmw9ppoyiKOcyiaTB7xOsWHlmOKu4G2NeJLOP/vNR9rkfuH8S/VIURVkwJFJmRi0Z0BmqiqIo004ylZpRSwZU3BVFUaadRMqouCuKoiw0EklDwD+zcqviriiKMs2o564oirIAUc9dURRlAZJImRldhQlU3BVFUaadRNLMaNEwUHFXFEWZdpLquSuKoiw8Euq5K4qiLDyS6rkriqIsPOJJg189d0VRlIVFUmeoKoqiLDwSqZQOqCqKoiw0NHJXFEVZgMS1toyiKMrCQyN3RVGUBcicLBwmItUi8qyI7BeRvSLyYbu9WESeEZHD9u8izz6fEJEjInJQRG6Zzg+gKIoy15mrhcMSwF8ZY9YBVwIfEpH1wH3AZmPMamCz/Rz7tTuB84Fbga+KiH86Oq8oijIfmJP13I0x9caYHfbjbmA/sAS4HXjQ3uxB4A778e3Aw8aYqDHmOHAEuHyK+60oijJvmPMrMYlIDXAxsAWoMMbUg3UBAMrtzZYAtZ7d6uy2oe91j4hsE5Ftzc3NE+i6oijK/GBOFw4TkVzgR8BfGGO6Rts0Q5sZ1mDMA8aYTcaYTWVlZWPthqIoyrxjzhYOE5EglrB/zxjzmN3cKCKV9uuVQJPdXgdUe3avAs5MTXcVRVHmH3OycJiICPBNYL8x5guel54E7rYf3w084Wm/U0TCIrIcWA1snbouK4qizC/is7BYR2AM21wD3AXsFpGddtsngX8GHhGR9wGngD8AMMbsFZFHgH1YmTYfMsYkp7rjiqIo84XZ8NzPKu7GmBfJ7KMD3DDCPvcD90+iX4qiKPOeeDLFf/36CD3RBIXZwRk9ts5QVRRFmSYefOkE/7n5MG+/ZAl/fE3NjB57LLaMoiiKMgF21nZQVZTNF965ccaPrZG7oijKNHGkqYc1FXmzcmwVd0VRlGkgkUxxrLmX1eW5s3J8FXdFUZRp4FRbH7FkilUq7oqiKAuHw009AKxWW0ZRFGXhcMQWd43cFUWZFFuPt3HnAy8TT6ZmuysKcLixm8UFWeSGZycpUcVdURYIO2vbeeVYG90DidnuioJly6yaJUsGVNwVZcEQTxr7t0bus00qZTja3MOqstmxZEDFXVEWDI6oq7jPPl0DcQbiKZYUZc9aH1TcFWWB4Ih6Ijls+YRzmmTK8NePvs7++tGWoZha2vviABRFZraejBcVd0VZICTUlslIa2+UR7fX8b0tJ2fsmO19MQAKVdwVRZksMdeW0cjdi3M+XjzcMmPH7HDFPTRjxxyKiruiLBBcWyalkbuXWMI6Hyda+6ht65uRY3a4toyKu6IokySecGwZjdy9eG2q3x6ZmehdPXdFUaaMeEqzZTLhRO4AL0yjuBtjePClEzR1D9DRF8MnkJ81h8VdRL4lIk0issfT9hkROS0iO+2f2zyvfUJEjojIQRG5Zbo6rihKOk7Ertky6ThjEaW5YV4+2oox03N+mruj/N2Te/nR9tO098UoyA7im+Gl9byMJXL/NnBrhvYvGmM22j8/BxCR9cCdwPn2Pl8VEf9UdVZRlJGJ2xFqXD33NJzzcllNEW29Meo7B6blOF32zOD6zn7a++Kz6rfDGMTdGPMboG2M73c78LAxJmqMOQ4cAS6fRP8URRkjzkBqPKHi7sWJ3C+sKgSYtnz3nqgl7mc6LFumYBb9dpic536viOyybZsiu20JUOvZps5uG4aI3CMi20RkW3Nz8yS6oSgKQMyxZVJqy3iJu+JeAEytuHcPxLntP15gz+lOugesQdT6zn465kPkPgJfA1YCG4F64PN2eyaDKeM3zRjzgDFmkzFmU1lZ2QS7oSiKg2vL6IBqGjE7i6goEmJpcYT99d1pr790pIXf/9pL9MXGX3DtZGsf++q72FnbQY9rywzQ0Ref1QlMMEFxN8Y0GmOSxpgU8A0GrZc6oNqzaRVwZnJdVBRlLGj5gcw4tkwoIKyrzBsWuT+zv5FtJ9t5em+D23akqYfGrrN7885M1M7+ON22LdPWG6O5Ozo/I3cRqfQ8fRvgZNI8CdwpImERWQ6sBrZOrouKooyFeErLD2TCuaMJ+n2sq8zneGtvWpR+qNGK5B/bcRqwrJZ3fO0l/uGn+8763k4+e9dA3I3cwbqgzGaOO8BZq8iLyEPAdUCpiNQBfwdcJyIbsSyXE8CfAhhj9orII8A+IAF8yBiTnJaeK4qSxmC2zPyP3Ova+yjJCZMdmnyyXdyN3H2sr8zHGDjY0M3FS62hwoMNPQR8wotHWqjv7OexHafp7I9zrLn3rO/tlBno6k+QE0q3dWaz9ACMQdyNMe/O0PzNUba/H7h/Mp1SFGX8DNoy8z9yf+uXf8v/ubqGP7th9aTfy7FlnMgdYF99FxcvLaK1J0pLT5S7rlzGd145yad/vJftJ63kwNq2PowxiIycq97ea0fu/XFyw+kXonlpyyiKMvdYKJ77QDxJW2+Mpu7olLxfzGPLVBVlU5wTYsfJDgAO2pbMzedX8Ke/s4IXjzTT3hfn9y5aTHc04douDj/cXsdXnj3iToTq6Lcj94E4PdEE+VmD8fKct2UURZkfODNUY/M8cneKbvXFpsbRdc5LOOBDRLi8ppgtx1sBy54BWLsoj2tXl/Hn16+mvrOfEy19/OT1M5xq66M4ZzAC/9+XT7CrrpOOvhifvG2d29fO/jgF2UFK88IE/T5ae+d3nruiKHOIhRK5d/Zbgtkfn5q1YL2RO8AVK4qpa+/ndEc/hxq7KYoEKcsNA5ATDrCqPI+lJREATg2pInm6vZ+8cIBvvHCcl4+1utkyXf1W5J4XDlBZmAWoLaMoyhSxUEr+OuI+dZF7Cp+A367zcsXyEgC2HGvlYEM3ayryhvnq1UWWuHtLBPfHkrT2xnj7Jda8zKNNPa5t09lvZcvkZgWoLLCW1lNxVxRlSkgsEFtmOsQ9FBiUuvMW5VGQHeQHr9ay90yXO8jqJTvkpzwvzMnWwYyZ0x39gFXGIOT3cdouMwBWXZnugQS54QBLiyPkhQNTkukzGVTcFWWBEFtotswUiXs0kXItGQCfT7isppgtx9soyQnxJ9cuz7jf0uJImi1zxhb3qqJsKguzON3RT3uvJe7JlKGha4C8rCD/77qVfOdPrpiSvk8GFXdFWSAslFTIwch9ajz3eDJFyJ8udb9/6RI2LMnn+++/kirbghnK0uIItW397nMncl9SlM3igmxOtfXRNZBgUX6W2+/ccIDS3DAbqwunpO+TQcVdURYAyZTBmbs03ycxddpWx1RF7kNtGYBbN1Ty0z+7lprSnBH3W1oS4UxnP9GE1Y/T7f34fcKi/CwWF2ZzyM60cQZfAfKy5k4Cooq7oiwAvCUH5nvJXydy750icY8NsWXGytLiCMZYog5W5L4oP4uA38eSwiz641b/lhUPintuWMVdUZQpxCvu873k71R77vGkGRa5jwXHbnEmU53u6GdJoZUJs9j+DbDME7nnauSuKMpU4l0Ue74XDnPEPZZMTcn4QSw5sci9xM59b+2xbKLT7f0stnPYlxR5xX3Q2smbxTVTh6LirigLAK8ILpRsGYC++OSj91giRcg//rVMS3KtPPXW3iiJZIqGrgFX1EeK3PPUllEUZSrx5rYvlMgdpsaayTSgOhaKIiF8Ai09MRq7oyRThiWFlpAvLhgU9+oitWUURZkm0myZBeC5ZwetCUBTMZEpPkFbxu8TinNCtPRE3UFVJ3LPDvkpzgnh9wmFkaA7kKoDqoqiTClpA6rzOHI3xtDZH3frs0wk1/1X+xr53paT7vOJZssAlOSEae2JuhOYltj9AlhcmEVhdhARoSDb8to1FVJRlCklvkBsmf54knjSUFlgiehEbJnvbTnJ158/5j6PTTBbBizfvbUn5k5g8nrtK8ty3eeOqOeF586A6ty5zCiKMmHSs2Xmry3j+O2L8i3RnIgt09Ybc2u+QOYZqmOlJDfMntOd1LX3U5wTIhIalMzP/N75DNgTnJzIPSc8u/VkvJz1E4vIt0SkSUT2eNqKReQZETls/y7yvPYJETkiIgdF5Jbp6riiKIN4l5Kbz1UhHXF3IvcJiXtfjK6BBEl77MGyZcafLQNQkhOipTualuPuUJQTcitA5mcHyQ76CUzwIjIdjKUn3wZuHdJ2H7DZGLMa2Gw/R0TWA3cC59v7fFVE5s6lTFEWKI64R0L+eZ0K2WmX0F1UMHHPva1nsMY6TDxbBqAsL0x3NMHxlp5h4u6lNDectqjHXOCsn9gY8xugbUjz7cCD9uMHgTs87Q8bY6LGmOPAEeDyqemqoigj4VgxkaB/Xpf87Zhk5D4QT7plC5z3mtyAqiXYtW39aX77UD58w2q+ftelEzrGdDHRe4gKY0w9gP273G5fAtR6tquz24YhIveIyDYR2dbc3DzBbswN6tr7+M2h+f0ZlPmNU08me75H7v3pkft4B1TbPV6747tPdIYqDM5ShfRZqUNZVJDFhiUFEzrGdDHVBlEmYyvjN80Y84AxZpMxZlNZWdkUd2Nm+daLJ3j//24jNc/zi5X5y6AtE5hwKuRTexoYmIIZoZPBsVKcSULjjdzbej3i7rFlwpPIlnEYzZaZi0xU3BtFpBLA/t1kt9cB1Z7tqoAzE+/e/KA3miCasKYnK8ps4Excyg76JzSJ6VRrHx/47nZ+tqt+qrs2Lhq7BggFfBRkBwkFfPSNcx3V9t7B2a2Ofz8ZW6Y0ZzByrxolcp+LTFTcnwTuth/fDTzhab9TRMIishxYDWydXBfnPk7pz6GL6SrKTOHYMpGwf0J57l0DlhDOdoCyv76bNRW5+HxCJOQ/qy1T39nP2776Wxrtfrf2Rt3XOvpibp37CYt73gKO3EXkIeBlYK2I1InI+4B/Bm4SkcPATfZzjDF7gUeAfcBTwIeMMbN7nzcDqLgrs81Ys2WMMWw/2Y4x6ds49kdzdzStPZUyfPB723n5aOsU9zhz3/bXd7HeXtM0EvSf1ZbZeryN1051sKuuE8Bd9g4sW8abIjoRIqEA2UE/2UE/hZG5M0FpLJx1EpMx5t0jvHTDCNvfD9w/mU7NNxyf8lSrirsyOzgilh0MjBq5v3KsjXd/4xV++IGr2FRT7LY7AcpQce8aiPPz3Q1UF0W4amXJNPR8kObuKK29MXfB6uwxRO51ds2Xlh6r3229MUSsC0NHX5yofUcz0Tx3sHz3rKAfkYm/x2wwdzLu5zFOdKGRuzJbOKmQ2SHfqOJ+orUXgOMtvWnt/XY+eVN3ui3TYfvWTUNEfzrYV98F4Ip7JBQYlufe1DXgWkgAtfb/XIvdv7a+GIXZQYpzQ3ROQeQOsKIsl/MW5U14/9lCyw9MAf0q7sos44hYTihAylh2is83PNJ0CmCd6UgX8ZFsGSe1cKjoTwf76601SdctGozch9oydz7wChurC/nCuzYCUNtui7sncnfKBHT0xQbFfRIzR7/2nkuYZ0E7oOI+JQyo567MMq4tE7ImhMdTKcK+4ZPDT7vi3p/WPpIt46QTNnbNTOS+pDCbAtvbjoT8aamNde19HGvpTRNa15axt3PEPSvop6M/Tsy1ZSYu7jlzqIzveFBbZgpw/jHaemN0e24ZlYVHb3T80+EnSjyZGnPeuWPLZNl10EcaVHXqkp/pHCLudoTcG0umfUZnIlDTDGTR7K/vci0ZsMTd25ctx6yJ8ida+xiIJ0mmjHuRcmyZ9t44xTkhCrKDdPZNjS0zXzn3PvE00B9PUmrPZKtt6z/L1sp8pbatj4s++0ter+0467ZHm3v4/357fFLH+8yTe7n7W2PLJLYWpBDXfhjJd3dE3RF5B+/ApTd6dzz3roEEA/Ek33n5BAcausb+IcZIPJniWHNPmrcdCQXS+rX1uCXuyZThaHMPDV0D7kXNsWVa7ci9MBK0I3fr9clE7vOVc+8TTwP9sSRrF+UCcKqt9yxbK/OV0x39JFKGk2Ow3x7dVsdnf7LPtQUmwuHGHnaf7hyWtpgJZ7UhJyskU9nfZMrQ0GlF4Kc7+tPe17tWaXPPcHEHONnax6ef2MsPXvVWGJka2ntjpMxg2QGwIndvv7Ycb2VFqbUY9aHGbncwdUVpDi09MYwxtPfFKIqEKMwO0dEXI2qX5A0F5qFpPklU3CdJKmWIJlKsLrciDo3cFy5OFNkzcHZrps2eTDMZG6elJ0pfLEl733CrL55McdXnNvP4a3X2c0PAJ27J2Uxlf1t6osSThpVlOUQTqTQ/2xshN3WlTwRyeOloCwDdY/j846XV7ou3sqJ3QLWxa4ATrX2887JqQn4fBxoGxX3j0kI6++O09lqTlpzIPWUGB4RD/nOvOK2K+yRxivVX5GchgnruC5heOy1vLILtCGfvBErWOjgRdG2GO4XT7f3Udw64GSZOWduAnSGTyXN3BlMvs/PbvRkz/bGk60s3ezJjOjyLVf/2iDWRaTq+4+0ZxD0SDBBLpEimjGvJXL2yhBVlORxq6KauvR8RuKiqELDudJz3yLcXz3Aspsnkuc9XVNwniRPxREJ+ckIBeqILfkLuOUuf/bftHoO4O5Fo7wS/DwPxpBshO+l+XhxryLmIOLaMI9CZyv6eGSLupz0ZM33xJJUFWQR8kmbLtPfF3ZoqW45Z4t7VP7EL1hefOcSfP/RaxtcyRe7OqkY90QSHG7vxCaxdlMfaRXkcbOimtr2Pyvws18rZcaodsKydQlvcnbuQoA6oKuPFyZTJDvotj3ASkZoyt5nqyP1wYzc3fuF5WnuGpxm2eiyTTFbfqWHibgj6fQR8ti2TKXK3B1EvX+5E7oPv2x9LkBMKUJobTrNlOvtiLC/NIeAT96LWHZ1Y5P7ysVZ+e6Ql42uOfVIUGRT3qqIIACdaejna3MvS4gjhgJ+1i/I40znAsweaqCqOuMkMT+9twGdH8oX2+zgXqsnkuc9Xzr1PPMU4qWpZIT854YC7UICy8Ogbj+fe40TuI2+7+3QnR5p6ONjQPey1Fk/GSqbI/ZQ909QbuQf8QsAdUM0cuedlBagqyiY76OfpvQ3c/MXnOdLUTX88SXbIT1leeFjkXpwToixvsDriRD33xq4BWnsHBzm9OJ+jyFO/Za2dOXOwsZujzT2sKLOSFq5cUULI72P94nz+4sbVlNnivquuk3WV+eSEA24dGOdCdS6mQs7P7Pw5RH/MLtjkRO4zmAetTD2plOG7W07y0131PHDXpW4ECINC3XOWu7NoIulGuaPZMk7t8sYMsz+d6oZBv7gTdbw4kbsT8TqLQAdHEffTHQMsKcxGRFhcmMUW28feV99NXyxJbjhAYXYwrTJkh519Up4Xpt7OtOnqH3/kbsxgpk5TV5Tq4kja6229MQqyg2lrkC4tjpAV9LG/vovjLb1cu7oUgEuWFnHwH291a71475YvXWYt5+xYNU65BU2FVMaNa8vYnvtkBtCU2ecTj+3mb5/Y61Yb9HK2yP3pvQ18/Ie70mqKjxa5OxFwptmfLd2WaK+vzKcuw4DqKduqce4QHFsm6GbLpNsyxhgONHSx1BbV8yrzWWwLYGdfjP5Ykqygn/L8sFs+N5kydA0kKMgOUpZnbbusJEL3QGJM6ZleOvsHi3g1ZpgQ1dYbc5e0c/D7hNXleTx7oIloIsVKO3IH0op4RUIBIvbMXEfc87OClOSEONZsifu5GLmfe594inHEPSvoJyd89hKlytzmuUNNXLXCqn54tLkn7bW+s3juT+9p4AfbajndMSjGo13snQJYmcTOsUY2VhdS196ftsqXMca1ZbqjCWKJ1KAt48s8iWnvmS7q2vu5cV0FAF9610ae+cjvAFYue388SSTkp6ooQkuPJfbOkneFkSDl+Zb1ccnSIhIp437vx4r3biBTzfi23hhFGRaYXlORxwm72urK8txhrzs4vvslS4vctmUlEXdgWbNllHHjZMtkB/1EwgF61JaZ13T2x9mwJJ+iSJCjzekT0pzxlJH+xk72iTfiH82WcSL3pkyRe0+U3HCAVRV5xJKptKqMbb0xemNJVpZZE3qcAlneSUxDB1R/vrsev0+4ab0l7kG/j5xwgJyQn87+OH0xR9ytzJi69j43x70oEmJVWS75WQEurCpI6/tYcSyZoY9PtPQST1o5997BVAfvjFVv5D6U0lzLOvKullRTkuM+1gFVZdwMpNkyfjddTpl/RBNJBuIpCrKDrCzLHR65O577COLuTO1PF/fBbVt6omm1YgZtmeGRbEtPjNLcENW2WHkHVR2/fWO1FaW29saIJ43tuQ+P3I0x/Hx3PVevLBkWHRdkW9P0B2xbxvHCa9v73MlTBZEgd121jOf++k3ugtHjzXX3fkbn8fOHmnnT55/jB6/WZrRlANbY4l4YCaalSQ7l7qtr+Kub16TZNcu84q62zPgQkRMisltEdorINrutWESeEZHD9u+is73PfCY9FXL8nnvXQJxP/3jPjBakUjLj2BAFkRArynI4NkTcR4vckylDvT0pyMm3Fhm0ZYwxvPW/XuRjP9zl7uPaMhkGVFu6o5TkDkai3lowpzwzM8GaAOTUlglkKD9woKGbE619vHlD5bDjFERCdPTF6bNtmWo7/bC2rZ/O/sHIPej3UZwTIi8rYJ+r8Ubu1p3H4oIsGrqiNHUN8JEf7MQYeL22wyobkEG811ZY4j5a1A5w+8YlvOuypWltNaWDg7Y6oDox3mSM2WiM2WQ/vw/YbIxZDWy2ny9YvLaM47mPZ7Bpy7E2vvPKSbadbJ+uLipjxFlQ2YncW3pibht4KidGhw8oNndH3UHM+s4B/D6hLDfsXrTrOwc40znAT3ad4XCjlfrY5RlQHfp+LT1RSnNDlNgLNHtLBTgrfl1kWyRO5B5IG1AdjNxfspfIu/688mGfuSA7QEtPlGTKEAkFKM0NkRX0UdvW5w4MOxOCwBqoBCty748lSYxxvdaGrgFKckJUFUdo7BzgC88coieaYGVZDq+eaCOeNBkj94r8MKW5YdZVjn+xDG/kHshQ236hMx2Xs9uBB+3HDwJ3TMMxpoRkyox71H8o7oBqyEckFCBp15oZK04qW1vv9NfLVkbHjdxtcQc42jIYvTtReDw5/G98ekh99KJIkNysgOu57z5trfFpDHz52SMAdNvHiyVS7rEdLHEPk58dxCfpNV72N3RRVZTNYnvB5nbbcw/5M5cf2H6yjaqi7LSiXA6F2SHXA3eWkqsqilDb3ueWHvB64fl25N41kOD3vvwin35iz/ATmYHGrgEq8rNYlJ9FQ9cAzx9q5oZ15dywrsIdMM0UuYsIP/zAVfz1zeeN6TheakqsyD0U8M27JfKmgsmKuwF+KSLbReQeu63CGFMPYP8eHi7MEd7yXy/yFfsfbSSeP9TM84eaR3y9P5bEJ9aATa5d1N/JmGnoHOA7L58Y9f2df9rWntio2ynTT5q425kZR5sGxd07njLUmnFmezqLOxfnhMgND9p0e0934hN471XL+MnrZ2jpidI1kHDF0psOGU+maO+LU5obxu8TCrKDtNnfE2eB60uXFbkRdZvHlnEidydLxLt9Jgqyg+4qS046YXVRtmXL9FnrkTpWDODWbGnpjnKkqYdHttVxsnV4JdS+WIL/3HzYPacNnQMsKsiiIj/MqbY+6jsHeMOqsrSIvDgn8wLUNaU57gIe46EwYtV1PxcHU2Hy4n6NMeYS4M3Ah0TkjWPdUUTuEZFtIrKtuXlk8Zwu4skUBxq6ONjYM+w1Y6yIfiCe5C9/sJN/f/rgiO/TH0+SbUc8zj+Hcyv+ox11fPqJvcNWt/HSZt/6em+7ldnBK+7VRdkE/ZKWMdMbGxTjoWMkjrhfbS8ibS31NrjYxJ4zXawqz+WNq8tIGWsFoa6BOKvsi4h3wNEpolVqzwotioTcwc3THf00dkW5dFkRAb/PEv7eGPFEKt2WsSN37/aZcKongkfciyPUtffxWm0HS4sjacv1OUK/317vNJkyfPXZo8Pe96k9DXzhmUPc96NdGGPcyL0if/Du4drVpWmLcxTnhIe9z2SpKYmck2mQMElxN8acsX83AY8DlwONIlIJYP9uGmHfB4wxm4wxm8rKyibTjQnR0hPFmPTbXYd7H3qNP/3Odn66q5623tio4uxM24bB5bicaM35h23PcAyHDteWUXGfbbziHvD7WFaS4/rjxhj6YknKbXEamgroTO3fsMTywUtywlbkbkf7e053smFxgStude19xBKpjOJ+xB7IdabVF+WE3O/JdntsxsnnLskJWeKesmvLOKmQtuc+dPuh5Hv8dGcVp+qiCF0DCV480sLtG5ekbZ8d9OP3ibuY9YVVBfxoR92wjB9n1aRf7Gngv58/RmtvjEWeIl9LiyNUF0dYWZbrRtbFGVIhJ8vy0hwioXNzIv6ExV1EckQkz3kM3AzsAZ4E7rY3uxt4YrKdnA6c2+BMwrvjZDu/3NfI39p+YmtvNG0SiRcnhQzwRO6DNahhdOF2jt86AXFPpgy3f+W3PLpt6hdPOBdxxN2Jzi9fXswrx1qJJpLEklbp2XI7mh4auTtT+5fZPm9xTsiuNZSgqWuApu4oG5YUuJOBjjZZdwSOuDt57KmU4V+fOkhpbphrVll3AUWRoHuHt+NkO5GQ383/LnLEPZki5BeC9iSm9t44D209xS/3NZLj2X4ohR67w/n+Ohk6xsDbL04XdxEhPyvAIfui98nb1pFIGR5/7XTadluOt3LjunJuXFfOvzx1AIBFBWEW2Re3N9ilBIJ+H6srrHNQnDv14v5XN6/li/Zi2ucak4ncK4AXReR1YCvwM2PMU8A/AzeJyGHgJvv5nMONqnvTB7IS9oSRcMBHXyzJirIc4kkzbMDLwZnZB4ORuzOT0fmHHVXcJ2HL7Kzt4PXaDl490TbufZXhdPbHyQsH3Pom168tpzeWZOvxNtdvdyLvTJ77ksJsd+JMSW7ISo2NJtlzxhpM3bCkgJKcECKe6DwvTEF2kMauAaKJJA+8cIydtR3c9+bzyLMzUwojnsj9VDsbqwvdPhY74p6wJzHZKw59d8tJPvHYbn62q56LPNsPpcATuWcHB20ZgE3LiqgpzRm2T15W0MqrD/i4vKaYS5YW8qPtdW5ygrOwxpUrSvjvP7qUj926ltLcEBdVF7KqPJfyvDBvuXAwLXN9ZT6hgI+c0NQvqFFdHHGrYJ5rTPh+xRhzDLgoQ3srcMNkOjVWmrujPL23gZvWV6R5eWPBWfB3qC3T0mOt5vKxW9aSlxUkFPDx0Udfp7knmnE03/HcYXjk7sw8HEvkPhFxf/aA5Xg1zMDK9POZx3bU8fTeBr5+16ZRt+vsi6fZFNesKiUc8PHrA00st0XOidyHivvpjn4uXVZEUU6IL77rIq5cUcK3XjxObzTBwQZLyNdV5hHw+yjJCXPEHqjNzwpSkR/m4a21PLKtloF4iiuWF6dFzI6A98eS7K/v5oPXrRx8LRJi8/5GUgYqC7Pd8gPN3VGWFGbz/muXc+mykcWtMDt95SOwrIzinBDvvbom4z752ZZsVBVm4/MJ77i0ik89voc9p7u4oKqAV+y671csLyHg9/HB61bxwetWuftv/dSNae/3wTet4vrzys/JjJbpZF6bUY1dA/zNj/dQnBPitguGT9AYfV97GbRYklgi5c5gc+perCrP5YZ1Fbxs5wi3dEdZUzH81rbfY8vkeiJ3Y4zr1bePKu4Tj9w32+Le2Dl8EowyyCPbannlWBv9scHxkUx09sfTI9mQn6tWlvDrA03caU+QKfOI+6sn2thYXUjUTmWsLLQCjLddXAXYCzzHk5xs7bUnAFnvXZ4Xdme/5mUF+ejNa3nhcAuhgI9rV5dyzarStEHMwkiQaMJKAEimjFsKFywrI2XgLRdWcvdVy/B79rt8eTF/fM3yUc9Nui1jfX9zwgG2/82NI4ptXtjap8qO8N9ywWI+++Q+HnutjguqCthyvI28cID1i/Mz7j+U5aU57sVTmTrmdY7QqvJcfELGethnwzsA1NE/KKwN9hRyZ+DH+WduzrCgAljlBxzBcP45eu2iS046WmtvjAMNXfzfb7+atlalMYaOvhg+sYRlpBXrM1Hf2c/++i5CAV/GQkyKRTSRdMsBDM1FH8pQcQdr4s/J1j722taKM6C642QHf/DfL/PEzjNuuuSK0vRZlM7F/kBDt1tGwHqPsJsnn5cV4ObzF/EPd2zg029Zz3Vry4fNpnTyzJ1ceWcRC7A88Y/evIYvvWsjAb+Vz+3kul8yQoaMl0y2DDBqFO1kzDifqSAS5JpVJTx/0Mp6e+lIC5ctL0670Cgzz7wW96ygn5qSnAmJu7cQU0dfnC88c4gfv3barVldWWB9cZ2MhZEyZry2jLMsWG80kfb+7X0xnjvYzK8PNLGztsNt744mSKSM63GOllUzlGcPWP9It21YRGd/PK1myULFGHPWv3VtWx/v+/arbpbL67WdrpBORNydFEKnpIDzfXjhsHX+d9d1cNA+1tohg5bOGMyhxm43yoVBawfSs1VGwhH3XXWWuHsvFKsr8rj3+tVpnrqTMXPpCBkyXrz546Pd1Xhx+uytyf6G1WUca+nlxcMtnGjt47q1M58Bp6Qzr8UdrJKgzsj9eGjsGnAHcNp7Yzz40gm+88pJGjoHCAV87oow+dkBQn7fiJG7V9yzAn5ErAJTjt/uE8tyqbMLPzkRIECHPZi6yp4NOR5rZvfpDooiQa5ZZWUdNJwD1sxTexq45Uu/Gfb3ru/s5/3/u40vPnOIOx94hc0Hmnh6bwMAW4+3utvVZVjRyEsmcXemsO87Y6X+5WVZlRSdi/f++m4ONnSTFfS5tdIdnIt9Xyzp1mwB0saHvBOERsL5Lu6u6yQS8o9aQAsg6LMGJ4debDKRGwrgBNiRMYr7YOTuEXf7e/iPP9sHwJvWztm5i+cM81/cF+VxorV33JFrU3fUrThX195PZ3+cfWe6qOvop7Igy70tFRFKc0Pu4glD6Y+lyLL/KXw+IRL00xtLurZPTWkObb0xt/DTntOD4u7MOnRmQ7aNY5bq4cYeVlfkufbRuWDNOAN13gtkIpniww/t5LmDTfzH5sN0D8QpyQmx57QlxluOt7G6PJeAT9KKb2Wisz8+bCZkbthaV/SAfceQHfKT6xHk/Q1dHGzoZnV53jAbIseTX+0tRetE7iKWuJ4NR8wPN3VTXRQ568BjwC9cvLRoTLaIz54BG/DJmItrOWMH1cWDn2lNhZUFc6ChmzUVucNWWlJmnnkv7uctyiNlcLMPxkI0kaStN+bm/jqpav3xJK8cbXVzcR2GrivpZcATuYN1K94XG7RlzluUR3tvzF0qbY8dAcKgDePU5R5rrrsxhsNNPawuz3X7mqls7EJju22NHPbMKv7ys0fYeqKNf/39C9nyyRv45V/+DleuLGHPmU7iyRTbT7Zz9coSKguzRrVlBuJJoonUsMgdrFmOTkmJnFDAtVsq8sN0DyTYeqIt42B7JDz4vfCKnbOqUW44kDZwOhLOUn8pky6oI3HXVTX8n2tqzrqdQ0F2cMyWDEBlQRYhv49lxYODoCLiRu/Xn1cx5vdSpo95L+7OP9UBjxebSKb496cPUptheTIY9M+dcqLeaLq1N0ZlQQZxz+C5G2PSbBmwxL03mqSp27J9qooitNri7vcJR5t73AkwThaNM5HFa8v0RhP808/3Z7RbmnuidPbHWV2eS0XByOLe1hvjc7/YPy/8+FTKjFr2uDeaYH+99Tc+bF/Ij7f08pVnj3D7xsW87eIqqzBVQRYXLCmgrr2fp/Y00BdLcs2qUpYUZqetRdoXS6RNTPPOTh2Kt7pgJOwnzxb3P7i0GrAKf2WaJOQMqALDBlRhsMLi2fBmtHgHU0fiIzet4YZ1YxfYgkgo7Tt8Nt5xSRVP/cW1w+5yfsf22W9ar5bMXGDei3tNSYRQwJfmw75wuIUvP3uER7fXZdzHSYNcVppDKOBz/VQniFpUkB4dleaGackQuceThmTKpEU9kZDfjdzL87MoioSIJlL0x5NcsbwYYwbrcjhpkDUlOYikR+6ffmIPD/zmGD/ddWbYcY/Ykevqijzywtb6kQ2dUeLJVFqVy5/trufrzx/jxcMtI5y9ucPXf3OMK/9p87Aa6g6v13aQTBkKI0E3O+UffrqPcMDPp353Xdq2GxZbJQC++MwhckJ+3rimjKqiiGvLRBNJ3vivz/EvTx9w9xlN3J3qgmAthO5E7u/cVI3jkKzJIO7OdiKwJIMtMxa/HaxZnM62XntnqijIDo7ZbweryuKKDPXVf+/CxTz+watHzatXZo55L+4Bv49VZblpkfsPd1iivtcTkXtxJjAtys+iKBKkN5YkHPBxQVUhQMbIvdWuee2N9py0xixv5G7PSmzuilKWF06rUX3rhkXA4J2CkwZZFAlRmB10y/4+/lodj+04nbatFydyXV2ei4iwKD+L0x193PzF3/CFZw652zkXrS2eQcWhJJIp/vv5o7Ne2+bx1+rojia49/uvZbzTcGqk3LFxCSdae3nhsJV99OEbVlOel/73Ot/Orz7W0sv16yrICvpZUphNY/cAsUSKFw+30NIT5du/PUFT9wDJlBk9crdzsEMBHwG/j6KcENXF2SwtibDcjuozRe6O516Rl0U4MPgdcdJrxxq5w2DGzHR42TetK+cW+7s5GXw+y+tX5gbzXtwBLqouZNuJNvpiCTr74zyzrxGw8oL7Ygmu+KdfpdVfcW7PK+zIGmBJYTYX2kWfhta9Ls0NkzJw1ze3cMdXf+uK+jdeOAZYIusQCfuteiLdA5TnhdNmtV6ytIjS3DBb7XIBbb0xCiMhfD5xZyEOxJN87ucHuHhpIdefV+7mNjtYfns3+VkBVyQq8rP49YEmjrf08tzBwQqbzh3C1uMjlyd4/lAz//yLA3x/y8mMr9d39vPYDmtqufdxa0+Urz13lF11HZOuiX+kqZtDjT3cuK6CffVdvO/BV90qiw7bT7WzpiKXS5YVkTLwr08dJC8c4K6rlg17v6KcEEvsWue32aJVVZSNMdbn+fnuBnJCfuLJFB/74S6u/Nxm/uz7rwGjR+5OdtV9t57H/7z3MgDWL7bWW/WmNzo42TJDffJwwE9hJOjO9BwLTsZM9RhsmfFy11U1fOLN686+oTKvmNczVB3u2LiYh7ae4um9DfTHUsQSKe7YuJgf7zzDj187Q2NXlKf3NvIHmyyP9MUjLe4Ua+efeUlRNhfYK9s4wuDgiKizos3fPbmHS5YW8ZXnjvDOTVW8cc1gTm9OKMDhxh5ae6PcsK4irUZ1dVGEt1+yhP954RjHW3rp6Iu7/7SluWH2nunimy8ep6k7ypfetZGtJ9p49mATvdEEOeEAn/vFfn65t5GAT1hlR+1gXYycZdX21XfRF0sQDvg52NBNwCfsOdNFTzRBbjiAMYa9Z7rYeryNd11WzS/2WCmDmw80ce/1q4ed2/t/tp+f7qqnqz/OD7bVsb++i+WlOTy1p4Gv/8a6uN191TI+e/uGtP121nYQjSe5YkXJWf9+v9ht9eEf79jATevL+exP9vHm/3iBJ++9hmUlObT3xnj5aCt3XlbtXkh3n+7kzsuq0+6avFxYVUBrb9T1gR1b5HhLL8/sa+DWDZWkjFXwak1Frju/IVOaoTNw6ExS80bP9735PBq7ohkzWJztM/nkN5xXwdpFoy8d58UJEqrGMKCqKLBAxP2ymmKqi7P59ksnOdPRz/rKfN59+VJ+vPMMX33OWozj1RNtpFKGgUSSl4+1cteVVsTnRO5VRdm89aLFhPw+97bewYnK3rxhETWlOXztuaM8sq2O8xfn85m3np+2bU7Y72Zl3LphkVujOjccID87wPuvXcGDL53gS786RG17n3v8D1y3knv+dxv/9vRBNi0r4qqVJfTHk65H39IT4+vPH8MnVtbEu+wLFQzmTV+9soSXjrays7aDRflZ9MeT3L5xMU/sPMO2E21ct7acTz6+h4e2ngLgWEsPv9zbQMjvY2dtB609UXcBZLB86F/uayTk9/GZn1j5y0G/8Oj2On61r5FrV5dSlhvmu1tO8b43rKCxe4BIyM+K0lz+5MFXae+L8593XszveopEgTUA+ZPXz/DGNWWU5ob42e56Ll1WxKKCLN512VIuqynmjq/8lj976DV++IGreWRbLdFEij+8YhnLSiLuOXj7JVUjfic+dut53HXlskFBtgX2y78+QtdAgtsuWMTFS4u4dnUpb7lwMa29UV6v7choexREghRFgm4k7qWqKDLiIKffJ1xWU+TWePfy+XcOK8s0KsU5ISvaH4eVo5zbLAhx9/mEt11cxX9uPkw44OM777vc/Yera+8nJ+Snsz/OwcZu6tr7iSVS7nqSRXZkvaQwm6ygnzuGlDgF2FhdyD/csYE7Ni4mK+inIi/MmkV5XLG8ZFgusSMmV60o4bKaYncNzqqibESEsrww77liGd/67XEA/vAKq2bJm9aW8+U/vIRPPb6bj96yFhFxa4M/vbeBH7xaywVLCvj728/nnu9s58qVg4NWG6sLWVyQxefefgG/82/PseNkO8vtqfB/dOUyfrarni3H27hgSQGPbqvljo2LCfh9fPcVS+Q/fMNq/mPzYZ472Mw7Lq2ivrOfA/Xd1HdaHvW3/ngTf/+TfbzlwsXUtvfx8NZTpAx89q3nc8myIn66u54/e2gHe850kRPy885N1bT0xFhVnsu9D+3g4z8KcNXKEr72nks43tLLnz+8k/31Xawsy+HqlaUcaOjmH+8YjPxXlOXyr79/IR/47g7u/f4O9tV3ceWKYndSzrKSHBKpFJtGmV4/tF7JooIsKguy2HayncqCLN6wupRwwO9eICoLst1ZyZlYZg96j5dHP3D1+HfKwAevW8UdG4d/NxVlJBaEuAP8waVVfH/LKT52y1rOW2RF3itKczjW0sv737iCL/3qMFuOtXKwsYfccIDLaixxLHQj95G9zIDf50b6wKjFmJz0tz+/wbI48rMD+H2SZvV8+IbVFOcEuXJFSdoiCrecv4ib11e4t/jledbiwN944Ti54QBf/sOLWVaSw9ZP3pBmA9y6YRG3nG/tt7o8l+0n2+mPJ/H7hAuWFLCppojHd5wmNxwgkTJ84LqVVORl8av9jcQTKf7fdSv5/tZTbD7QyG0XVPLeb27lcFMP4YCPVeW5vGltOW9aa1Xte/5QM0/sPENBdpDr15UTDvi587Jq/vflk1xUVcDR5l7+58XjXLWihG/cvYlvvnCcuvY+Ht1ex8d+tItnDzTh9wkfv/U8/nPzYb7zyknee9Uy3nNF+sr1t26o5G9+dx3/8tQB4knDJ28b9IT//vbzCQf8Y8oRdwj6fbz48evt5eh846578je/u85dAHs2WFWe66bMKspYWDDiXl0cYcsnb0j7pz1/SQHHWnp5zxXLeHRbHU++foaTrX1cu7rUrQLpeN5LpijF7B2XVlGeH+bKFdbFQ0RYU5HHRdWF7jYFkWBGf9vZ3vv4giX5PHuwmX96+wVuvnUmf9dpu3RZET/fXU9bX5yVZTlkBf389S1recfXXubzvzzI+sp89+L3lT+8hI6+OFlBP797QSXffukE++tf4ERrL3dftYyHX63lriuXpR3vDatKWVYS4eb1FW4GyF/euIZFBVncdeUynjvYzH0/2sVf3rSG3HCAD99ofU6/T3j41Voq8sP84J6rqCnN4YoVxeyq7eC9V9Vk/Ex/cu0Krl5ZyguHm7l5/WDe9rWrJ1a3xO8T/L6J1QzfVKPpfcr8Qiab6TAVbNq0yWzbtm3K33d3XSc7a9u566oaPvLITh7bcZqyvDDfvHsTF9ppj5v3N3Lv91/jxY+/Kc1vnkqSKYPAuCJNh63H2zjQ0MV7r6oZ0/Y/ef0Mf/aQlfnx7sur+dzbLwTgIz/YyWOvnebTb1nP+94w/M4jlkjxn5sP89XnjnDvm1bxkZvXEktYiy4PFd5oIknAN3L06y2h7NAfS/K154/ytouXaHlXRZkiRGS7MSbjQgULWty97K/v4snXz3DPtSvS0hOdWaYLZZ1FYwx17f3EkymqiyNuvZCWnihf/vUR/vKmNRnT/Ry6BqzViHThBEWZ+8yKuIvIrcB/AH7gf4wxIy63NxPiriiKstAYTdynZRKTiPiBrwBvBtYD7xaR9dNxLEVRFGU40zVD9XLgiDHmmDEmBjwM3D5Nx1IURVGGMF3ivgSo9Tyvs9sURVGUGWC6xD3TaFyauS8i94jINhHZ1tzcnGFzRVEUZaJMl7jXAdWe51VAWu1aY8wDxphNxphNZWW63qKiKMpUMl3i/iqwWkSWi0gIuBN4cpqOpSiKogxhWpK7jTEJEbkXeBorFfJbxpi903EsRVEUZTjTNnPHGPNz4OfT9f6KoijKyMyJGaoi0gxkXi1ibJQCc3EtOe3X+NB+jZ+52jft1/iYaL+WGWMyDlrOCXGfLCKybaRZWrOJ9mt8aL/Gz1ztm/ZrfExHvxbEMnuKoihKOiruiqIoC5CFIu4PzHYHRkD7NT60X+NnrvZN+zU+prxfC8JzVxRFUdJZKJG7oiiK4kHFXVEUZQEyr8VdRG4VkYMickRE7pvFflSLyLMisl9E9orIh+32z4jIaRHZaf/cNgt9OyEiu+3jb7PbikXkGRE5bP8uOtv7TEO/1nrOy04R6RKRv5iNcyYi3xKRJhHZ42kb8RyJyCfs79xBEbllhvv1byJyQER2icjjIlJot9eISL/nvP33dPVrlL6N+Leb5XP2A0+fTojITrt9xs7ZKBoxfd8zY8y8/MEqa3AUWAGEgNeB9bPUl0rgEvtxHnAIa5GSzwAfneXzdAIoHdL2r8B99uP7gH+ZA3/LBmDZbJwz4I3AJcCes50j++/6OhAGltvfQf8M9utmIGA//hdPv2q8283SOcv4t5vtczbk9c8DfzvT52wUjZi279l8jtznzIIgxph6Y8wO+3E3sJ+5Xb/+duBB+/GDwB2z1xUAbgCOGmMmM0t5whhjfgO0DWke6RzdDjxsjIkaY44DR7C+izPSL2PML40xCfvpK1gVV2ecEc7ZSMzqOXMQa2HgdwIPTcexR2MUjZi279l8Fvc5uSCIiNQAFwNb7KZ77Vvob82G/YFVR/+XIrJdRO6x2yqMMfVgfemA8lnol5c7Sf+Hm+1zBiOfo7n0vfu/wC88z5eLyGsi8ryIXDtLfcr0t5sr5+xaoNEYc9jTNuPnbIhGTNv3bD6L+1kXBJlpRCQX+BHwF8aYLuBrwEpgI1CPdUs401xjjLkEaz3bD4nIG2ehDyMiVknotwKP2k1z4ZyNxpz43onIp4AE8D27qR5Yaoy5GPgI8H0RyZ/hbo30t5sT5wx4N+lBxIyfswwaMeKmGdrGdc7ms7ifdUGQmUREglh/tO8ZYx4DMMY0GmOSxpgU8A2m6VZ0NIwxZ+zfTcDjdh8aRaTS7ncl0DTT/fLwZmCHMaYR5sY5sxnpHM36905E7gbeArzH2Aatffveaj/ejuXRrpnJfo3yt5sL5ywAvB34gdM20+csk0Ywjd+z+Szuc2ZBENvL+yaw3xjzBU97pWeztwF7hu47zf3KEZE85zHWYNwerPN0t73Z3cATM9mvIaRFU7N9zjyMdI6eBO4UkbCILAdWA1tnqlMicivwceCtxpg+T3uZiPjtxyvsfh2bqX7Zxx3pbzer58zmRuCAMabOaZjJczaSRjCd37OZGCmexhHo27BGnY8Cn5rFfrwB65ZpF7DT/rkN+A6w225/Eqic4X6twBpxfx3Y65wjoATYDBy2fxfP0nmLAK1Agadtxs8Z1sWlHohjRUzvG+0cAZ+yv3MHgTfPcL+OYHmxzvfsv+1t32H/jV8HdgC/NwvnbMS/3WyeM7v928AHhmw7Y+dsFI2Ytu+Zlh9QFEVZgMxnW0ZRFEUZARV3RVGUBYiKu6IoygJExV1RFGUBouKuKIqyAFFxVxRFWYCouCuKoixA/n8p29MXqYsyOwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(r.total_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "16af2a8bbb083ea23e5e41c7f5787656b2ce26968575d8763f2c4b17f9cd711f"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('py38')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}