4-gpt2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device cuda\n",
      "model loaded to cuda\n",
      "model compiled to cuda\n"
     ]
    }
   ],
   "source": [
    "\"\"\" model: gpt2\n",
    "- (Vaswani et al. 2017 https://arxiv.org/abs/1706.03762)\n",
    "- (Radford et al. 2019 https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf)\n",
    "- (Brown et al. 2020 https://arxiv.org/abs/2005.14165)\n",
    "\n",
    "pre-gpt       ->    gpt2                                 URL\n",
    "-----------------------------------------------------------------------------------------\n",
    "- ReLU        ->    GeLU: (Hendrycks, Gimpel 2016)       https://arxiv.org/abs/1606.08415\n",
    "- BatchNorm   ->    LayerNorm: (Ba et al. 2016)          https://arxiv.org/abs/1607.06450\n",
    "- N/A         ->    Residuals: (He et al. 2015)          https://arxiv.org/abs/1512.03385)\n",
    "\n",
    "\n",
    "Dimension key:\n",
    "\n",
    "# windows\n",
    "B: batch size\n",
    "T: sequence length\n",
    "\n",
    "# input/output\n",
    "V: vocabulary size\n",
    "D: model dimension (n_embd)\n",
    "\n",
    "# attention\n",
    "N: number of transformer blocks (n_layer)\n",
    "H: number of attention heads in a layer (n_head)\n",
    "K: size of each attention key or value (n_k)\n",
    "\"\"\"\n",
    "from dataclasses import dataclass\n",
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "torch.manual_seed(1337)\n",
    "device = \"cuda\" if torch.cuda.is_available() else 'cpu'\n",
    "print(f\"using device {device}\")\n",
    "\n",
    "@dataclass\n",
    "class GPTConfig:\n",
    "    # windows: B, T\n",
    "    batch_size: int = -1   # B\n",
    "    block_size: int = 1024  # T\n",
    "    # input/output:  V, D\n",
    "    vocab_size: int = 50304  # V (256 bytes + 50,000 BPE merges + 1 <|endoftext|> token)\n",
    "    n_embd: int = 768      # D\n",
    "    # attn: NH\n",
    "    n_layer: int = 12      # N\n",
    "    n_head: int = 12       # H\n",
    "\n",
    "class MHA(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        T, D, H = config.block_size, config.n_embd, config.n_head\n",
    "        assert D % H == 0\n",
    "\n",
    "        self.H = H\n",
    "\n",
    "        self.c_attn = nn.Linear(D, 3 * D)\n",
    "        self.c_proj = nn.Linear(D, D)\n",
    "        self.c_proj.GPT2_SCALE_INIT = 1\n",
    "        self.register_buffer('bias', torch.tril(torch.ones(T, T)).view(1, 1, T, T)) # tril -> bias for HF\n",
    "\n",
    "    def forward(self, X_BTD):\n",
    "        B,T,D = X_BTD.shape\n",
    "        H = self.H\n",
    "        # 1. project to learned QKV subspaces Q=WqX, K=WkX, V=WvX\n",
    "        Wq_DK, Wk_DK, Wv_DK = self.c_attn(X_BTD).split(D, dim=2)\n",
    "        Q_BHTK, K_BHTK, V_BHTK = Wq_DK.view(B, T, H, D // H).transpose(1, 2), Wk_DK.view(B, T, H, D // H).transpose(1, 2), Wv_DK.view(B, T, H, D // H).transpose(1, 2)\n",
    "\n",
    "        # 2. evaluate scores A(QKV) = softmax(QK^T/sqrt(d_k))V\n",
    "        # A_BHTT = Q_BHTK @ K_BHTK.transpose(-2, -1) * (1.0 / math.sqrt(K_BHTK.size(-1)))\n",
    "        # A_BHTT = A_BHTT.masked_fill(self.bias[:, :, :T, :T]==0, float('-inf'))\n",
    "        # A_BHTT = F.softmax(A_BHTT, dim=-1) # todo, when dim=-1?\n",
    "        # S_BHTD = A_BHTT @ V_BHTK\n",
    "        S_BHTD = F.scaled_dot_product_attention(Q_BHTK, K_BHTK, V_BHTK, is_causal=True)\n",
    "\n",
    "        # 3. contextualize the embeddings\n",
    "        S_BTD = S_BHTD.transpose(1, 2).contiguous().view(B, T, D) # performs cat\n",
    "        S_BTD = self.c_proj(S_BTD)\n",
    "\n",
    "        return S_BTD\n",
    "\n",
    "class FFN(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        D = config.n_embd\n",
    "        self.c_fc = nn.Linear(D, 4*D) # projecting up to extract features from context embeddings\n",
    "        self.gelu = nn.GELU(approximate='tanh') # (Hendrycks et al. https://arxiv.org/abs/1606.08415)\n",
    "        self.c_proj = nn.Linear(4*D, D) # projecting back down to residual pathway\n",
    "        self.c_proj.GPT2_SCALE_INIT = 1\n",
    "\n",
    "    def forward(self, X_BTD):\n",
    "        X_BT4D = self.c_fc(X_BTD)\n",
    "        X_BT4D = self.gelu(X_BT4D)\n",
    "        X_BTD = self.c_proj(X_BT4D)\n",
    "        return X_BTD\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# class LayerNorm(nn.Module): # manual inefficient LayerNorm implementation\n",
    "#     def __init__(self):\n",
    "#         super().__init__()\n",
    "\n",
    "#     def forward():\n",
    "#         # ...\n",
    "\n",
    "class Block(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        D, H = config.n_embd, config.n_head\n",
    "        self.ln_1 = nn.LayerNorm(D)\n",
    "        self.attn = MHA(config)\n",
    "        self.mlp = FFN(config) # .mlp for HF\n",
    "        self.ln_2 = nn.LayerNorm(D)\n",
    "\n",
    "    def forward(self, X_BTD):\n",
    "        # residuals:\n",
    "        # - (He et al. 2015 https://arxiv.org/abs/1512.03385)\n",
    "        # - (Elhage et al. 2021 https://transformer-circuits.pub/2021/framework/index.html)\n",
    "        X_BTD = X_BTD + self.attn(self.ln_1(X_BTD))\n",
    "        X_BTD = X_BTD + self.mlp(self.ln_2(X_BTD))\n",
    "        return X_BTD\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class GPT(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "        B, T = config.batch_size, config.block_size\n",
    "        V, D = config.vocab_size, config.n_embd\n",
    "        N, H = config.n_layer, config.n_head\n",
    "\n",
    "        self.transformer = nn.ModuleDict(dict(\n",
    "            wte = nn.Embedding(V, D), # Wt\n",
    "            wpe = nn.Embedding(T, D), # Wp\n",
    "            h = nn.ModuleList([Block(config) for _ in range(N)]),\n",
    "            ln_f = nn.LayerNorm(D),\n",
    "        ))\n",
    "        self.lm_head = nn.Linear(D, V, bias=False)\n",
    "        self.transformer.wte.weight = self.lm_head.weight # weight sharing (40m/120m ~30% save)\n",
    "        self.apply(self._init_weights) # weight init (roughly Xavier)\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        std=0.02 # default std for nn.Linear() and nn.Embedding(). nn.LayerNorm defaults are OK\n",
    "        if isinstance(module, nn.Linear):\n",
    "            if hasattr(module, 'GPT2_SCALE_INIT'):\n",
    "                std = (2 * self.config.n_layer ** -0.5)\n",
    "            if module.bias is not None:\n",
    "                torch.nn.init.zeros_(module.bias) # instead of default of unit gaussian\n",
    "\n",
    "        if isinstance(module, nn.Linear) or isinstance(module, nn.Embedding):\n",
    "            torch.nn.init.normal_(module.weight, mean=0.0, std=std) # ~ 1/sqrt(D={768, 1024, 1280, 1600}) (Xavier init)\n",
    "\n",
    "    def forward(self, X_BT, Y_BT=None): # Some(Y_BT) => training, None => inference\n",
    "        B, T = X_BT.shape\n",
    "        # 1. embedding: BTD\n",
    "        Xtok_BTD = self.transformer.wte(X_BT)\n",
    "        Xpos_TD = self.transformer.wpe(torch.arange(0, T, dtype=torch.long, device=X_BT.device))\n",
    "        X_BTD = Xtok_BTD + Xpos_TD\n",
    "        # 2. N transformer blocks: Nx(BTD -> BTK -> BTD)\n",
    "        for h in self.transformer.h:\n",
    "            X_BTD = h(X_BTD)\n",
    "        # 3. logits: BTD -> BTV\n",
    "        X_BTD = self.transformer.ln_f(X_BTD)\n",
    "        logits_BTV = self.lm_head(X_BTD)\n",
    "        loss = None\n",
    "\n",
    "        if Y_BT is not None:\n",
    "            V = self.config.vocab_size\n",
    "            loss = F.cross_entropy(logits_BTV.view(B*T, V), Y_BT.view(B*T)) # reshape for .cross_entropy()\n",
    "        return logits_BTV, loss\n",
    " \n",
    "    @classmethod\n",
    "    def from_pretrained(cls, model_type):\n",
    "        \"\"\"Loads pretrained GPT-2 model weights from huggingface\"\"\"\n",
    "        assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'}\n",
    "        from transformers import GPT2LMHeadModel\n",
    "        print(\"loading weights from pretrained gpt: %s\" % model_type)\n",
    "\n",
    "        # n_layer, n_head and n_embd are determined from model_type\n",
    "        config_args = {\n",
    "            'gpt2':         dict(n_layer=12, n_head=12, n_embd=768),  # 124M params\n",
    "            'gpt2-medium':  dict(n_layer=24, n_head=16, n_embd=1024), # 350M params\n",
    "            'gpt2-large':   dict(n_layer=36, n_head=20, n_embd=1280), # 774M params\n",
    "            'gpt2-xl':      dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params\n",
    "        }[model_type]\n",
    "        config_args['vocab_size'] = 50257 # always 50257 for GPT model checkpoints\n",
    "        config_args['block_size'] = 1024 # always 1024 for GPT model checkpoints\n",
    "\n",
    "\n",
    "\n",
    "        # 1. model init\n",
    "        model_hf, model = GPT2LMHeadModel.from_pretrained(model_type), GPT(GPTConfig(**config_args))\n",
    "        sdhf, sd = model_hf.state_dict(), model.state_dict()\n",
    "        sdhf_keys, sd_keys = sdhf.keys(), sd.keys() # .collect::<Vec<_>>() semantics\n",
    "        # filter\n",
    "        sdhf_keys = [k for k in sdhf_keys if not k.endswith('.attn.masked_bias')] # ignore these, just a buffer\n",
    "        sdhf_keys = [k for k in sdhf_keys if not k.endswith('.attn.bias')] # same, just the mask (buffer)\n",
    "        sd_keys = [k for k in sd_keys if not k.endswith('.attn.bias')] # discard this mask / buffer, not a param\n",
    "\n",
    "        # 2. copy\n",
    "        transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight']\n",
    "        assert len(sdhf_keys) == len(sd_keys), f\"mismatched keys: {len(sd_keys_hf)} != {len(sd_keys)}\"\n",
    "        for k in sdhf_keys:\n",
    "            if any(k.endswith(w) for w in transposed):\n",
    "                # special treatment for the Conv1D weights we need to transpose\n",
    "                assert sdhf[k].shape[::-1] == sd[k].shape\n",
    "                with torch.no_grad():\n",
    "                    sd[k].copy_(sdhf[k].t())\n",
    "            else:\n",
    "                # vanilla copy over the other parameters\n",
    "                assert sdhf[k].shape == sd[k].shape\n",
    "                with torch.no_grad():\n",
    "                    sd[k].copy_(sdhf[k])\n",
    "\n",
    "        return model\n",
    "\n",
    "# model = GPT.from_pretrained('gpt2')\n",
    "model = GPT(GPTConfig())\n",
    "model.to(device) # 2x speedup in latency and throughput\n",
    "print(f'model loaded to {device}')\n",
    "model = torch.compile(model)\n",
    "print(f'model compiled to {device}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded 338025 tokens\n",
      "1 epoch = 20 batches\n",
      "step:    0 | loss: 10.974476 | norm: 9.6957 | latency(ms): 330.83 | throughput(tok/s): 49524.20\n",
      "step:    1 | loss: 9.555631 | norm: 4.8675 | latency(ms): 95.73 | throughput(tok/s): 171145.34\n",
      "step:    2 | loss: 9.298462 | norm: 6.9572 | latency(ms): 94.81 | throughput(tok/s): 172815.28\n",
      "step:    3 | loss: 9.038286 | norm: 3.5688 | latency(ms): 94.08 | throughput(tok/s): 174156.33\n",
      "step:    4 | loss: 8.726753 | norm: 3.5111 | latency(ms): 93.60 | throughput(tok/s): 175047.57\n",
      "step:    5 | loss: 8.588159 | norm: 2.6650 | latency(ms): 93.75 | throughput(tok/s): 174763.11\n",
      "step:    6 | loss: 8.409719 | norm: 1.7345 | latency(ms): 93.81 | throughput(tok/s): 174656.07\n",
      "step:    7 | loss: 8.173799 | norm: 1.9291 | latency(ms): 93.56 | throughput(tok/s): 175119.39\n",
      "step:    8 | loss: 7.880895 | norm: 1.7339 | latency(ms): 93.58 | throughput(tok/s): 175081.47\n",
      "step:    9 | loss: 7.657985 | norm: 1.9489 | latency(ms): 93.74 | throughput(tok/s): 174787.56\n",
      "step:   10 | loss: 7.443972 | norm: 1.9300 | latency(ms): 93.56 | throughput(tok/s): 175118.05\n",
      "step:   11 | loss: 7.290960 | norm: 1.4244 | latency(ms): 94.12 | throughput(tok/s): 174076.04\n",
      "step:   12 | loss: 7.079813 | norm: 1.2834 | latency(ms): 93.75 | throughput(tok/s): 174769.78\n",
      "step:   13 | loss: 7.011836 | norm: 1.2306 | latency(ms): 93.77 | throughput(tok/s): 174729.78\n",
      "step:   14 | loss: 6.956169 | norm: 1.0882 | latency(ms): 93.94 | throughput(tok/s): 174409.60\n",
      "step:   15 | loss: 6.749241 | norm: 1.1275 | latency(ms): 93.76 | throughput(tok/s): 174736.00\n",
      "step:   16 | loss: 6.696558 | norm: 1.0080 | latency(ms): 94.01 | throughput(tok/s): 174274.70\n",
      "step:   17 | loss: 6.622490 | norm: 0.9316 | latency(ms): 93.92 | throughput(tok/s): 174437.05\n",
      "step:   18 | loss: 6.585139 | norm: 0.9970 | latency(ms): 93.79 | throughput(tok/s): 174696.03\n",
      "step:   19 | loss: 6.412384 | norm: 1.1624 | latency(ms): 94.07 | throughput(tok/s): 174169.58\n",
      "step:   20 | loss: 6.452065 | norm: 1.2328 | latency(ms): 93.95 | throughput(tok/s): 174390.13\n",
      "step:   21 | loss: 6.182490 | norm: 0.9593 | latency(ms): 93.78 | throughput(tok/s): 174702.24\n",
      "step:   22 | loss: 6.279158 | norm: 1.1000 | latency(ms): 94.03 | throughput(tok/s): 174240.67\n",
      "step:   23 | loss: 6.224972 | norm: 0.9862 | latency(ms): 93.92 | throughput(tok/s): 174437.94\n",
      "step:   24 | loss: 6.161290 | norm: 0.8480 | latency(ms): 93.69 | throughput(tok/s): 174873.40\n",
      "step:   25 | loss: 6.392858 | norm: 1.2824 | latency(ms): 94.18 | throughput(tok/s): 173965.43\n",
      "step:   26 | loss: 6.462299 | norm: 1.3572 | latency(ms): 94.12 | throughput(tok/s): 174079.57\n",
      "step:   27 | loss: 6.335101 | norm: 1.1299 | latency(ms): 93.78 | throughput(tok/s): 174700.91\n",
      "step:   28 | loss: 6.244492 | norm: 0.8820 | latency(ms): 94.03 | throughput(tok/s): 174234.05\n",
      "step:   29 | loss: 6.170014 | norm: 1.1879 | latency(ms): 93.97 | throughput(tok/s): 174355.62\n",
      "step:   30 | loss: 6.154809 | norm: 1.0515 | latency(ms): 93.82 | throughput(tok/s): 174637.87\n",
      "step:   31 | loss: 6.154753 | norm: 0.7005 | latency(ms): 94.39 | throughput(tok/s): 173569.10\n",
      "step:   32 | loss: 6.129659 | norm: 1.3038 | latency(ms): 93.84 | throughput(tok/s): 174589.06\n",
      "step:   33 | loss: 6.265074 | norm: 1.3181 | latency(ms): 94.07 | throughput(tok/s): 174167.81\n",
      "step:   34 | loss: 6.325114 | norm: 1.0380 | latency(ms): 94.39 | throughput(tok/s): 173572.60\n",
      "step:   35 | loss: 6.163735 | norm: 1.3287 | latency(ms): 93.86 | throughput(tok/s): 174553.58\n",
      "step:   36 | loss: 6.112011 | norm: 1.3488 | latency(ms): 93.94 | throughput(tok/s): 174416.24\n",
      "step:   37 | loss: 6.109898 | norm: 0.9988 | latency(ms): 94.37 | throughput(tok/s): 173615.14\n",
      "step:   38 | loss: 6.103150 | norm: 1.0398 | latency(ms): 94.47 | throughput(tok/s): 173429.36\n",
      "step:   39 | loss: 5.942720 | norm: 1.1576 | latency(ms): 94.06 | throughput(tok/s): 174188.56\n",
      "step:   40 | loss: 6.173913 | norm: 1.3886 | latency(ms): 94.38 | throughput(tok/s): 173602.86\n",
      "step:   41 | loss: 5.911260 | norm: 1.1421 | latency(ms): 94.14 | throughput(tok/s): 174032.84\n",
      "step:   42 | loss: 6.039691 | norm: 0.8292 | latency(ms): 94.00 | throughput(tok/s): 174295.47\n",
      "step:   43 | loss: 5.935438 | norm: 1.0527 | latency(ms): 94.32 | throughput(tok/s): 173698.08\n",
      "step:   44 | loss: 5.866347 | norm: 1.1020 | latency(ms): 94.31 | throughput(tok/s): 173726.18\n",
      "step:   45 | loss: 6.074138 | norm: 0.9927 | latency(ms): 93.92 | throughput(tok/s): 174438.82\n",
      "step:   46 | loss: 6.194418 | norm: 0.8467 | latency(ms): 94.27 | throughput(tok/s): 173805.27\n",
      "step:   47 | loss: 6.058079 | norm: 0.9522 | latency(ms): 94.29 | throughput(tok/s): 173755.61\n",
      "step:   48 | loss: 5.984130 | norm: 0.9553 | latency(ms): 94.26 | throughput(tok/s): 173808.35\n",
      "step:   49 | loss: 5.900980 | norm: 0.9828 | latency(ms): 94.29 | throughput(tok/s): 173757.37\n",
      "5.900979995727539\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAASqxJREFUeJzt3XlcVOX+B/DPmRlmBgQGFGSXxQ1XNFTCJTVJ0n6mtpmaml0tTbslbdp1yxbrdq+lZWq2aNt1yaVSs4wUN9REcQMUBQHZQWHYtzm/P9ApCoUBZs7M8Hm/XvMqzzzn8J2TNh+f8yyCKIoiiIiIiMyYTOoCiIiIiBrCwEJERERmj4GFiIiIzB4DCxEREZk9BhYiIiIyewwsREREZPYYWIiIiMjsMbAQERGR2VNIXUBL0Ol0yMjIgIODAwRBkLocIiIiagRRFFFUVARPT0/IZHfuQ7GKwJKRkQEfHx+pyyAiIqImSEtLg7e39x3bGBxYDh48iPfeew8xMTHIzMzEjh07MG7cuDuec+DAAURERODChQvw8fHBwoUL8eSTT+rfX7p0KV5//fU653Tt2hUJCQmNqsnBwQFA7Qd2dHQ06PMQERGRNLRaLXx8fPTf43dicGApKSlBUFAQnnrqKTz00EMNtk9OTsYDDzyAWbNm4ZtvvkFkZCRmzJgBDw8PhIeH69v16NEDv/766x+FKRpf2q3HQI6OjgwsREREFqYxwzkMDiyjRo3CqFGjGt1+7dq18Pf3x3//+18AQLdu3XD48GG8//77dQKLQqGAu7u7oeUQERFRK2D0WULR0dEICwurcyw8PBzR0dF1jiUmJsLT0xMBAQGYPHkyUlNTb3vNiooKaLXaOi8iIiKyXkYPLFlZWXBzc6tzzM3NDVqtFmVlZQCAkJAQbNiwAXv37sWaNWuQnJyMIUOGoKioqN5rLl++HBqNRv/igFsiIiLrZhbrsIwaNQqPPvooevfujfDwcOzZswcFBQXYsmVLve0XLFiAwsJC/SstLc3EFRMREZEpGX1as7u7O7Kzs+scy87OhqOjI2xtbes9x8nJCV26dMHly5frfV+lUkGlUrV4rURERGSejN7DEhoaisjIyDrH9u3bh9DQ0NueU1xcjCtXrsDDw8PY5REREZEFMDiwFBcXIzY2FrGxsQBqpy3HxsbqB8kuWLAAU6dO1befNWsWkpKS8MorryAhIQEff/wxtmzZgnnz5unbvPTSS4iKisLVq1dx9OhRjB8/HnK5HBMnTmzmxyMiIiJrYPAjoZMnT2L48OH6X0dERAAApk2bhg0bNiAzM7PODB9/f3/s3r0b8+bNw8qVK+Ht7Y1PP/20zpTma9euYeLEicjPz4erqysGDx6MY8eOwdXVtTmfjYiIiKyEIIqiKHURzaXVaqHRaFBYWMiF44iIiCyEId/fZjFLiIiIiOhOGFiIiIjI7DGwEBERkdljYLmDovIq/PeXi3j1u7OwgqE+REREFouB5Q5s5DJ8+NtlbD6ZBm1ZtdTlEBERtVoMLHegtpHDxb52Rd20G6USV0NERNR6MbA0wMu5dvuAazfKJK6EiIio9WJgaYD3zcCSXsDAQkREJBUGlgZ463tY+EiIiIhIKgwsDfB24iMhIiIiqTGwNMDb2Q4AAwsREZGUGFgaoB/DwkdCREREkmFgacCtWULa8moUllVJXA0REVHrxMDSADulAm3bKAEA6XwsREREJAkGlkbg1GYiIiJpMbA0Aqc2ExERSYuBpRE4U4iIiEhaDCyN4OXEHhYiIiIpMbA0AsewEBERSYuBpRH4SIiIiEhaDCyNcGstloLSKhSVcy0WIiIiU2NgaQR7lQJOdjYA+FiIiIhICgwsjfTHEv0MLERERKbGwNJI3k4cx0JERCQVBpZG8uLicURERJJhYGmkP1a7ZQ8LERGRqTGwNNKtqc0cdEtERGR6DCyNxB4WIiIi6TCwNNKtMSzXSypRWlktcTVEREStCwNLIzmqbeCoVgDg1GYiIiJTY2AxAJfoJyIikgYDiwE4tZmIiEgaDCwG4MBbIiIiaTCwGED/SIhTm4mIiEyKgcUA7GEhIiKSBgOLAbycbm2AyDEsREREpsTAYgCfm4+E8oorUVZZI3E1RERErQcDiwEcbRVwUN1ci4XjWIiIiEyGgcUAgiBwajMREZEEGFgMxIG3REREpsfAYiDu2kxERGR6DCwGYg8LERGR6TGwGOjW1GaOYSEiIjIdBhYDcQNEIiIi02NgMdCtR0K5RRUor+JaLERERKbAwGIgJzsbtFHKAQAZHHhLRERkEgwsBqq7FgsDCxERkSkwsDQBx7EQERGZFgNLE9wax5JewJlCREREpsDA0gRci4WIiMi0GFiawMuJj4SIiIhMyeDAcvDgQYwZMwaenp4QBAE7d+5s8JwDBw7grrvugkqlQqdOnbBhw4a/tVm9ejX8/PygVqsREhKCEydOGFqayXhzA0QiIiKTMjiwlJSUICgoCKtXr25U++TkZDzwwAMYPnw4YmNj8cILL2DGjBn4+eef9W02b96MiIgILFmyBKdOnUJQUBDCw8ORk5NjaHkmcSuw5BRVoKKaa7EQEREZmyCKotjkkwUBO3bswLhx427b5tVXX8Xu3btx/vx5/bHHH38cBQUF2Lt3LwAgJCQE/fv3x0cffQQA0Ol08PHxwXPPPYf58+f/7ZoVFRWoqKjQ/1qr1cLHxweFhYVwdHRs6sdpNFEU0W3xXpRX6XDgpWHwc2lj9J9JRERkbbRaLTQaTaO+v40+hiU6OhphYWF1joWHhyM6OhoAUFlZiZiYmDptZDIZwsLC9G3+avny5dBoNPqXj4+P8T5APQRB4NRmIiIiEzJ6YMnKyoKbm1udY25ubtBqtSgrK0NeXh5qamrqbZOVlVXvNRcsWIDCwkL9Ky0tzWj13w6nNhMREZmOQuoCmkKlUkGlUklaA6c2ExERmY7RA4u7uzuys7PrHMvOzoajoyNsbW0hl8shl8vrbePu7m7s8pqMU5uJiIhMx+iPhEJDQxEZGVnn2L59+xAaGgoAUCqVCA4OrtNGp9MhMjJS38YccWozERGR6RgcWIqLixEbG4vY2FgAtdOWY2NjkZqaCqB2fMnUqVP17WfNmoWkpCS88sorSEhIwMcff4wtW7Zg3rx5+jYRERFYv349Nm7ciPj4eMyePRslJSWYPn16Mz+e8ejHsLCHhYiIyOgMfiR08uRJDB8+XP/riIgIAMC0adOwYcMGZGZm6sMLAPj7+2P37t2YN28eVq5cCW9vb3z66acIDw/Xt5kwYQJyc3OxePFiZGVloU+fPti7d+/fBuKak1uzhLK05ais1kGp4KLBRERExtKsdVjMhSHzuFuKKIoIXLQXFdU6HHx5ODq0szPJzyUiIrIWZrUOi7USBAFeHMdCRERkEgwszaBfPK6A41iIiIiMiYGlGbycuBYLERGRKTCwNAOnNhMREZkGA0szcGozERGRaTCwNAM3QCQiIjINBpZmuNXDkqUtR3WNTuJqiIiIrBcDSzO42quglMtQoxORWVgudTlERERWi4GlGWSyP9ZiSefUZiIiIqNhYGmmP2YKMbAQEREZCwNLM/2xFgunNhMRERkLA0szsYeFiIjI+BhYmunW1GauxUJERGQ8DCzNpN8AsYCPhIiIiIyFgaWZbj0SyiwoR41OlLgaIiIi68TA0kztHdSwkQuo1onI1nItFiIiImNgYGkmuUyA582ZQqdSb0hcDRERkXViYGkBo3t5AACW70lASUW1xNUQERFZHwaWFvDcvZ3g7WyL9IIyrNh3SepyiIiIrA4DSwuwUyrw5rieAIAvjiTj3LVCiSsiIiKyLgwsLWRY1/YYE+QJnQgs2HGWuzcTERG1IAaWFrT4/7rDUa3A+XQtNhy9KnU5REREVoOBpQW5OqiwYHQ3AMCKfZe4gzMREVELYWBpYRP6+aC/nzNKK2uweOd5iCIXkyMiImouBpYWJpMJWP5QL9jIBUQm5OCn81lSl0RERGTxGFiMoFN7B8we2hEAsPSHC9CWV0lcERERkWVjYDGSZ4d3QoBLG+QUVeDfexOkLoeIiMiiMbAYidpGjjfH167N8s3xVMSkcNl+IiKipmJgMaKBHV3wSLA3RBF4bfs5VHFtFiIioiZhYDGyf43uhrZtlLiYXYRPDiZJXQ4REZFFYmAxMuc2Six8oHZtllWRiUjJL5G4IiIiIsvDwGIC4/t6YVCndqio1mHJDxe4NgsREZGBGFhMQBAEvDG2J5RyGQ5czMXPF7KlLomIiMiiMLCYSICrPZ6+JwAAsOzHCyitrJa4IiIiIsvBwGJCc4Z3gpeTLTIKy/HRb5elLoeIiMhiMLCYkK1SjiVjugMA1h9KwuWcYokrIiIisgwMLCZ2X3c33BvYHlU1Ipb8wM0RiYiIGoOBxcQEQcDSMT2gVMhw5HI+dp/LlLokIiIis8fAIoEO7ezw7LDazRHf2BWH4goOwCUiIroTBhaJzBraEb7t7JCtrcDKXy9JXQ4REZFZY2CRiNpGjqUP9gAAfH7kKi5mFUlcERERkfliYJHQ8K7tMbK7G2p0IhZ9zwG4REREt8PAIrHFY7pDbSPDieTr2BmbLnU5REREZomBRWLeznZ47t7OAIC3diegsKxK4oqIiIjMDwOLGZg5JAABrm2QV1yB9/dxAC4REdFfMbCYAaVChmUP9gQAfBl9FRcyCiWuiIiIyLwwsJiJwZ1d8EBvD+hE4O098VKXQ0REZFYYWMzIglGBUMgEHLmcj5iUG1KXQ0REZDYYWMyIt7MdHrrLCwDw4W+JEldDRERkPhhYzMyzwzpBJgAHLubi7LUCqcshIiIyCwwsZsbPpQ3G9bnVy3JZ4mqIiIjMQ5MCy+rVq+Hn5we1Wo2QkBCcOHHitm2rqqqwbNkydOzYEWq1GkFBQdi7d2+dNkuXLoUgCHVegYGBTSnNKjw7vBMEAdgXl424DK3U5RAREUnO4MCyefNmREREYMmSJTh16hSCgoIQHh6OnJycetsvXLgQ69atw4cffoi4uDjMmjUL48ePx+nTp+u069GjBzIzM/Wvw4cPN+0TWYFO7e3xQC8PAMBH+zmWhYiIyODAsmLFCsycORPTp09H9+7dsXbtWtjZ2eHzzz+vt/1XX32F1157DaNHj0ZAQABmz56N0aNH47///W+ddgqFAu7u7vqXi4vLbWuoqKiAVqut87I2c+/tBAD46XwWErO5MSIREbVuBgWWyspKxMTEICws7I8LyGQICwtDdHR0vedUVFRArVbXOWZra/u3HpTExER4enoiICAAkydPRmpq6m3rWL58OTQajf7l4+NjyMewCIHujgjv4QZRBD7az7EsRETUuhkUWPLy8lBTUwM3N7c6x93c3JCVlVXvOeHh4VixYgUSExOh0+mwb98+bN++HZmZmfo2ISEh2LBhA/bu3Ys1a9YgOTkZQ4YMQVFR/T0LCxYsQGFhof6VlpZmyMewGLf2GPrxTAaScoslroaIiEg6Rp8ltHLlSnTu3BmBgYFQKpWYO3cupk+fDpnsjx89atQoPProo+jduzfCw8OxZ88eFBQUYMuWLfVeU6VSwdHRsc7LGvX00mBEYHvoRGD1/itSl0NERCQZgwKLi4sL5HI5srOz6xzPzs6Gu7t7vee4urpi586dKCkpQUpKChISEmBvb4+AgIDb/hwnJyd06dIFly/zUchzI2p7WXbGpiM1v1TiaoiIiKRhUGBRKpUIDg5GZGSk/phOp0NkZCRCQ0PveK5arYaXlxeqq6uxbds2jB079rZti4uLceXKFXh4eBhSnlXq4+OEIZ1dUKMTsSaKAY6IiFongx8JRUREYP369di4cSPi4+Mxe/ZslJSUYPr06QCAqVOnYsGCBfr2x48fx/bt25GUlIRDhw7h/vvvh06nwyuvvKJv89JLLyEqKgpXr17F0aNHMX78eMjlckycOLEFPqLl++fNXpbvYq4hvaBM4mqIiIhMT2HoCRMmTEBubi4WL16MrKws9OnTB3v37tUPxE1NTa0zPqW8vBwLFy5EUlIS7O3tMXr0aHz11VdwcnLSt7l27RomTpyI/Px8uLq6YvDgwTh27BhcXV2b/wmtQH+/tggNaIfopHysPXAFb4zrKXVJREREJiWIoihKXURzabVaaDQaFBYWWu0A3KNX8jBp/XEoFTIcemU43BzVDZ9ERERkxgz5/uZeQhYiNKAd+vk6o7Jah3VRSVKXQ0REZFIMLBZCEAT9jKFvT6Qgt6hC4oqIiIhMh4HFgtzT2QVB3hqUV+mwLorrshARUevBwGJBBEHQzxj69HAyPj3ER0NERNQ6MLBYmHsD2+Ppe2oX3Xtzdzz+8/NFWMG4aSIiojtiYLEwgiBgwahAvBzeFUDtxohLfrgAnY6hhYiIrBcDiwUSBAFzhnfCG+N6QhCAL6NT8OLWM6iq0UldGhERkVEwsFiwKXf74oMJfSCXCdhxOh2zv45BeVWN1GURERG1OAYWCze2jxc+mRIMlUKGX+Nz8OQXJ1BUXiV1WURERC2KgcUKjOjmho1PDYC9SoFjSdcx+dPjuF5SKXVZRERELYaBxUrcHdAO/5t5N5ztbHD2WiEeWxeNrMJyqcsiIiJqEQwsVqSXtwZbZ4XC3VGNyznFeHTdUeQXc0VcIiKyfAwsVqZTewd8NzsUHdraIe16GWZ/cwqV1Zw9RERElo2BxQp5O9vh8yf7wV6lwInk63j9xwtSl0RERNQsDCxWqlN7B6ya2AeCAHxzPBVfHUuRuiQiIqImY2CxYvcGuulXxH39hws4lpQvcUVERERNw8Bi5WYP7YgHgzxRrRPx7DenkHa9VOqSiIiIDMbAYuUEQcC7D/dGTy9HXC+pxMwvT6KkolrqsoiIiAzCwNIK2Crl+GRKP7jYq5CQVYSXtp7hZolERGRRGFhaCU8nW6ybcheUchl+Op+FVb8lSl0SERFRozGwtCLBvm3x5rieAIAPfk3E3vOZEldERETUOAwsrcxj/X3w5EA/AEDEljNIyNJKWxAREVEjMLC0Qgsf6IZBndqhtLIGMzaeRGEpd3cmIiLzxsDSCinkMqyedBc6tLXDtRtleO+XBKlLIiIiuiMGllbKyU6Jdx/uDaB2Jdxz1wolroiIiOj2GFhasdCO7TC2jydEEVj4/XlOdSYiIrPFwNLK/Wt0N9irFDiTVoDNJ9OkLoeIiKheDCytXHtHNebd1wUA8O7eBNwoqZS4IiIior9jYCFMC/VFoLsDCkqr8O+fL0pdDhER0d8wsBAUchmWja1dUG7T76mITSuQtiAiIqK/YGAhAMAA/7Z46C4viCKwaOd51HAALhERmREGFtJbMKobHFQKnEsvxP9OpEpdDhERkR4DC+m5Oqjw4sjaAbjv/XwR+cUVEldERERUi4GF6njibl9093BEYVkV/r2XA3CJiMg8MLBQHQq5DG+M6wEA2HwyDTEpNySuiIiIiIGF6hHs2xaPBnsD4ABcIiIyDwwsVK9XRwXCUa1AXKYW3xxPkbocIiJq5RhYqF4u9iq8fH8ggNoBuLlFHIBLRETSYWCh25o0oAN6ejmiqLwaq/dflrocIiJqxRhY6LbkMgGv3uxl2fR7Kq5znyEiIpIIAwvd0eBOLujlpUF5lQ4bjl6VuhwiImqlGFjojgRBwOxhHQEAG49eRXFFtcQVERFRa8TAQg0K7+GOAJc2KCyrwiYu2U9ERBJgYKEGyWUCnhkaAABYfygJFdU1EldEREStDQMLNcq4vl5wc1QhW1uBnafTpS6HiIhaGQYWahSVQo4Zg2t7WdZFJXH1WyIiMikGFmq0iSEdoLG1QVJeCX65kCV1OURE1IowsFCj2asUmBbqCwD4+MAViCJ7WYiIyDQYWMgg0wb6QW0jw7n0Qhy5nC91OURE1EowsJBB2tmr8Hj/DgCANVFcrp+IiEyjSYFl9erV8PPzg1qtRkhICE6cOHHbtlVVVVi2bBk6duwItVqNoKAg7N27t1nXJGnNGOIPhUzAkcv5OJNWIHU5RETUChgcWDZv3oyIiAgsWbIEp06dQlBQEMLDw5GTk1Nv+4ULF2LdunX48MMPERcXh1mzZmH8+PE4ffp0k69J0vJ2tsODfTwBAGsOXJG4GiIiag0E0cCRkyEhIejfvz8++ugjAIBOp4OPjw+ee+45zJ8//2/tPT098a9//Qtz5szRH3v44Ydha2uLr7/+uknX/CutVguNRoPCwkI4Ojoa8nGoiRKzi3Df+wchCMC+eUPRqb291CUREZGFMeT726AelsrKSsTExCAsLOyPC8hkCAsLQ3R0dL3nVFRUQK1W1zlma2uLw4cPN+uaWq22zotMq7ObA+7r7gZRBNZFsZeFiIiMy6DAkpeXh5qaGri5udU57ubmhqys+tflCA8Px4oVK5CYmAidTod9+/Zh+/btyMzMbPI1ly9fDo1Go3/5+PgY8jGohdzaFHFnbDoyCsokroaIiKyZ0WcJrVy5Ep07d0ZgYCCUSiXmzp2L6dOnQyZr+o9esGABCgsL9a+0tLQWrJga664Ozrg7oC2qakR8djhZ6nKIiMiKGZQaXFxcIJfLkZ2dXed4dnY23N3d6z3H1dUVO3fuRElJCVJSUpCQkAB7e3sEBAQ0+ZoqlQqOjo51XiSN2cM6AQD+dyIVN0oqJa6GiIislUGBRalUIjg4GJGRkfpjOp0OkZGRCA0NveO5arUaXl5eqK6uxrZt2zB27NhmX5Okd09nF/TwdERpZQ2+OMJeFiIiMg6Dn8tERERg/fr12LhxI+Lj4zF79myUlJRg+vTpAICpU6diwYIF+vbHjx/H9u3bkZSUhEOHDuH++++HTqfDK6+80uhrkvkSBAFzhtf2snxyKAnXbpRKXBEREVkjhaEnTJgwAbm5uVi8eDGysrLQp08f7N27Vz9oNjU1tc74lPLycixcuBBJSUmwt7fH6NGj8dVXX8HJyanR1yTzNqqnO0L82+J48nW8sSsO66b0k7okIiKyMgavw2KOuA6L9C5mFWH0qkOo0YnY+NQADO3iKnVJRERk5oy2DgvR7XR1d8C0UD8AwNIfLqCiukbagoiIyKowsFCLeeG+znCxVyE5rwSfHuIAXCIiajkMLNRiHNU2+NcDgQCAj367jHQuJkdERC2EgYVa1Lg+Xhjg1xZlVTV4a3ec1OUQEZGVYGChFiUIAl4f2wNymYA957JwODFP6pKIiMgKMLBQi+vm4Ygpd/sCABb/cB6V1TqJKyIiIkvHwEJGMe++LnCxVyIptwSfcwVcIiJqJgYWMgqNrQ3mj+oGAFgVmYjMQg7AJSKipmNgIaN5qK8Xgn2dUVpZg7d2x0tdDhERWTAGFjIamUzAsrE9IBOAXWczcfQKB+ASEVHTMLCQUfXw1OCJmwNwl3x/AVU1HIBLRESGY2Aho3vxvq5o10aJxJxibDhyVepyiIjIAjGwkNFp7Gzw6v21K+B+8OslZBWWS1wRERFZGgYWMolHgr1xVwcnlFTW4E2ugEtERAZiYCGTkMkEvDGup34A7pHLHIBLRESNx8BCJtPDU4OpoX4AgEXfn0dFdY20BRERkcVgYCGTihjZBS72KiTlluDTQ1wBl4iIGoeBhUzKUW2Dfz1QOwD3w98Sce1GqcQVERGRJWBgIZMb18cLA/zborxKh2U/cgAuERE1jIGFTE4QBLwxtifkMgG/xGVjf0KO1CUREZGZY2AhSXR1d8BTg/wAAEt+uIDyKg7AJSKi22NgIck8H9YFbo4qpF4vxZoDV6Quh4iIzBgDC0nGXqXAov/rDgBYE3UFKfklEldERETmioGFJPVALw8M7uSCymodlv5wAaIoSl0SERGZIQYWkpQgCHh9bA/YyAXsv5iLX+KypS6JiIjMEAMLSa6jqz2evicAALDsxziUVlZLXBEREZkbBhYyC3OHd4aXky3SC8rw0W+XpS6HiIjMDAMLmQVbpRxLxtQOwP3kYBKOJ+VLXBEREZkTBhYyG/d1d8ODQZ6o1omY/c0ppF3nsv1ERFSLgYXMhiAIePfh3ujlpcH1kkrM/PIkiis4noWIiBhYyMzYKuX4ZGowXB1USMgqwrzNsdDpONWZiKi1Y2Ahs+OhscUnU4KhVMiwLy4b/913UeqSiIhIYgwsZJb6dnDGuw/3AgCs3n8F38emS1wRERFJiYGFzNb4vt6YNbQjAODl784iNq1A2oKIiEgyDCxk1l4O74qwbu1RWa3D01+eRFZhudQlERGRBBhYyKzJZQI+eLwvurjZI6eoAk9/dRLlVTVSl0VERCbGwEJmz16lwKdT+8PZzgZnrxXi5e/OcpNEIqJWhoGFLEKHdnZY80QwFDIBP57JwMcHrkhdEhERmRADC1mMuwPa4fWxPQAA7/18kcv3ExG1IgwsZFEmh/hiQj8fAMDSH+NQw0XliIhaBQYWsjjzRwVCY2uD+Ewt/nciVepyiIjIBBhYyOI4t1HixZFdAAD/+eUiCkorJa6IiIiMjYGFLNKkAR0Q6O6AgtIqvL/vktTlEBGRkTGwkEVSyGVYPKY7AOCrYylIyNJKXBERERkTAwtZrIEdXTC6lzt0IvD6D3Fcm4WIyIoxsJBFe210N6gUMkQn5eOn81lSl0NEREbCwEIWzdvZDs/c3CDxrd3xKKvksv1ERNaIgYUs3uyhHeGpUSO9oAzrDnIFXCIia8TAQhbPVinHaw90AwCsOXAF126USlwRERG1NAYWsgoP9PJAiH9bVFTrsHxPgtTlEBFRC2NgIasgCAKWPtgDMgHYfS4T0Ve4zxARkTVpUmBZvXo1/Pz8oFarERISghMnTtyx/QcffICuXbvC1tYWPj4+mDdvHsrLy/XvL126FIIg1HkFBgY2pTRqxbp5OGJSSAcAwOs/XkB1jU7iioiIqKUYHFg2b96MiIgILFmyBKdOnUJQUBDCw8ORk5NTb/tvv/0W8+fPx5IlSxAfH4/PPvsMmzdvxmuvvVanXY8ePZCZmal/HT58uGmfiFq1F+/rCo2tDRKyirjPEBGRFTE4sKxYsQIzZ87E9OnT0b17d6xduxZ2dnb4/PPP621/9OhRDBo0CJMmTYKfnx9GjhyJiRMn/q1XRqFQwN3dXf9ycXFp2ieiVq3uPkOXkFtUIXFFRETUEgwKLJWVlYiJiUFYWNgfF5DJEBYWhujo6HrPGThwIGJiYvQBJSkpCXv27MHo0aPrtEtMTISnpycCAgIwefJkpKbe/m/HFRUV0Gq1dV5Et9zaZ6iwrAr3vR+FDUeSUcXHQ0REFs2gwJKXl4eamhq4ubnVOe7m5oasrPpXGZ00aRKWLVuGwYMHw8bGBh07dsSwYcPqPBIKCQnBhg0bsHfvXqxZswbJyckYMmQIioqK6r3m8uXLodFo9C8fHx9DPgZZOYVchpWP90Xn9vYoKK3C0h/jEP7BQUTGZ3P5fiIiC2X0WUIHDhzA22+/jY8//hinTp3C9u3bsXv3brzxxhv6NqNGjcKjjz6K3r17Izw8HHv27EFBQQG2bNlS7zUXLFiAwsJC/SstLc3YH4MsTFd3B/z0/BC8Oa4n2rVRIim3BP/YeBJPfHYccRnskSMisjQKQxq7uLhALpcjOzu7zvHs7Gy4u7vXe86iRYswZcoUzJgxAwDQq1cvlJSU4Omnn8a//vUvyGR/z0xOTk7o0qULLl++XO81VSoVVCqVIaVTK6SQy/DE3b4Y28cTHx+4gs8OJ+PI5Xw88OEhPBrsjZdGdkV7R7XUZRIRUSMY1MOiVCoRHByMyMhI/TGdTofIyEiEhobWe05paenfQolcLgeA23bPFxcX48qVK/Dw8DCkPKJ6Oaht8Or9gYiMGIoxQZ4QRWDLyWsY9p8DWBWZiNLKaqlLJCKiBhj8SCgiIgLr16/Hxo0bER8fj9mzZ6OkpATTp08HAEydOhULFizQtx8zZgzWrFmDTZs2ITk5Gfv27cOiRYswZswYfXB56aWXEBUVhatXr+Lo0aMYP3485HI5Jk6c2EIfkwjwaWuHDyf2xbbZA9G3gxNKK2uwYt8lDHrnN7y/7xKul1RKXSIREd2GQY+EAGDChAnIzc3F4sWLkZWVhT59+mDv3r36gbipqal1elQWLlwIQRCwcOFCpKenw9XVFWPGjMFbb72lb3Pt2jVMnDgR+fn5cHV1xeDBg3Hs2DG4urq2wEckqivY1xnbZw/ErrOZ+M8vF5GSX4qVkYn45GASJvT3wYwh/vB2tpO6TCIi+hNBtIJpE1qtFhqNBoWFhXB0dJS6HLIgNToRP53PxNqoKzifXjsYVy4TMKa3B54Z2hHdPPj7iYjIWAz5/mZgIULteKojl/OxNuoKDl/O0x8f1tUVs4Z2RIh/WwiCIGGFRETWh4GFqBnOXSvE2oNX8NO5TOhu/ukY39cLKx4LYmghImpBhnx/c7dmor/o5a3B6kl3Yf9Lw/DE3R2gkAnYcTodBy7mSl0aEVGrxcBCdBu+7drgzXG98I8h/gCAN3bHcYl/IiKJMLAQNWDu8E761XK/OZYidTlERK0SAwtRAxzUNoi4uQP0B5GJKCjlei1ERKbGwELUCBP6+SDQ3QEFpVVYGZkodTlERK0OAwtRIyjkMix8oDsA4KvoFFzJLZa4IiKi1oWBhaiRBnd2QVi39qjWiXh7d7zU5RARtSoMLEQGeG10NyhkAiITcnAokdOciYhMhYGFyAABrvaYEuoLAHhzVzyqOc2ZiMgkGFiIDPT8iM5wsrPBxewibD6ZJnU5REStAgMLkYGc7JR4YURnAMCKXy5BW14lcUVERNaPgYWoCSbf7YuOrm2QX1KJ1b9dlrocIiKrx8BC1AQ2f5rm/MWRq0jJL5G4IiIi68bAQtREw7q6YkhnF1TW6LB8T4LU5RARWTUGFqImEgQBi/6vO2QCsPdCFo4l5UtdEhGR1WJgIWqGLm4OmBTSAQCwaOd5nEq9IXFFRETWiYGFqJnmhXWBk50NEnOK8dDHR/Ho2qPYF5cNnU6UujQiIqshiKJo8f9X1Wq10Gg0KCwshKOjo9TlUCt0Na8Eq/dfxs7YdFTV1P6R6ujaBjOHBGBcXy+obeQSV0hEZH4M+f5mYCFqQdnacnxx5Cq+OZ6CovJqAICLvQrTB/nhiRBfaOxsJK6QiMh8MLAQSayovAqbf0/DZ4eTkVlYDgCwU8rxeP8OeH5EZwYXIiIwsEhdDpFeVY0Ou85mYF1UEhKyigAA7R1UeGNcT4T3cJe4OiIiaTGwEJkZURQRdSkXy3bFISm3dpG5B3p5YOmDPeDqoJK4OiIiaRjy/c1ZQkQmIAgChnVtjz3/HII5wztCLhOw+1wmwlZE4buYa7CCvzcQERkVAwuRCalt5Hg5PBA/zB2EHp6OKCyrwktbz2DaF7/j2o1SqcsjIjJbDCxEEujhqcHOOYPw6v2BUCpkOHgpFyPfP4iNR69y/RYionpwDAuRxK7kFmPBtnM4cfU6ACDY1xkzhwRgWFdXrt9CRFaNg26JLIxOJ+KbE6l4Z088SiprANROg743sD1G9/LA8K7tYatkeCEi68LAQmShMgrK8PnhZPx0PgvpBWX647Y2cgwPdNWHlzYqhYRVEhG1DAYWIgsniiLOXCvET+cysftcJq7d+CO8qG1kGNalPV4c2QWd3RwkrJKIqHkYWIisiCiKOJ+uxZ7zmdhzLhMp+bWzibydbfHLvHtgp2RvCxFZJq7DQmRFBEFAL28NXr0/EAdeGoZdzw2Gl5Mtrt0ow39+viR1eUREJsHAQmRBBEFATy8N3hrfEwDwxdFknE69IXFVRETGx8BCZIGGdW2Ph/p6QRSB+dvOobJaJ3VJRERGxcBCZKEW/V93tGujxMXsIqw5cEXqcoiIjIqBhchCObdRYsmDPQAAH+1PxKXsIokrIiIyHgYWIgs2prcHRgS2R1WNiFe3nUUNl/UnIivFwEJkwQRBwJvje8JepcDp1AJ8GX1V6pKIiIyCgYXIwnlobDF/VCAA4L2fL3LXZyKySgwsRFZg0oAOGODfFqWVNXhtx3lYwXqQzXYxqwjfx6ajuoYzqIisAQMLkRWQyQS881AvKBUyHLyUi+2n0hs8J+16KdYcuIJvj6eaoELTKa2sxlu74zB61SE8vykWS3+8wABHZAW4pjeRlQhwtcfzIzrjvZ8v4o3dcRja1RUu9qo6bQpKK7H7XCZ2nk7H71f/WHCum4cD+nZwNnXJLW7/xRws3HG+zsaRXx9LhW/bNph5T4CElRFRczGwEFmRp+8JwO6zmYjL1GLpDxfw0aS7UF5Vg/0JOdhxOh37L+agqqa2t0EQgHZtlMgrrsS3x1MtOrDkFJVj2Y9x2HU2EwDg5WSLZWN7IDmvBG/ujsfbP8XD29kWo3p5SFwpETUVAwuRFbGRy/DvR3pj7Ooj2HU2E5XVJxGdlI+i8mp9m24ejhjf1xMPBnnh2o1SPLI2Gj+ezcDC/+sOja2NhNUbTqcTsflkGpbviYe2vBoyAZg+yB8R93VBG5UCoigi9XopvoxOwQubY+GuUVt0MCNqzRhYiKxMTy8NZgzxx7qoJPwSlw0A8NCoMbaPF8b19USg+x87oro5qtDFzR6XsovxfWw6pob6SVS14S7nFGHB9nP6R1s9vRyxfHxv9PLW6NsIgoDF/9cd126U4beEHMz88iR2PDsIPm3tpCqbiJpIEK1gNJoh21MTtQblVTVY/P15yAQBY/t4IcS/LWQyod62XxxJxus/xiHQ3QE/PT8EglB/O3Py2eFkvPNTPKpqRNjayPHiyC54cqAfFPL65xGUVFTjsXXRuJChRaf29tg2ayA0dpbVm0RkjQz5/mZgIWrlCkurMODtX1FRrcO22QMR7Gvej0yu5BYjbEUURBG4N7A9lo3tAW/nhntMsrXlGLf6CDILyxEa0A4bnxoApYITJYmkZMj3N/+0ErVyGjsb/F9vTwCwiCnO6w8mQRSBEYHt8dm0fo0KKwDg5qjGZ9P6o41SjuikfCzYfo7TnYksCAMLEWFSSAcAwK6zGSgsrZK4mtvL0Zbr15iZPayjwY+vuns6YvXkuyCXCdh26ho+/O2yMcokIiNgYCEi3NXBCYHuDqio1mH76WtSl3Nbnx1JRmWNDv18ndHPr22TrjGsa+1jJABYse8Sdp5ueJE9IpJekwLL6tWr4efnB7VajZCQEJw4ceKO7T/44AN07doVtra28PHxwbx581BeXt6saxJRyxEEQd/L8u3xVLN8VKItr8K3x2ofWc0a2rFZ15oc4otnbi4k98p3Z7H/Yk6z6yMi4zI4sGzevBkRERFYsmQJTp06haCgIISHhyMnp/4/8N9++y3mz5+PJUuWID4+Hp999hk2b96M1157rcnXJKKWN66vF2xt5EjMKUZMyo2GTzCxb46loqiiGp3b2+PewPbNvt6r9wdidC93VNbo8I8Nv2Pj0avNL9KCiKKIXWczMGrlITy/6TR0OvMLqUR/ZnBgWbFiBWbOnInp06eje/fuWLt2Lezs7PD555/X2/7o0aMYNGgQJk2aBD8/P4wcORITJ06s04Ni6DUrKiqg1WrrvIioeRzVNhgTVLsSrLkNvi2vqsHnR5IBAM8M7XjbKdqGkMkEfDChLx4N9oZOBJb8cAGLdp5HVSvYLDEm5ToeWnMUc789jfhMLb6PzcDXx1OkLovojgwKLJWVlYiJiUFYWNgfF5DJEBYWhujo6HrPGThwIGJiYvQBJSkpCXv27MHo0aObfM3ly5dDo9HoXz4+PoZ8DCK6jUkhvgCAXecyUVBaKXE1f9hxOh25RRXw0KjxYJBni11XqahdGXjBqEAIAvDVsRRM/+J3FJaZ78Dj5kjJL8Gz38Tg4TXROJ1aAFsbOcK61fZWvfNTAlLzSyWukOj2DAoseXl5qKmpgZubW53jbm5uyMrKqvecSZMmYdmyZRg8eDBsbGzQsWNHDBs2TP9IqCnXXLBgAQoLC/WvtLQ0Qz4GEd1GkLcG3T0cUVmtw7ZG7Pj8Z8Z6pFCjE/HJwSQAwD8G+7f42imCIOCZoR2x7olg2CnlOHw5D+M/PoKreSUt+nOkVFBaiTd2xSFsRRT2nMuCIAAT+vkg6uVh+GRKPwzwb4vSyhq8su0MHw2R2TL6LKEDBw7g7bffxscff4xTp05h+/bt2L17N954440mX1OlUsHR0bHOi4iar+7g25RGDb7V6US8uzcBgYv34rn/ncblnOIWremXC1lIziuBxtYGEwd0aNFr/9nIHu7YOisUnho1knJLMO7jI4i+km+0n2cKFdU1+PRQEoa+dwCfHU5GVY2Ie7q4Ys8/h+DdR3qjvaMaMpmA9x7pDbWNDMeSruObE+b1OJDoFoMCi4uLC+RyObKzs+scz87Ohru7e73nLFq0CFOmTMGMGTPQq1cvjB8/Hm+//TaWL18OnU7XpGsSkfGM7eMJO6UcV3JLcCL5+h3bVtXo8NLWM1hz4Aoqq3X48UwGRr4fhRc2nUZSbvODiyiKWBt1BQAwNdQXbVTG3f6sh6cGO+cOQpCPEwpKqzDls+PY/LtlfoEn55Vg5PsH8ebueBSWVSHQ3QEbnxqAL58agG4edf+S59uuDV69PxAAsHxPPNKu89EQmR+DAotSqURwcDAiIyP1x3Q6HSIjIxEaGlrvOaWlpZDJ6v4YuVwOoPZ/Rk25JhEZj4PaRj9O5Ns7/G27tLIaM788ie2n0yGXCXjl/q64r7sbdCKwMzYDYSuiELEltlmPVqKT8nHmWiFUChmmDfRr8nUM0d5Bjc1P340xQZ6o1ol4dds5vLU7DjUW9KikpKIaz3x1Ein5pXB1UOHdh3th9z+HYGgX19ueMy3UDwP8ah8Nzd9+1iyntlPrZvAjoYiICKxfvx4bN25EfHw8Zs+ejZKSEkyfPh0AMHXqVCxYsEDffsyYMVizZg02bdqE5ORk7Nu3D4sWLcKYMWP0waWhaxKRad16LPTTuSxcL/n74NvrJZWYuP44DlzMhdpGhvVTg/HssE5YP7Uffpw7GCMC20MnAttPpWPEiii8vPVMkwZ0ro2qHbvyWD8fuNirmvehDKC2kWPV433wQlhnAMD6Q8mY9vkJZBWWN3Cm9ERRxKvbzuJSdjFcHVTY/dxgTOjfAfIGZlbJZALevflo6MjlfPzvBMcGknkxuH91woQJyM3NxeLFi5GVlYU+ffpg7969+kGzqampdXpUFi5cCEEQsHDhQqSnp8PV1RVjxozBW2+91ehrEpFp9fZ2Qk8vR5xP12L7qWuYMSRA/961G6WY+vkJJOWWwMnOBp9N619nw8Re3hp89mR/nEkrwAe/XsL+i7nYGnMNO06n4+G7vPF8WGd4Otk2WMOFjEIcvJQLmQDM/NPPNxVBEPBCWBd0dLXHy9+dweHLeQj/4CDeGt9Tv/eSOfrscDJ2nc2EQiZgzeS70N5R3ehz/V3a4OXwQLyxKw5v7Y7DPV1cGr1XE5GxcbdmIqrXt8dT8dqOcwhwbYPIiKEQBAEJWVpM+/wEsrUV8NSo8eU/BqBTe4c7Xud06g28/2siDl7KBQDY2sgx995OmDHEHyqF/Lbn/fN/p/HDmQyMCfLEhxP7tuhnM9SV3GLM2xyLs9cKAQDj+3rh9bE94Ki2kbSuvzqWlI/Jnx5HjU7E0jHd8eQgf4OvUaMTMWFdNE6m3MCQzi748qkBBu/ZRNRY3K2ZiJrtwT6eaKOUIym3BMeSruN4Uj4eXRuNbG0FurjZY9uzAxsMKwDQt4MzvnxqALbNDkV/P2eUVdXgvZ8vYtQHh/Qh5q/Srpdi19kMANAvoS+ljq722DZ7IJ67txNkQu26MKM+OIRjSeYziyirsBxzvz2FGp2IcX08mzzmRy4T8O9HekOlkOFQYh42/85HQ2QeGFiIqF72KgXG9vUCACzbFYcpn59AUXk1+vk6Y+szA+Ghafixzp8F+7bFlmdC8f6EILjYq5CUV4Kpn5/A7K9jkF5QVqft+kNJ0InAkM4u6OmlabHP1Bw2chleHNkVW2eFokNbO6QXlGHi+mNYviceFdU1ktZWWa3D7G9ikFdciUB3Byx/qHezekUCXO3x0siuAIA3d8f/7b8PSW9fXDZe2nrGIsZVtRQGFiK6rUk31z2Jz9SislqHsG5u+HpGCDR2TXsUIggCxvf1xm8vDcVTg/whlwn46XwWwv4bhdX7L6Oiugb5xRXYcrL2b/Wzm7nJoTEE+7bFnueHYEI/H4gisO5gEsatPoqLWUWS1fTGrjicTi2Ao1qBdVOCYau8/aO2xnpqsD/6dnBCcUU1Fmw/x1lDZiTqUi5mfR2D72KuYdbXMaistv7tJACOYSGiBjy85ihiUm7g8f4+eHNcTyjkLff3nIQsLRbvvIATV2vXe/F3aYNuHg7Ycy4Lvb01+H7OILMeP/HzhSws2H4O10sqoVTI8Fg/b3Ru7wA/lzbwb9cGXs62Dc7Oaa5tMdfw4tYzAIDPn+yHewNbbrLC5ZxijF51CJXVOvz74d54rL/lbIMiiiLmbY5FyvVSfP2PEKOv4WMqFzIK8djaaJRU/tGr9+RAPyx9sIeEVTWdId/fDCxEdEe5RRVIzClCaEA7o4QHURTxfWwG3toTj9yiCv3xjyffhdG9PFr857W0nKJyvPLdWRy4+PfxODZyAT5t7eDfrg38XGpfrvYqNHQb7ZRy9PZ2gsb2zj1ZFzIK8dDHR1FRrcPzIzpj3n1dmvNR6rUu6gqW/5QAtY0MTw3yx4whAWjbRtniP6el/X71Oh5dW7sf3fKHehl1lWRTySgow/iPjyBbW4HQgHaYNtAXs74+BQBYNbFvi+6zZSoMLERkcYrKq/DBr4nYcPQqunk44Ps5g43eO9FSRFHEzxeycCq1AMl5JbiaV4KU66XN6qoXBKCrmwMG+LdFf7+2GODfFm5/mqJcUFqJMR8dRtr1Mgzv6orPpvVvkV2s/6pGJ+LJL07gUGIegNowNeVuX8wYEgBXB9OtjWOop788iV/ialdQ7+2twQ9zB0tcUfNoy6vw6JpoXMwuQuf29vhu9kBobG3w3s8JWL3/CuyUcnw/ZxA6uzU8EN6cMLAQkcUqLK2CjUKAndKyu/B1OhEZhWVIyS/Vh5ir+SW4UdrwTtB5xRVIqWehvQ5t7W6GF2fsOZeFqEu56NDWDj/OHdzkcUWNodOJ+DU+G6t+S8T5dC0AQG0jw6QBvnhmaECdIGUOkvNKcO9/D0AUAYVMQLVOxO5/DkYPT/MYwG2oymodpm84gSOX89HeQYUdcwbB6+ZaRjU6EVM+O46jV/LRqb09vp8zyKIefzGwEBFZuJyicvyefAO/X72OE8nXEZ+lxV//b622kWH77EHo7mma/++Jooj9F3OwMvIyzqQVAACUChkm9vfBrGEdDZ45ZiyLdp7HV8dScG9ge9jayLH7XCam3O2LN8b1lLo0g4miiBe3nsH2U+mwU8qx5ZnQv82cyyuuwAOrDiFbW4ExQZ5Y9Xgfsx779WcMLEREVkZbXoWYlBv4Pfk6fr96Hcl5pVj6YHdJVt0VRRGHEvOwMjIRMSk3AABKuQyje7nDxV4FhVwGG7kAhUwGhVyAjVyAjVwGhVwGpVyAg9oGGlsbON76p60CDmqbFnkEeKOkEqHvRKK8SodvZ4bc7IE4AQe1AideC2uRGVSmtGLfJayKTIRcJuDTaf0wvGv7etvFpFzHhHXHUN2MRQOlYMj3t+X0GxERtWKOahsM79r+tl9YpiQIAu7p4oohnV0QfSUfq35LxLGk69gZm9Gs6zqoFXBU28C5jQ3+Mdgf4/t6G3yNb46noLxKhx6ejggNaAdRBHza2iLtehl2n8vEI8GGX1MqW06mYVVkIgDgzXE97/jfPti3LRaM7la7rcKeePT2ccJdHZxv294SMbAQEVGTCIKAgZ1cMLCTC04kX8fhxFxU1OhQXSOiukaHypv/rNaJqLp5vLJGh6LyKhSWVUFbVo3CsiqUVdVO0S0qr0ZReTXSC8owf9s59PVxhp9Lm0bXU1Fdg43RKQBq958SBAGCAEzo54P//HIJm06kWkxgOXgpF69tPwcAmDu8U6NmOT01yA+nUm5g97lMzPnmFHY9NxjtTLhpqLExsBARUbMN8K+dydQUldU6aMuroC2rDTL/3nsR0Un5WLD9HL6dGdLo8Rjfx2Ygt6gCHho1Huj9x5T4R/v54P1fE3Ey5QYSs4vMfiZNXIYWz35zCtU6EeP7euHFkY2bri4ItTtux2dpkZRbghc2x2LD9AEWM9uuIVzploiIJKVUyOBir0KAqz36dnDGuw/3htpGhuik/EbvZSSKIj47lAygdiE1mz8tcOjmqNY/Ttlk5nsjVVbrMOvrGBRXVCM0oB3efdiwbRbsVQqsfSIYtjby2nFGv14yYrWmxcBCRERmpUM7O7x4X+1eRm/tiUe2tuH9cg4m5uFidhHaKOV4vJ7HJxMH1K7Su/3UNcn3frqTHaevIfV6KVwdVFg7JRhKheFf013cHPDOw70AAKt+u4ytJ807pDUWAwsREZmd6YP80Ntbg6Lyaiz+/nyD7dcfTAIATOjfod4Vgod2cYW7oxo3Sqvw84Vsg+u5XlJp8DmGqqrR4aP9lwHU7lLe0ErHdzK2jxemhvoCAF7+7ixe2HQa2vKG1wAyZwwsRERkdhRyGd59uDcUMgE/X8jGT+cyb9s2LkOLw5fzIBNqg87trvdYv9oBt5tOpBpUy3s/J+CuN/Zh/raz0OmMtxLIztPpSLteBhd7JSaH+Db7ekvG9EDEfV0glwnYGZuB0SsPISblegtUKg0GFiIiMkvdPBwx6+aO3Yt/uIDC26wS/Onh2t6VUb084NPW7rbXe6y/DwQBOHolHyn5JY2q4YczGVi9/wqA2vEvr+04Z5TQUl2jw+qbvSszhwS0yHoxcpmAf47ojC3PhMLb2RbXbpThsXXH8MGvl1BdY3k7PDOwEBGR2Zp7bycEuLZBblEF3t4T/7f3s7Xl+PFM7fovM4cE3PFa3s52GNLZFUDjBt/GZWjxyne1O2Hf08UVMqH2vEXfn0dLr7n649kMXM0vhbOdDZ64u/m9K38W7OuMPc8Pwbg+nqjRifjg10Q8/skxpF3/+/YP5oyBhYiIzJbaRo53H+4NANh8Mg1HL+fVeX/D0auoqhHR388ZfXycGrzepJuDb7eevIaqO/Qy3CipxDNfn0R5lQ5DOrvgiyf747+PBUEQgG+Op2LpDxdaLLTU6ER8+Ftt78qMIQFG2QvIUW2DDx7viw8m9IG9SoGTKTcweuUhfB+b3uI/y1gYWIiIyKz192uLKTd7HeZvP4eyytpZPiUV1fjmWO1CcTMa6F25ZUQ3N7jYq5BXXIHI+Jx629ToRPxz02mkXS+DT1tbfDixL+QyAeP7euO9R2pDy8boFCzbFdcioWX3uUwk5ZbAyc4G0wb6Nft6dzKurxf2/HMI7urghKKKajy/KRYRm2NRZAEDchlYiIjI7L1yf1d4aNRIvV6K92+uLbL1ZBq05dXwa2eHsG5ujbqOjVymX+120+/1D7597+eLOJSYB1sbOT6Z0g9Odkr9e48Ee+Pdh2p7fL44chVv7Y5vVmjR6UR8eHP5/X8M8oe9CXZa7tDODlueCcU/R3SGTAC2n07H2NVHkJRbbPSf3RwMLEREZPYc1DZ48+Zuy58eSsLp1Bv4/MhVAMA/BvsbtJrr4/1rHwtFXcpFekFZnfd2nc3A2qjaQbbvPtIb3Tz+viHfY/198Pb42nVOPj2cjHf2JjQ5tPx0PguJOcVwUCsw7TYznIxBIZch4r4u2PxMKDw0aiTllmDs6iM4cLH+XidzwMBCREQWYUQ3N4wJ8oROBJ784nekXi+Fk50NHgn2Meg6fi5t9BsjbvnT4NuELC1e3noWQO06KA8G3X4n7EkhHfDGzQC1LioJ//nlosGhRacT8eFvtb0rTw3yh6O66euuNFV/v7b4fu4gBPs6o6i8Gk9t+B2fHLzS4oOKWwIDCxERWYwlY7rDyc4GhWW1Yy6eCPFt0hTgx/WDb9NQoxNRUFqJp7+MQVlVDQZ3csHL4V0bvMaUu33x+oM9AACr91/BB78mGlTDL3HZSMgqgr1KgacG+Rv8GVpKewc1vp0Zggn9fKATgbf3JCBiyxmUV5nXisAMLEREZDFc7FVY/H/dAQBKuQxTBzZtCnB4D3c42dkgo7AcBy7m4PlNsUi9Xgpv59pBtgp5474epw30w6Kb9ayMTMSbu+IatfS/KIpYdXPsypMD/aCxM33vyp+pFHK883AvvP5gD8hlAnacTseEddHIKmx4WwRTYWAhIiKLMr6vF5Y/1AvrpgajvYO6SddQ28jxUN/awbcvbI5F1KVcqG1kWDclGM5tlA2cXdc/BvvjX6O7Aagd0zL2oyO4kFF4x3Mi43MQl6lFG6Uc/xgsXe/KnwmCgGkD/fDVUwPgZGeDM9cKMeajwziVekPq0gAwsBARkYURBAETB3TQ78DcVLc2RCwqrwYAvPtwb/Tw1DTpWjPvCcDaJ4LRro0SCVlFGLf6CD76LbHeFWVFUcSqm2NXpg70MzggGdvATi74Yc5gdHVzQG5RBR5fdwzfxVyTuiwGFiIiap06uzlggF9bAMCMwf4Y28erWde7v6c7fp53D8J7uKGqRsR/frmEh9dG48pfpgsfuJiLs9cKYWsjxwwz6V35qw7t7LDt2YEY2d0NlTU6vLT1DJb9GCfpkv4MLERE1Gp9OKkv1j4RjAU3H+k0l4u9CmufCMaKx4LgoFbgTFoBRq88hM8PJ0OnEyGKIlbeHLsyJdQX7exVLfJzjcFepcDaJ4LxzxGdAdSuW5N2o6yBs4xHEM1x7pKBtFotNBoNCgsL4ej49znzREREppZZWIZXvjuLQ4m12wncHdAWY/t4YcH2c1ApZDj86r1wdTDfwPJnP53LhFwmYGQP9xa9riHf3wwsRERERiKKIr4+noq3d8ej7E/ThJ8a5I/FY7pLWJl5MOT7m4+EiIiIjEQQBEy52xc/PT8E/XydAQBKhQzPDG3c3kf0B+NvWkBERNTK+bm0weZnQrH7XCY8NWq4OTZtOnZrxsBCRERkAnKZcMfl/unO+EiIiIiIzB4DCxEREZk9BhYiIiIyewwsREREZPYYWIiIiMjsMbAQERGR2WNgISIiIrPHwEJERERmj4GFiIiIzB4DCxEREZk9BhYiIiIyewwsREREZPYYWIiIiMjsWcVuzaIoAgC0Wq3ElRAREVFj3frevvU9fidWEViKiooAAD4+PhJXQkRERIYqKiqCRqO5YxtBbEysMXM6nQ4ZGRlwcHCAIAgtem2tVgsfHx+kpaXB0dGxRa9Nf8f7bVq836bF+21avN+m1ZT7LYoiioqK4OnpCZnszqNUrKKHRSaTwdvb26g/w9HRkb/hTYj327R4v02L99u0eL9Ny9D73VDPyi0cdEtERERmj4GFiIiIzB4DSwNUKhWWLFkClUoldSmtAu+3afF+mxbvt2nxfpuWse+3VQy6JSIiIuvGHhYiIiIyewwsREREZPYYWIiIiMjsMbAQERGR2WNgISIiIrPHwNKA1atXw8/PD2q1GiEhIThx4oTUJVmFgwcPYsyYMfD09IQgCNi5c2ed90VRxOLFi+Hh4QFbW1uEhYUhMTFRmmIt3PLly9G/f384ODigffv2GDduHC5evFinTXl5OebMmYN27drB3t4eDz/8MLKzsyWq2LKtWbMGvXv31q/2GRoaip9++kn/Pu+1cb3zzjsQBAEvvPCC/hjvectZunQpBEGo8woMDNS/b8x7zcByB5s3b0ZERASWLFmCU6dOISgoCOHh4cjJyZG6NItXUlKCoKAgrF69ut73//3vf2PVqlVYu3Ytjh8/jjZt2iA8PBzl5eUmrtTyRUVFYc6cOTh27Bj27duHqqoqjBw5EiUlJfo28+bNw48//oitW7ciKioKGRkZeOihhySs2nJ5e3vjnXfeQUxMDE6ePIl7770XY8eOxYULFwDwXhvT77//jnXr1qF37951jvOet6wePXogMzNT/zp8+LD+PaPea5Fua8CAAeKcOXP0v66pqRE9PT3F5cuXS1iV9QEg7tixQ/9rnU4nuru7i++9957+WEFBgahSqcT//e9/ElRoXXJyckQAYlRUlCiKtffWxsZG3Lp1q75NfHy8CECMjo6Wqkyr4uzsLH766ae810ZUVFQkdu7cWdy3b584dOhQ8fnnnxdFkb+/W9qSJUvEoKCget8z9r1mD8ttVFZWIiYmBmFhYfpjMpkMYWFhiI6OlrAy65ecnIysrKw6916j0SAkJIT3vgUUFhYCANq2bQsAiImJQVVVVZ37HRgYiA4dOvB+N1NNTQ02bdqEkpIShIaG8l4b0Zw5c/DAAw/UubcAf38bQ2JiIjw9PREQEIDJkycjNTUVgPHvtVXs1mwMeXl5qKmpgZubW53jbm5uSEhIkKiq1iErKwsA6r33t96jptHpdHjhhRcwaNAg9OzZE0Dt/VYqlXBycqrTlve76c6dO4fQ0FCUl5fD3t4eO3bsQPfu3REbG8t7bQSbNm3CqVOn8Pvvv//tPf7+blkhISHYsGEDunbtiszMTLz++usYMmQIzp8/b/R7zcBC1IrMmTMH58+fr/PMmVpe165dERsbi8LCQnz33XeYNm0aoqKipC7LKqWlpeH555/Hvn37oFarpS7H6o0aNUr/771790ZISAh8fX2xZcsW2NraGvVn85HQbbi4uEAul/9tdHN2djbc3d0lqqp1uHV/ee9b1ty5c7Fr1y7s378f3t7e+uPu7u6orKxEQUFBnfa8302nVCrRqVMnBAcHY/ny5QgKCsLKlSt5r40gJiYGOTk5uOuuu6BQKKBQKBAVFYVVq1ZBoVDAzc2N99yInJyc0KVLF1y+fNnov78ZWG5DqVQiODgYkZGR+mM6nQ6RkZEIDQ2VsDLr5+/vD3d39zr3XqvV4vjx47z3TSCKIubOnYsdO3bgt99+g7+/f533g4ODYWNjU+d+X7x4EampqbzfLUSn06GiooL32ghGjBiBc+fOITY2Vv/q168fJk+erP933nPjKS4uxpUrV+Dh4WH839/NHrZrxTZt2iSqVCpxw4YNYlxcnPj000+LTk5OYlZWltSlWbyioiLx9OnT4unTp0UA4ooVK8TTp0+LKSkpoiiK4jvvvCM6OTmJ33//vXj27Flx7Nixor+/v1hWViZx5ZZn9uzZokajEQ8cOCBmZmbqX6Wlpfo2s2bNEjt06CD+9ttv4smTJ8XQ0FAxNDRUwqot1/z588WoqCgxOTlZPHv2rDh//nxREATxl19+EUWR99oU/jxLSBR5z1vSiy++KB44cEBMTk4Wjxw5IoaFhYkuLi5iTk6OKIrGvdcMLA348MMPxQ4dOohKpVIcMGCAeOzYMalLsgr79+8XAfztNW3aNFEUa6c2L1q0SHRzcxNVKpU4YsQI8eLFi9IWbaHqu88AxC+++ELfpqysTHz22WdFZ2dn0c7OThw/fryYmZkpXdEW7KmnnhJ9fX1FpVIpurq6iiNGjNCHFVHkvTaFvwYW3vOWM2HCBNHDw0NUKpWil5eXOGHCBPHy5cv69415rwVRFMXm99MQERERGQ/HsBAREZHZY2AhIiIis8fAQkRERGaPgYWIiIjMHgMLERERmT0GFiIiIjJ7DCxERERk9hhYiIiIyOwxsBAREZHZY2AhIiIis8fAQkRERGbv/wGJQCsXmkslIAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\" training loop\n",
    "(Jordan et al. 2024) URL: https://github.com/KellerJordan/modded-nanogpt\n",
    "124M 10x speedup in wallclock time: 45m -> 4m\n",
    "========================================================================\n",
    "- network architecture: rotary embeddings, QK-norm, ReLU^2\n",
    "- muon optimizer\n",
    "- untie head & embedding, FP8 matmul for head, softcap logits (gemma 2)\n",
    "- projection and classification layers init to zero (muP)\n",
    "- skip connections from embedding to every block (and between) via U-net\n",
    "- flexattention with long-short sliding window attention (gemma 2), window size warmup\n",
    "\n",
    "        124M history:\n",
    "        01. 45.0m baseline\n",
    "        02. 31.4m tuned lr, rotary embeddings\n",
    "        03. 24.9m muon optimizer\n",
    "        04. 22.3m muon improvements\n",
    "        05. 15.2m pad embeddings, ReLU^2, zero init, QK-norm\n",
    "        06. 13.1m muon overhead\n",
    "        07. 12.0m pytorch 2.5.0\n",
    "        08. 10.8m united embedding and head\n",
    "        09. 08.2m value and embed skip connections, momentum warmup, logit softcap\n",
    "        10. 07.8m bfloat16 act\n",
    "        11. 07.2m u-net pattern skip connections, double lr\n",
    "        12. 05.0m 1024-ctx dense causal attn -> 64K-ctx flex attention\n",
    "        13. 04.6m attention window warmup\n",
    "        14. 04.4m value embededdings\n",
    "\"\"\"\n",
    "import time\n",
    "import torch\n",
    "import tiktoken\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# 1. dataloader\n",
    "class DataLoaderLite:\n",
    "    def __init__(self, B, T):\n",
    "        self.B = B\n",
    "        self.T = T\n",
    "        with open('./data/shakespeare.txt', 'r') as f:\n",
    "            text = f.read()\n",
    "        encoder = tiktoken.get_encoding('gpt2')\n",
    "        self.tokens = torch.tensor(encoder.encode(text))\n",
    "        self.i = 0\n",
    "\n",
    "        print(f\"loaded {len(self.tokens)} tokens\")\n",
    "        print(f\"1 epoch = {len(self.tokens) // (B*T)} batches\")\n",
    "\n",
    "    def next_batch(self):\n",
    "        B, T = self.B, self.T\n",
    "        tokens = self.tokens[self.i:self.i+(B*T+1)]\n",
    "        X_BT, Y_BT = tokens[:-1].view(B,T), tokens[1:].view(B,T)\n",
    "        self.i += B*T\n",
    "        if self.i + (B*T+1) > len(self.tokens):\n",
    "            self.i = 0\n",
    "\n",
    "        return X_BT, Y_BT\n",
    "# print(X_BT)\n",
    "# print(Y_BT)\n",
    "# for b in range(B):\n",
    "#     print('batch', b)\n",
    "#     for t in range(T):\n",
    "#         context = X_BT[b, :t+1]\n",
    "#         target = Y_BT[b, t]\n",
    "#         print('x:', context, '->', 'y:', target)\n",
    "\n",
    "# print(\"==========================================\")\n",
    "\n",
    "# 2. training loop\n",
    "\n",
    "train_loader = DataLoaderLite(B=16, T=1024)\n",
    "torch.set_float32_matmul_precision('high') # highest (fp32) -> high (tf32). 3x instead of advertised 8x speedup (deep learning is memory-bound)\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=3e-4, betas=(0.9, 0.95), eps=1e-8)\n",
    "steps, losses = [], []\n",
    "for step in range(50):\n",
    "    # preprocess\n",
    "    t0 = time.time()\n",
    "    X_BT, Y_BT = train_loader.next_batch()\n",
    "    X_BT, Y_BT = X_BT.to(device), Y_BT.to(device)\n",
    "\n",
    "    # 1. forward\n",
    "    optimizer.zero_grad()\n",
    "    with torch.autocast(device_type=device, dtype=torch.bfloat16): # with ampere and up\n",
    "        logits_BTV, loss = model(X_BT, Y_BT)\n",
    "    # 2. backward\n",
    "    loss.backward()\n",
    "    norm = torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) # global norm\n",
    "    # 3. step\n",
    "    optimizer.step()\n",
    "\n",
    "    # postprocess\n",
    "    torch.cuda.synchronize()\n",
    "    t1 = time.time()\n",
    "    latency = (t1-t0)*1000\n",
    "    throughput = (train_loader.B * train_loader.T) / (t1-t0)\n",
    "\n",
    "    steps.append(step)\n",
    "    losses.append(loss.log10().item())\n",
    "    print(f\"step: {step:4d} | loss: {loss.item():.6f} | norm: {norm:.4f} | latency(ms): {latency:.2f} | throughput(tok/s): {throughput:.2f}\")\n",
    "\n",
    "plt.plot(steps, losses)\n",
    "print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "B, T_MAX = 5, 30\n",
    "model.eval()\n",
    "\n",
    "import tiktoken\n",
    "encoder = tiktoken.get_encoding('gpt2')\n",
    "tokens = encoder.encode(\"Hello, I'm a language model,\")\n",
    "tokens_T = torch.tensor(tokens, dtype=torch.long) # # (T,)\n",
    "tokens_BT = tokens_T.unsqueeze(0).repeat(5, 1) # (B,T)\n",
    "X_BT = tokens_BT.to(device)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed(1337)\n",
    "while X_BT.size(1) < T_MAX:\n",
    "    with torch.no_grad():\n",
    "        logits_BTV, _ = model(X_BT)\n",
    "        logits_BV = logits_BTV[:, -1, :]\n",
    "        probs_ = F.softmax(logits_BV, dim=-1)\n",
    "        topk_probs_, topk_indices_ = torch.topk(probs_, 50, dim=-1)\n",
    "\n",
    "        X_B1 = torch.gather(topk_indices_, -1, torch.multinomial(topk_probs_, 1))\n",
    "        X_BT = torch.cat((X_BT, X_B1), dim=1)\n",
    "\n",
    "for b in range(B):\n",
    "    tokens = X_BT[b, :T_MAX].tolist()\n",
    "    decoded = encoder.decode(tokens)\n",
    "    print(\">\", decoded)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}