Копия_блокнота_"VALL_E_Toy_Example_ipynb".ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AnLimt/vall-e_test/blob/main/%D0%9A%D0%BE%D0%BF%D0%B8%D1%8F_%D0%B1%D0%BB%D0%BE%D0%BA%D0%BD%D0%BE%D1%82%D0%B0_%22VALL_E_Toy_Example_ipynb%22.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v3UFvU5_-9-Z",
        "outputId": "0c0c6dd4-65f8-4fef-8606-5899c96bcfe9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-02-21 13:19:05--  https://github.com/korakot/kora/releases/download/v0.10/py310.sh\n",
            "Resolving github.com (github.com)... 140.82.113.3\n",
            "Connecting to github.com (github.com)|140.82.113.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/266951884/0d0623be-3dec-4820-9e7b-69a3a5a75ef7?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230221%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230221T131905Z&X-Amz-Expires=300&X-Amz-Signature=e4d0ac5d5a9778b61203e06377de0dd77d96de09d9ca001f15dff817559e61bb&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=266951884&response-content-disposition=attachment%3B%20filename%3Dpy310.sh&response-content-type=application%2Foctet-stream [following]\n",
            "--2023-02-21 13:19:05--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/266951884/0d0623be-3dec-4820-9e7b-69a3a5a75ef7?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230221%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230221T131905Z&X-Amz-Expires=300&X-Amz-Signature=e4d0ac5d5a9778b61203e06377de0dd77d96de09d9ca001f15dff817559e61bb&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=266951884&response-content-disposition=attachment%3B%20filename%3Dpy310.sh&response-content-type=application%2Foctet-stream\n",
            "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 364632383 (348M) [application/octet-stream]\n",
            "Saving to: ‘py310.sh’\n",
            "\n",
            "py310.sh            100%[===================>] 347.74M  93.6MB/s    in 3.7s    \n",
            "\n",
            "2023-02-21 13:19:09 (93.6 MB/s) - ‘py310.sh’ saved [364632383/364632383]\n",
            "\n",
            "PREFIX=/usr/local\n",
            "Unpacking payload ...\n",
            "Collecting package metadata (current_repodata.json): - \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\bdone\n",
            "Solving environment: | \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\bdone\n",
            "\n",
            "## Package Plan ##\n",
            "\n",
            "  environment location: /usr/local\n",
            "\n",
            "  added / updated specs:\n",
            "    - _libgcc_mutex==0.1=conda_forge\n",
            "    - _openmp_mutex==4.5=2_gnu\n",
            "    - aiohttp==3.8.3=py310h5764c6d_1\n",
            "    - aiosignal==1.2.0=pyhd8ed1ab_0\n",
            "    - alsa-lib==1.2.7.2=h166bdaf_0\n",
            "    - argon2-cffi-bindings==21.2.0=py310h5764c6d_3\n",
            "    - argon2-cffi==21.3.0=pyhd8ed1ab_0\n",
            "    - asttokens==2.0.8=pyhd8ed1ab_0\n",
            "    - async-timeout==4.0.2=pyhd8ed1ab_0\n",
            "    - attr==2.5.1=h166bdaf_1\n",
            "    - attrs==22.1.0=pyh71513ae_1\n",
            "    - backcall==0.2.0=pyh9f0ad1d_0\n",
            "    - backports.functools_lru_cache==1.6.4=pyhd8ed1ab_0\n",
            "    - backports==1.0=py_2\n",
            "    - beautifulsoup4==4.11.1=pyha770c72_0\n",
            "    - bleach==5.0.1=pyhd8ed1ab_0\n",
            "    - brotlipy==0.7.0=py310h5764c6d_1005\n",
            "    - bzip2==1.0.8=h7f98852_4\n",
            "    - ca-certificates==2022.9.24=ha878542_0\n",
            "    - cachetools==5.2.0=pyhd8ed1ab_0\n",
            "    - certifi==2022.9.24=pyhd8ed1ab_0\n",
            "    - cffi==1.15.1=py310h255011f_2\n",
            "    - charset-normalizer==2.1.1=pyhd8ed1ab_0\n",
            "    - colorama==0.4.6=pyhd8ed1ab_0\n",
            "    - conda-package-handling==1.9.0=py310h5764c6d_1\n",
            "    - conda==22.9.0=py310hff52083_1\n",
            "    - cryptography==38.0.2=py310h597c629_2\n",
            "    - dbus==1.13.6=h5008d03_3\n",
            "    - debugpy==1.6.3=py310hd8f1fbe_1\n",
            "    - decorator==5.1.1=pyhd8ed1ab_0\n",
            "    - defusedxml==0.7.1=pyhd8ed1ab_0\n",
            "    - entrypoints==0.4=pyhd8ed1ab_0\n",
            "    - executing==1.1.1=pyhd8ed1ab_0\n",
            "    - expat==2.5.0=h27087fc_0\n",
            "    - fftw==3.3.10=nompi_hf0379b8_105\n",
            "    - flit-core==3.7.1=pyhd8ed1ab_0\n",
            "    - font-ttf-dejavu-sans-mono==2.37=hab24e00_0\n",
            "    - font-ttf-inconsolata==3.000=h77eed37_0\n",
            "    - font-ttf-source-code-pro==2.038=h77eed37_0\n",
            "    - font-ttf-ubuntu==0.83=hab24e00_0\n",
            "    - fontconfig==2.14.1=hc2a2eb6_0\n",
            "    - fonts-conda-ecosystem==1=0\n",
            "    - fonts-conda-forge==1=0\n",
            "    - freetype==2.12.1=hca18f0e_0\n",
            "    - frozenlist==1.3.1=py310h5764c6d_1\n",
            "    - gettext==0.21.1=h27087fc_0\n",
            "    - glib-tools==2.74.1=h6239696_0\n",
            "    - glib==2.74.1=h6239696_0\n",
            "    - google-auth==2.13.0=pyh1a96a4e_0\n",
            "    - google-colab==1.0.0=pyh44b312d_0\n",
            "    - gst-plugins-base==1.20.3=h57caac4_2\n",
            "    - gstreamer==1.20.3=hd4edc92_2\n",
            "    - icu==70.1=h27087fc_0\n",
            "    - idna==3.4=pyhd8ed1ab_0\n",
            "    - importlib-metadata==5.0.0=pyha770c72_1\n",
            "    - importlib_resources==5.10.0=pyhd8ed1ab_0\n",
            "    - ipykernel==6.16.2=pyh210e3f2_0\n",
            "    - ipython==8.5.0=pyh41d4057_1\n",
            "    - ipython_genutils==0.2.0=py_1\n",
            "    - ipywidgets==8.0.2=pyhd8ed1ab_1\n",
            "    - jack==1.9.21=h2a1e645_0\n",
            "    - jedi==0.18.1=pyhd8ed1ab_2\n",
            "    - jinja2==3.1.2=pyhd8ed1ab_1\n",
            "    - jpeg==9e=h166bdaf_2\n",
            "    - jsonschema==4.16.0=pyhd8ed1ab_0\n",
            "    - jupyter==1.0.0=py310hff52083_7\n",
            "    - jupyter_client==7.4.4=pyhd8ed1ab_0\n",
            "    - jupyter_console==6.4.4=pyhd8ed1ab_0\n",
            "    - jupyter_core==4.11.1=py310hff52083_1\n",
            "    - jupyterlab_pygments==0.2.2=pyhd8ed1ab_0\n",
            "    - jupyterlab_widgets==3.0.3=pyhd8ed1ab_0\n",
            "    - keyutils==1.6.1=h166bdaf_0\n",
            "    - krb5==1.19.3=h3790be6_0\n",
            "    - lame==3.100=h166bdaf_1003\n",
            "    - ld_impl_linux-64==2.39=hc81fddc_0\n",
            "    - libblas==3.9.0=16_linux64_openblas\n",
            "    - libcap==2.66=ha37c62d_0\n",
            "    - libcblas==3.9.0=16_linux64_openblas\n",
            "    - libclang13==14.0.6=default_h3a83d3e_0\n",
            "    - libclang==14.0.6=default_h2e3cab8_0\n",
            "    - libcups==2.3.3=h3e49a29_2\n",
            "    - libdb==6.2.32=h9c3ff4c_0\n",
            "    - libedit==3.1.20191231=he28a2e2_2\n",
            "    - libevent==2.1.10=h9b69904_4\n",
            "    - libffi==3.4.2=h7f98852_5\n",
            "    - libflac==1.4.2=h27087fc_0\n",
            "    - libgcc-ng==12.2.0=h65d4601_19\n",
            "    - libgfortran-ng==12.2.0=h69a702a_19\n",
            "    - libgfortran5==12.2.0=h337968e_19\n",
            "    - libglib==2.74.1=h7a41b64_0\n",
            "    - libgomp==12.2.0=h65d4601_19\n",
            "    - libiconv==1.17=h166bdaf_0\n",
            "    - liblapack==3.9.0=16_linux64_openblas\n",
            "    - libllvm14==14.0.6=he0ac6c6_0\n",
            "    - libnsl==2.0.0=h7f98852_0\n",
            "    - libogg==1.3.4=h7f98852_1\n",
            "    - libopenblas==0.3.21=pthreads_h78a6416_3\n",
            "    - libopus==1.3.1=h7f98852_1\n",
            "    - libpng==1.6.38=h753d276_0\n",
            "    - libpq==14.5=hd77ab85_1\n",
            "    - libsndfile==1.1.0=h27087fc_0\n",
            "    - libsodium==1.0.18=h36c2ea0_1\n",
            "    - libsqlite==3.39.4=h753d276_0\n",
            "    - libstdcxx-ng==12.2.0=h46fd767_19\n",
            "    - libtool==2.4.6=h9c3ff4c_1008\n",
            "    - libudev1==251=h166bdaf_0\n",
            "    - libuuid==2.32.1=h7f98852_1000\n",
            "    - libvorbis==1.3.7=h9c3ff4c_0\n",
            "    - libxcb==1.13=h7f98852_1004\n",
            "    - libxkbcommon==1.0.3=he3ba5ed_0\n",
            "    - libxml2==2.10.3=h7463322_0\n",
            "    - libzlib==1.2.13=h166bdaf_4\n",
            "    - markupsafe==2.1.1=py310h5764c6d_2\n",
            "    - matplotlib-inline==0.1.6=pyhd8ed1ab_0\n",
            "    - mistune==2.0.4=pyhd8ed1ab_0\n",
            "    - mpg123==1.30.2=h27087fc_1\n",
            "    - multidict==6.0.2=py310h5764c6d_2\n",
            "    - mysql-common==8.0.31=haf5c9bc_0\n",
            "    - mysql-libs==8.0.31=h28c427c_0\n",
            "    - nbclient==0.7.0=pyhd8ed1ab_0\n",
            "    - nbconvert-core==7.2.3=pyhd8ed1ab_0\n",
            "    - nbconvert-pandoc==7.2.3=pyhd8ed1ab_0\n",
            "    - nbconvert==7.2.3=pyhd8ed1ab_0\n",
            "    - nbformat==5.7.0=pyhd8ed1ab_0\n",
            "    - ncurses==6.3=h27087fc_1\n",
            "    - nest-asyncio==1.5.6=pyhd8ed1ab_0\n",
            "    - notebook==6.4.12=pyha770c72_0\n",
            "    - nspr==4.32=h9c3ff4c_1\n",
            "    - nss==3.78=h2350873_0\n",
            "    - numpy==1.23.4=py310h53a5b5f_1\n",
            "    - openssl==1.1.1q=h166bdaf_1\n",
            "    - packaging==21.3=pyhd8ed1ab_0\n",
            "    - pandas==1.5.1=py310h769672d_1\n",
            "    - pandoc==2.19.2=h32600fe_1\n",
            "    - pandocfilters==1.5.0=pyhd8ed1ab_0\n",
            "    - parso==0.8.3=pyhd8ed1ab_0\n",
            "    - pcre2==10.37=hc3806b6_1\n",
            "    - pexpect==4.8.0=pyh9f0ad1d_2\n",
            "    - pickleshare==0.7.5=py_1003\n",
            "    - pip==22.3=pyhd8ed1ab_0\n",
            "    - pkgutil-resolve-name==1.3.10=pyhd8ed1ab_0\n",
            "    - ply==3.11=py_1\n",
            "    - portpicker==1.5.2=pyhd8ed1ab_0\n",
            "    - prometheus_client==0.15.0=pyhd8ed1ab_0\n",
            "    - prompt-toolkit==3.0.31=pyha770c72_0\n",
            "    - prompt_toolkit==3.0.31=hd8ed1ab_0\n",
            "    - psutil==5.9.3=py310h5764c6d_1\n",
            "    - pthread-stubs==0.4=h36c2ea0_1001\n",
            "    - ptyprocess==0.7.0=pyhd3deb0d_0\n",
            "    - pulseaudio==14.0=habe0971_10\n",
            "    - pure_eval==0.2.2=pyhd8ed1ab_0\n",
            "    - pyasn1-modules==0.2.7=py_0\n",
            "    - pyasn1==0.4.8=py_0\n",
            "    - pycosat==0.6.4=py310h5764c6d_1\n",
            "    - pycparser==2.21=pyhd8ed1ab_0\n",
            "    - pygments==2.13.0=pyhd8ed1ab_0\n",
            "    - pyopenssl==22.1.0=pyhd8ed1ab_0\n",
            "    - pyparsing==3.0.9=pyhd8ed1ab_0\n",
            "    - pyqt5-sip==12.11.0=py310hd8f1fbe_2\n",
            "    - pyqt==5.15.7=py310h29803b5_2\n",
            "    - pyrsistent==0.18.1=py310h5764c6d_2\n",
            "    - pysocks==1.7.1=pyha2e5f31_6\n",
            "    - python-dateutil==2.8.2=pyhd8ed1ab_0\n",
            "    - python-fastjsonschema==2.16.2=pyhd8ed1ab_0\n",
            "    - python==3.10.6=h582c2e5_0_cpython\n",
            "    - python_abi==3.10=2_cp310\n",
            "    - pytz==2022.5=pyhd8ed1ab_0\n",
            "    - pyu2f==0.1.5=pyhd8ed1ab_0\n",
            "    - pyzmq==24.0.1=py310h330234f_1\n",
            "    - qt-main==5.15.6=hc525480_0\n",
            "    - qtconsole-base==5.3.2=pyha770c72_0\n",
            "    - qtconsole==5.3.2=pyhd8ed1ab_0\n",
            "    - qtpy==2.2.1=pyhd8ed1ab_0\n",
            "    - readline==8.1.2=h0f457ee_0\n",
            "    - requests==2.28.1=pyhd8ed1ab_1\n",
            "    - rsa==4.9=pyhd8ed1ab_0\n",
            "    - ruamel_yaml==0.15.80=py310h5764c6d_1008\n",
            "    - send2trash==1.8.0=pyhd8ed1ab_0\n",
            "    - setuptools==65.5.0=pyhd8ed1ab_0\n",
            "    - sip==6.7.2=py310hd8f1fbe_1\n",
            "    - six==1.16.0=pyh6c4a22f_0\n",
            "    - soupsieve==2.3.2.post1=pyhd8ed1ab_0\n",
            "    - sqlite==3.39.4=h4ff8645_0\n",
            "    - stack_data==0.5.1=pyhd8ed1ab_0\n",
            "    - terminado==0.17.0=pyh41d4057_0\n",
            "    - tinycss2==1.2.1=pyhd8ed1ab_0\n",
            "    - tk==8.6.12=h27826a3_0\n",
            "    - toml==0.10.2=pyhd8ed1ab_0\n",
            "    - toolz==0.12.0=pyhd8ed1ab_0\n",
            "    - tornado==6.2=py310h5764c6d_1\n",
            "    - tqdm==4.64.1=pyhd8ed1ab_0\n",
            "    - traitlets==5.5.0=pyhd8ed1ab_0\n",
            "    - typing-extensions==4.4.0=hd8ed1ab_0\n",
            "    - typing_extensions==4.4.0=pyha770c72_0\n",
            "    - tzdata==2022e=h191b570_0\n",
            "    - urllib3==1.26.11=pyhd8ed1ab_0\n",
            "    - wcwidth==0.2.5=pyh9f0ad1d_2\n",
            "    - webencodings==0.5.1=py_1\n",
            "    - wheel==0.37.1=pyhd8ed1ab_0\n",
            "    - widgetsnbextension==4.0.3=pyhd8ed1ab_0\n",
            "    - xcb-util-image==0.4.0=h166bdaf_0\n",
            "    - xcb-util-keysyms==0.4.0=h166bdaf_0\n",
            "    - xcb-util-renderutil==0.3.9=h166bdaf_0\n",
            "    - xcb-util-wm==0.4.1=h166bdaf_0\n",
            "    - xcb-util==0.4.0=h166bdaf_0\n",
            "    - xorg-libxau==1.0.9=h7f98852_0\n",
            "    - xorg-libxdmcp==1.1.3=h7f98852_0\n",
            "    - xz==5.2.6=h166bdaf_0\n",
            "    - yaml==0.2.5=h7f98852_2\n",
            "    - yarl==1.8.1=py310h5764c6d_0\n",
            "    - zeromq==4.3.4=h9c3ff4c_1\n",
            "    - zipp==3.10.0=pyhd8ed1ab_0\n",
            "    - zstd==1.5.2=h6239696_4\n",
            "\n",
            "\n",
            "The following NEW packages will be INSTALLED:\n",
            "\n",
            "  _libgcc_mutex      conda-forge/linux-64::_libgcc_mutex-0.1-conda_forge\n",
            "  _openmp_mutex      conda-forge/linux-64::_openmp_mutex-4.5-2_gnu\n",
            "  aiohttp            conda-forge/linux-64::aiohttp-3.8.3-py310h5764c6d_1\n",
            "  aiosignal          conda-forge/noarch::aiosignal-1.2.0-pyhd8ed1ab_0\n",
            "  alsa-lib           conda-forge/linux-64::alsa-lib-1.2.7.2-h166bdaf_0\n",
            "  argon2-cffi        conda-forge/noarch::argon2-cffi-21.3.0-pyhd8ed1ab_0\n",
            "  argon2-cffi-bindi~ conda-forge/linux-64::argon2-cffi-bindings-21.2.0-py310h5764c6d_3\n",
            "  asttokens          conda-forge/noarch::asttokens-2.0.8-pyhd8ed1ab_0\n",
            "  async-timeout      conda-forge/noarch::async-timeout-4.0.2-pyhd8ed1ab_0\n",
            "  attr               conda-forge/linux-64::attr-2.5.1-h166bdaf_1\n",
            "  attrs              conda-forge/noarch::attrs-22.1.0-pyh71513ae_1\n",
            "  backcall           conda-forge/noarch::backcall-0.2.0-pyh9f0ad1d_0\n",
            "  backports          conda-forge/noarch::backports-1.0-py_2\n",
            "  backports.functoo~ conda-forge/noarch::backports.functools_lru_cache-1.6.4-pyhd8ed1ab_0\n",
            "  beautifulsoup4     conda-forge/noarch::beautifulsoup4-4.11.1-pyha770c72_0\n",
            "  bleach             conda-forge/noarch::bleach-5.0.1-pyhd8ed1ab_0\n",
            "  brotlipy           conda-forge/linux-64::brotlipy-0.7.0-py310h5764c6d_1005\n",
            "  bzip2              conda-forge/linux-64::bzip2-1.0.8-h7f98852_4\n",
            "  ca-certificates    conda-forge/linux-64::ca-certificates-2022.9.24-ha878542_0\n",
            "  cachetools         conda-forge/noarch::cachetools-5.2.0-pyhd8ed1ab_0\n",
            "  certifi            conda-forge/noarch::certifi-2022.9.24-pyhd8ed1ab_0\n",
            "  cffi               conda-forge/linux-64::cffi-1.15.1-py310h255011f_2\n",
            "  charset-normalizer conda-forge/noarch::charset-normalizer-2.1.1-pyhd8ed1ab_0\n",
            "  colorama           conda-forge/noarch::colorama-0.4.6-pyhd8ed1ab_0\n",
            "  conda              conda-forge/linux-64::conda-22.9.0-py310hff52083_1\n",
            "  conda-package-han~ conda-forge/linux-64::conda-package-handling-1.9.0-py310h5764c6d_1\n",
            "  cryptography       conda-forge/linux-64::cryptography-38.0.2-py310h597c629_2\n",
            "  dbus               conda-forge/linux-64::dbus-1.13.6-h5008d03_3\n",
            "  debugpy            conda-forge/linux-64::debugpy-1.6.3-py310hd8f1fbe_1\n",
            "  decorator          conda-forge/noarch::decorator-5.1.1-pyhd8ed1ab_0\n",
            "  defusedxml         conda-forge/noarch::defusedxml-0.7.1-pyhd8ed1ab_0\n",
            "  entrypoints        conda-forge/noarch::entrypoints-0.4-pyhd8ed1ab_0\n",
            "  executing          conda-forge/noarch::executing-1.1.1-pyhd8ed1ab_0\n",
            "  expat              conda-forge/linux-64::expat-2.5.0-h27087fc_0\n",
            "  fftw               conda-forge/linux-64::fftw-3.3.10-nompi_hf0379b8_105\n",
            "  flit-core          conda-forge/noarch::flit-core-3.7.1-pyhd8ed1ab_0\n",
            "  font-ttf-dejavu-s~ conda-forge/noarch::font-ttf-dejavu-sans-mono-2.37-hab24e00_0\n",
            "  font-ttf-inconsol~ conda-forge/noarch::font-ttf-inconsolata-3.000-h77eed37_0\n",
            "  font-ttf-source-c~ conda-forge/noarch::font-ttf-source-code-pro-2.038-h77eed37_0\n",
            "  font-ttf-ubuntu    conda-forge/noarch::font-ttf-ubuntu-0.83-hab24e00_0\n",
            "  fontconfig         conda-forge/linux-64::fontconfig-2.14.1-hc2a2eb6_0\n",
            "  fonts-conda-ecosy~ conda-forge/noarch::fonts-conda-ecosystem-1-0\n",
            "  fonts-conda-forge  conda-forge/noarch::fonts-conda-forge-1-0\n",
            "  freetype           conda-forge/linux-64::freetype-2.12.1-hca18f0e_0\n",
            "  frozenlist         conda-forge/linux-64::frozenlist-1.3.1-py310h5764c6d_1\n",
            "  gettext            conda-forge/linux-64::gettext-0.21.1-h27087fc_0\n",
            "  glib               conda-forge/linux-64::glib-2.74.1-h6239696_0\n",
            "  glib-tools         conda-forge/linux-64::glib-tools-2.74.1-h6239696_0\n",
            "  google-auth        conda-forge/noarch::google-auth-2.13.0-pyh1a96a4e_0\n",
            "  google-colab       conda-forge/noarch::google-colab-1.0.0-pyh44b312d_0\n",
            "  gst-plugins-base   conda-forge/linux-64::gst-plugins-base-1.20.3-h57caac4_2\n",
            "  gstreamer          conda-forge/linux-64::gstreamer-1.20.3-hd4edc92_2\n",
            "  icu                conda-forge/linux-64::icu-70.1-h27087fc_0\n",
            "  idna               conda-forge/noarch::idna-3.4-pyhd8ed1ab_0\n",
            "  importlib-metadata conda-forge/noarch::importlib-metadata-5.0.0-pyha770c72_1\n",
            "  importlib_resourc~ conda-forge/noarch::importlib_resources-5.10.0-pyhd8ed1ab_0\n",
            "  ipykernel          conda-forge/noarch::ipykernel-6.16.2-pyh210e3f2_0\n",
            "  ipython            conda-forge/noarch::ipython-8.5.0-pyh41d4057_1\n",
            "  ipython_genutils   conda-forge/noarch::ipython_genutils-0.2.0-py_1\n",
            "  ipywidgets         conda-forge/noarch::ipywidgets-8.0.2-pyhd8ed1ab_1\n",
            "  jack               conda-forge/linux-64::jack-1.9.21-h2a1e645_0\n",
            "  jedi               conda-forge/noarch::jedi-0.18.1-pyhd8ed1ab_2\n",
            "  jinja2             conda-forge/noarch::jinja2-3.1.2-pyhd8ed1ab_1\n",
            "  jpeg               conda-forge/linux-64::jpeg-9e-h166bdaf_2\n",
            "  jsonschema         conda-forge/noarch::jsonschema-4.16.0-pyhd8ed1ab_0\n",
            "  jupyter            conda-forge/linux-64::jupyter-1.0.0-py310hff52083_7\n",
            "  jupyter_client     conda-forge/noarch::jupyter_client-7.4.4-pyhd8ed1ab_0\n",
            "  jupyter_console    conda-forge/noarch::jupyter_console-6.4.4-pyhd8ed1ab_0\n",
            "  jupyter_core       conda-forge/linux-64::jupyter_core-4.11.1-py310hff52083_1\n",
            "  jupyterlab_pygmen~ conda-forge/noarch::jupyterlab_pygments-0.2.2-pyhd8ed1ab_0\n",
            "  jupyterlab_widgets conda-forge/noarch::jupyterlab_widgets-3.0.3-pyhd8ed1ab_0\n",
            "  keyutils           conda-forge/linux-64::keyutils-1.6.1-h166bdaf_0\n",
            "  krb5               conda-forge/linux-64::krb5-1.19.3-h3790be6_0\n",
            "  lame               conda-forge/linux-64::lame-3.100-h166bdaf_1003\n",
            "  ld_impl_linux-64   conda-forge/linux-64::ld_impl_linux-64-2.39-hc81fddc_0\n",
            "  libblas            conda-forge/linux-64::libblas-3.9.0-16_linux64_openblas\n",
            "  libcap             conda-forge/linux-64::libcap-2.66-ha37c62d_0\n",
            "  libcblas           conda-forge/linux-64::libcblas-3.9.0-16_linux64_openblas\n",
            "  libclang           conda-forge/linux-64::libclang-14.0.6-default_h2e3cab8_0\n",
            "  libclang13         conda-forge/linux-64::libclang13-14.0.6-default_h3a83d3e_0\n",
            "  libcups            conda-forge/linux-64::libcups-2.3.3-h3e49a29_2\n",
            "  libdb              conda-forge/linux-64::libdb-6.2.32-h9c3ff4c_0\n",
            "  libedit            conda-forge/linux-64::libedit-3.1.20191231-he28a2e2_2\n",
            "  libevent           conda-forge/linux-64::libevent-2.1.10-h9b69904_4\n",
            "  libffi             conda-forge/linux-64::libffi-3.4.2-h7f98852_5\n",
            "  libflac            conda-forge/linux-64::libflac-1.4.2-h27087fc_0\n",
            "  libgcc-ng          conda-forge/linux-64::libgcc-ng-12.2.0-h65d4601_19\n",
            "  libgfortran-ng     conda-forge/linux-64::libgfortran-ng-12.2.0-h69a702a_19\n",
            "  libgfortran5       conda-forge/linux-64::libgfortran5-12.2.0-h337968e_19\n",
            "  libglib            conda-forge/linux-64::libglib-2.74.1-h7a41b64_0\n",
            "  libgomp            conda-forge/linux-64::libgomp-12.2.0-h65d4601_19\n",
            "  libiconv           conda-forge/linux-64::libiconv-1.17-h166bdaf_0\n",
            "  liblapack          conda-forge/linux-64::liblapack-3.9.0-16_linux64_openblas\n",
            "  libllvm14          conda-forge/linux-64::libllvm14-14.0.6-he0ac6c6_0\n",
            "  libnsl             conda-forge/linux-64::libnsl-2.0.0-h7f98852_0\n",
            "  libogg             conda-forge/linux-64::libogg-1.3.4-h7f98852_1\n",
            "  libopenblas        conda-forge/linux-64::libopenblas-0.3.21-pthreads_h78a6416_3\n",
            "  libopus            conda-forge/linux-64::libopus-1.3.1-h7f98852_1\n",
            "  libpng             conda-forge/linux-64::libpng-1.6.38-h753d276_0\n",
            "  libpq              conda-forge/linux-64::libpq-14.5-hd77ab85_1\n",
            "  libsndfile         conda-forge/linux-64::libsndfile-1.1.0-h27087fc_0\n",
            "  libsodium          conda-forge/linux-64::libsodium-1.0.18-h36c2ea0_1\n",
            "  libsqlite          conda-forge/linux-64::libsqlite-3.39.4-h753d276_0\n",
            "  libstdcxx-ng       conda-forge/linux-64::libstdcxx-ng-12.2.0-h46fd767_19\n",
            "  libtool            conda-forge/linux-64::libtool-2.4.6-h9c3ff4c_1008\n",
            "  libudev1           conda-forge/linux-64::libudev1-251-h166bdaf_0\n",
            "  libuuid            conda-forge/linux-64::libuuid-2.32.1-h7f98852_1000\n",
            "  libvorbis          conda-forge/linux-64::libvorbis-1.3.7-h9c3ff4c_0\n",
            "  libxcb             conda-forge/linux-64::libxcb-1.13-h7f98852_1004\n",
            "  libxkbcommon       conda-forge/linux-64::libxkbcommon-1.0.3-he3ba5ed_0\n",
            "  libxml2            conda-forge/linux-64::libxml2-2.10.3-h7463322_0\n",
            "  libzlib            conda-forge/linux-64::libzlib-1.2.13-h166bdaf_4\n",
            "  markupsafe         conda-forge/linux-64::markupsafe-2.1.1-py310h5764c6d_2\n",
            "  matplotlib-inline  conda-forge/noarch::matplotlib-inline-0.1.6-pyhd8ed1ab_0\n",
            "  mistune            conda-forge/noarch::mistune-2.0.4-pyhd8ed1ab_0\n",
            "  mpg123             conda-forge/linux-64::mpg123-1.30.2-h27087fc_1\n",
            "  multidict          conda-forge/linux-64::multidict-6.0.2-py310h5764c6d_2\n",
            "  mysql-common       conda-forge/linux-64::mysql-common-8.0.31-haf5c9bc_0\n",
            "  mysql-libs         conda-forge/linux-64::mysql-libs-8.0.31-h28c427c_0\n",
            "  nbclient           conda-forge/noarch::nbclient-0.7.0-pyhd8ed1ab_0\n",
            "  nbconvert          conda-forge/noarch::nbconvert-7.2.3-pyhd8ed1ab_0\n",
            "  nbconvert-core     conda-forge/noarch::nbconvert-core-7.2.3-pyhd8ed1ab_0\n",
            "  nbconvert-pandoc   conda-forge/noarch::nbconvert-pandoc-7.2.3-pyhd8ed1ab_0\n",
            "  nbformat           conda-forge/noarch::nbformat-5.7.0-pyhd8ed1ab_0\n",
            "  ncurses            conda-forge/linux-64::ncurses-6.3-h27087fc_1\n",
            "  nest-asyncio       conda-forge/noarch::nest-asyncio-1.5.6-pyhd8ed1ab_0\n",
            "  notebook           conda-forge/noarch::notebook-6.4.12-pyha770c72_0\n",
            "  nspr               conda-forge/linux-64::nspr-4.32-h9c3ff4c_1\n",
            "  nss                conda-forge/linux-64::nss-3.78-h2350873_0\n",
            "  numpy              conda-forge/linux-64::numpy-1.23.4-py310h53a5b5f_1\n",
            "  openssl            conda-forge/linux-64::openssl-1.1.1q-h166bdaf_1\n",
            "  packaging          conda-forge/noarch::packaging-21.3-pyhd8ed1ab_0\n",
            "  pandas             conda-forge/linux-64::pandas-1.5.1-py310h769672d_1\n",
            "  pandoc             conda-forge/linux-64::pandoc-2.19.2-h32600fe_1\n",
            "  pandocfilters      conda-forge/noarch::pandocfilters-1.5.0-pyhd8ed1ab_0\n",
            "  parso              conda-forge/noarch::parso-0.8.3-pyhd8ed1ab_0\n",
            "  pcre2              conda-forge/linux-64::pcre2-10.37-hc3806b6_1\n",
            "  pexpect            conda-forge/noarch::pexpect-4.8.0-pyh9f0ad1d_2\n",
            "  pickleshare        conda-forge/noarch::pickleshare-0.7.5-py_1003\n",
            "  pip                conda-forge/noarch::pip-22.3-pyhd8ed1ab_0\n",
            "  pkgutil-resolve-n~ conda-forge/noarch::pkgutil-resolve-name-1.3.10-pyhd8ed1ab_0\n",
            "  ply                conda-forge/noarch::ply-3.11-py_1\n",
            "  portpicker         conda-forge/noarch::portpicker-1.5.2-pyhd8ed1ab_0\n",
            "  prometheus_client  conda-forge/noarch::prometheus_client-0.15.0-pyhd8ed1ab_0\n",
            "  prompt-toolkit     conda-forge/noarch::prompt-toolkit-3.0.31-pyha770c72_0\n",
            "  prompt_toolkit     conda-forge/noarch::prompt_toolkit-3.0.31-hd8ed1ab_0\n",
            "  psutil             conda-forge/linux-64::psutil-5.9.3-py310h5764c6d_1\n",
            "  pthread-stubs      conda-forge/linux-64::pthread-stubs-0.4-h36c2ea0_1001\n",
            "  ptyprocess         conda-forge/noarch::ptyprocess-0.7.0-pyhd3deb0d_0\n",
            "  pulseaudio         conda-forge/linux-64::pulseaudio-14.0-habe0971_10\n",
            "  pure_eval          conda-forge/noarch::pure_eval-0.2.2-pyhd8ed1ab_0\n",
            "  pyasn1             conda-forge/noarch::pyasn1-0.4.8-py_0\n",
            "  pyasn1-modules     conda-forge/noarch::pyasn1-modules-0.2.7-py_0\n",
            "  pycosat            conda-forge/linux-64::pycosat-0.6.4-py310h5764c6d_1\n",
            "  pycparser          conda-forge/noarch::pycparser-2.21-pyhd8ed1ab_0\n",
            "  pygments           conda-forge/noarch::pygments-2.13.0-pyhd8ed1ab_0\n",
            "  pyopenssl          conda-forge/noarch::pyopenssl-22.1.0-pyhd8ed1ab_0\n",
            "  pyparsing          conda-forge/noarch::pyparsing-3.0.9-pyhd8ed1ab_0\n",
            "  pyqt               conda-forge/linux-64::pyqt-5.15.7-py310h29803b5_2\n",
            "  pyqt5-sip          conda-forge/linux-64::pyqt5-sip-12.11.0-py310hd8f1fbe_2\n",
            "  pyrsistent         conda-forge/linux-64::pyrsistent-0.18.1-py310h5764c6d_2\n",
            "  pysocks            conda-forge/noarch::pysocks-1.7.1-pyha2e5f31_6\n",
            "  python             conda-forge/linux-64::python-3.10.6-h582c2e5_0_cpython\n",
            "  python-dateutil    conda-forge/noarch::python-dateutil-2.8.2-pyhd8ed1ab_0\n",
            "  python-fastjsonsc~ conda-forge/noarch::python-fastjsonschema-2.16.2-pyhd8ed1ab_0\n",
            "  python_abi         conda-forge/linux-64::python_abi-3.10-2_cp310\n",
            "  pytz               conda-forge/noarch::pytz-2022.5-pyhd8ed1ab_0\n",
            "  pyu2f              conda-forge/noarch::pyu2f-0.1.5-pyhd8ed1ab_0\n",
            "  pyzmq              conda-forge/linux-64::pyzmq-24.0.1-py310h330234f_1\n",
            "  qt-main            conda-forge/linux-64::qt-main-5.15.6-hc525480_0\n",
            "  qtconsole          conda-forge/noarch::qtconsole-5.3.2-pyhd8ed1ab_0\n",
            "  qtconsole-base     conda-forge/noarch::qtconsole-base-5.3.2-pyha770c72_0\n",
            "  qtpy               conda-forge/noarch::qtpy-2.2.1-pyhd8ed1ab_0\n",
            "  readline           conda-forge/linux-64::readline-8.1.2-h0f457ee_0\n",
            "  requests           conda-forge/noarch::requests-2.28.1-pyhd8ed1ab_1\n",
            "  rsa                conda-forge/noarch::rsa-4.9-pyhd8ed1ab_0\n",
            "  ruamel_yaml        conda-forge/linux-64::ruamel_yaml-0.15.80-py310h5764c6d_1008\n",
            "  send2trash         conda-forge/noarch::send2trash-1.8.0-pyhd8ed1ab_0\n",
            "  setuptools         conda-forge/noarch::setuptools-65.5.0-pyhd8ed1ab_0\n",
            "  sip                conda-forge/linux-64::sip-6.7.2-py310hd8f1fbe_1\n",
            "  six                conda-forge/noarch::six-1.16.0-pyh6c4a22f_0\n",
            "  soupsieve          conda-forge/noarch::soupsieve-2.3.2.post1-pyhd8ed1ab_0\n",
            "  sqlite             conda-forge/linux-64::sqlite-3.39.4-h4ff8645_0\n",
            "  stack_data         conda-forge/noarch::stack_data-0.5.1-pyhd8ed1ab_0\n",
            "  terminado          conda-forge/noarch::terminado-0.17.0-pyh41d4057_0\n",
            "  tinycss2           conda-forge/noarch::tinycss2-1.2.1-pyhd8ed1ab_0\n",
            "  tk                 conda-forge/linux-64::tk-8.6.12-h27826a3_0\n",
            "  toml               conda-forge/noarch::toml-0.10.2-pyhd8ed1ab_0\n",
            "  toolz              conda-forge/noarch::toolz-0.12.0-pyhd8ed1ab_0\n",
            "  tornado            conda-forge/linux-64::tornado-6.2-py310h5764c6d_1\n",
            "  tqdm               conda-forge/noarch::tqdm-4.64.1-pyhd8ed1ab_0\n",
            "  traitlets          conda-forge/noarch::traitlets-5.5.0-pyhd8ed1ab_0\n",
            "  typing-extensions  conda-forge/noarch::typing-extensions-4.4.0-hd8ed1ab_0\n",
            "  typing_extensions  conda-forge/noarch::typing_extensions-4.4.0-pyha770c72_0\n",
            "  tzdata             conda-forge/noarch::tzdata-2022e-h191b570_0\n",
            "  urllib3            conda-forge/noarch::urllib3-1.26.11-pyhd8ed1ab_0\n",
            "  wcwidth            conda-forge/noarch::wcwidth-0.2.5-pyh9f0ad1d_2\n",
            "  webencodings       conda-forge/noarch::webencodings-0.5.1-py_1\n",
            "  wheel              conda-forge/noarch::wheel-0.37.1-pyhd8ed1ab_0\n",
            "  widgetsnbextension conda-forge/noarch::widgetsnbextension-4.0.3-pyhd8ed1ab_0\n",
            "  xcb-util           conda-forge/linux-64::xcb-util-0.4.0-h166bdaf_0\n",
            "  xcb-util-image     conda-forge/linux-64::xcb-util-image-0.4.0-h166bdaf_0\n",
            "  xcb-util-keysyms   conda-forge/linux-64::xcb-util-keysyms-0.4.0-h166bdaf_0\n",
            "  xcb-util-renderut~ conda-forge/linux-64::xcb-util-renderutil-0.3.9-h166bdaf_0\n",
            "  xcb-util-wm        conda-forge/linux-64::xcb-util-wm-0.4.1-h166bdaf_0\n",
            "  xorg-libxau        conda-forge/linux-64::xorg-libxau-1.0.9-h7f98852_0\n",
            "  xorg-libxdmcp      conda-forge/linux-64::xorg-libxdmcp-1.1.3-h7f98852_0\n",
            "  xz                 conda-forge/linux-64::xz-5.2.6-h166bdaf_0\n",
            "  yaml               conda-forge/linux-64::yaml-0.2.5-h7f98852_2\n",
            "  yarl               conda-forge/linux-64::yarl-1.8.1-py310h5764c6d_0\n",
            "  zeromq             conda-forge/linux-64::zeromq-4.3.4-h9c3ff4c_1\n",
            "  zipp               conda-forge/noarch::zipp-3.10.0-pyhd8ed1ab_0\n",
            "  zstd               conda-forge/linux-64::zstd-1.5.2-h6239696_4\n",
            "\n",
            "\n",
            "Preparing transaction: \\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\bdone\n",
            "Executing transaction: | \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\b/ \b\b- \b\b\\ \b\b| \b\bdone\n",
            "installation finished.\n",
            "WARNING:\n",
            "    You currently have a PYTHONPATH environment variable set. This may cause\n",
            "    unexpected behavior when running the Python interpreter in Colab.\n",
            "    For best results, please verify that your PYTHONPATH only points to\n",
            "    directories of packages that are compatible with the Python interpreter\n",
            "    in Colab: /usr/local\n",
            "Installed kernelspec py310 in /root/.local/share/jupyter/kernels/py310\n"
          ]
        }
      ],
      "source": [
        "!wget https://github.com/korakot/kora/releases/download/v0.10/py310.sh\n",
        "!bash ./py310.sh -b -f -p /usr/local\n",
        "!python -m ipykernel install --name \"py310\" --user"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python --version; pip --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3BHP6TZK_LD3",
        "outputId": "f4f8b174-4d82-4f25-8bb8-b5b5e653d0e1"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python 3.10.6\n",
            "pip 22.3 from /usr/local/lib/python3.10/site-packages/pip (python 3.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone --recurse-submodules https://github.com/AnLimt/vall-e_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2KusNev0_Mef",
        "outputId": "8883137d-4933-4741-dfb5-cb9ff0341ada"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'vall-e_test'...\n",
            "remote: Enumerating objects: 407, done.\u001b[K\n",
            "remote: Counting objects: 100% (71/71), done.\u001b[K\n",
            "remote: Compressing objects: 100% (30/30), done.\u001b[K\n",
            "remote: Total 407 (delta 45), reused 41 (delta 41), pack-reused 336\u001b[K\n",
            "Receiving objects: 100% (407/407), 1.94 MiB | 3.65 MiB/s, done.\n",
            "Resolving deltas: 100% (205/205), done.\n",
            "Submodule 'mini_vall_e/utils' (https://github.com/enhuiz/pytorch-training-utils.git) registered for path 'vall_e/utils'\n",
            "Cloning into '/content/vall-e_test/vall_e/utils'...\n",
            "remote: Enumerating objects: 331, done.        \n",
            "remote: Counting objects: 100% (36/36), done.        \n",
            "remote: Compressing objects: 100% (12/12), done.        \n",
            "remote: Total 331 (delta 29), reused 25 (delta 24), pack-reused 295        \n",
            "Receiving objects: 100% (331/331), 58.42 KiB | 14.60 MiB/s, done.\n",
            "Resolving deltas: 100% (207/207), done.\n",
            "Submodule path 'vall_e/utils': checked out '603cb4c4840a5787c6803e9595badba6b98e93ac'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd vall-e"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hEsvTQZ3_N2p",
        "outputId": "09d70e2b-7cac-45f2-f9e1-6c86de290c1e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/vall-e\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install ."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "TMOeLp1C_Oxa",
        "outputId": "bf0ea604-5473-4af6-a5c1-ef5c1a0ea61a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Processing /content/vall-e\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting coloredlogs>=15.0.1\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting deepspeed>=0.7.7\n",
            "  Downloading deepspeed-0.8.0.tar.gz (749 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m749.9/749.9 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting diskcache>=5.4.0\n",
            "  Downloading diskcache-5.4.0-py3-none-any.whl (44 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.0/45.0 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting einops>=0.6.0\n",
            "  Downloading einops-0.6.0-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting encodec>=0.1.1\n",
            "  Downloading encodec-0.1.1.tar.gz (3.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.7/3.7 MB\u001b[0m \u001b[31m79.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting g2p_en>=2.1.0\n",
            "  Downloading g2p_en-2.1.0-py3-none-any.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m111.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting humanize>=4.4.0\n",
            "  Downloading humanize-4.4.0-py3-none-any.whl (106 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.6/106.6 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting matplotlib>=3.6.0\n",
            "  Downloading matplotlib-3.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.8/11.8 MB\u001b[0m \u001b[31m58.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.23.3 in /usr/local/lib/python3.10/site-packages (from vall-e==0.0.1.dev20230119012039) (1.23.4)\n",
            "Collecting omegaconf>=2.2.3\n",
            "  Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting openTSNE>=0.6.2\n",
            "  Downloading openTSNE-0.6.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m25.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas>=1.5.0 in /usr/local/lib/python3.10/site-packages (from vall-e==0.0.1.dev20230119012039) (1.5.1)\n",
            "Collecting soundfile>=0.11.0\n",
            "  Downloading soundfile-0.11.0-py2.py3-none-any.whl (23 kB)\n",
            "Collecting torch>=1.13.0\n",
            "  Downloading torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl (887.5 MB)\n",
            "\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m887.4/887.5 MB\u001b[0m \u001b[31m150.1 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0mtcmalloc: large alloc 1109319680 bytes == 0x55ec9d612000 @  0x7fc1e2841680 0x7fc1e2861da2 0x55ec657667db 0x55ec6583d978 0x55ec657ca0f7 0x55ec657c9efb 0x55ec6574b3d1 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec65730107 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6573f86f 0x55ec6572f4ee 0x55ec6574cdf2 0x55ec657e39e4 0x55ec65819503 0x55ec657337d9 0x55ec6574cb6c\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m887.5/887.5 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting torchaudio>=0.13.0\n",
            "  Downloading torchaudio-0.13.1-cp310-cp310-manylinux1_x86_64.whl (4.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.2/4.2 MB\u001b[0m \u001b[31m115.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.64.1 in /usr/local/lib/python3.10/site-packages (from vall-e==0.0.1.dev20230119012039) (4.64.1)\n",
            "Collecting humanfriendly>=9.1\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hjson\n",
            "  Downloading hjson-3.1.0-py3-none-any.whl (54 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.0/54.0 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting ninja\n",
            "  Downloading ninja-1.11.1-py2.py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (145 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m146.0/146.0 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/site-packages (from deepspeed>=0.7.7->vall-e==0.0.1.dev20230119012039) (21.3)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/site-packages (from deepspeed>=0.7.7->vall-e==0.0.1.dev20230119012039) (5.9.3)\n",
            "Collecting py-cpuinfo\n",
            "  Downloading py_cpuinfo-9.0.0-py3-none-any.whl (22 kB)\n",
            "Collecting pydantic\n",
            "  Downloading pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m87.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting distance>=0.1.3\n",
            "  Downloading Distance-0.1.3.tar.gz (180 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m180.3/180.3 kB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting nltk>=3.2.4\n",
            "  Downloading nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m77.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting inflect>=0.3.1\n",
            "  Downloading inflect-6.0.2-py3-none-any.whl (34 kB)\n",
            "Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3.6.0->vall-e==0.0.1.dev20230119012039) (3.0.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/site-packages (from matplotlib>=3.6.0->vall-e==0.0.1.dev20230119012039) (2.8.2)\n",
            "Collecting kiwisolver>=1.0.1\n",
            "  Downloading kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m83.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting contourpy>=1.0.1\n",
            "  Downloading contourpy-1.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (300 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.3/300.3 kB\u001b[0m \u001b[31m36.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pillow>=6.2.0\n",
            "  Downloading Pillow-9.4.0-cp310-cp310-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m92.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting cycler>=0.10\n",
            "  Downloading cycler-0.11.0-py3-none-any.whl (6.4 kB)\n",
            "Collecting fonttools>=4.22.0\n",
            "  Downloading fonttools-4.38.0-py3-none-any.whl (965 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m965.4/965.4 kB\u001b[0m \u001b[31m68.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting antlr4-python3-runtime==4.9.*\n",
            "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting PyYAML>=5.1.0\n",
            "  Downloading PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (682 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m682.2/682.2 kB\u001b[0m \u001b[31m61.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scikit-learn>=0.20\n",
            "  Downloading scikit_learn-1.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (9.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.5/9.5 MB\u001b[0m \u001b[31m121.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting scipy\n",
            "  Downloading scipy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (34.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m34.4/34.4 MB\u001b[0m \u001b[31m19.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/site-packages (from pandas>=1.5.0->vall-e==0.0.1.dev20230119012039) (2022.5)\n",
            "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/site-packages (from soundfile>=0.11.0->vall-e==0.0.1.dev20230119012039) (1.15.1)\n",
            "Collecting nvidia-cuda-nvrtc-cu11==11.7.99\n",
            "  Downloading nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m86.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu11==11.7.99\n",
            "  Downloading nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m60.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.10/site-packages (from torch>=1.13.0->vall-e==0.0.1.dev20230119012039) (4.4.0)\n",
            "Collecting nvidia-cudnn-cu11==8.5.0.96\n",
            "  Downloading nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu11==11.10.3.66\n",
            "  Downloading nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wheel in /usr/local/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.13.0->vall-e==0.0.1.dev20230119012039) (0.37.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.13.0->vall-e==0.0.1.dev20230119012039) (65.5.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/site-packages (from cffi>=1.0->soundfile>=0.11.0->vall-e==0.0.1.dev20230119012039) (2.21)\n",
            "Collecting click\n",
            "  Downloading click-8.1.3-py3-none-any.whl (96 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.6/96.6 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting joblib\n",
            "  Downloading joblib-1.2.0-py3-none-any.whl (297 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting regex>=2021.8.3\n",
            "  Downloading regex-2022.10.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (770 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m770.5/770.5 kB\u001b[0m \u001b[31m61.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib>=3.6.0->vall-e==0.0.1.dev20230119012039) (1.16.0)\n",
            "Collecting threadpoolctl>=2.0.0\n",
            "  Downloading threadpoolctl-3.1.0-py3-none-any.whl (14 kB)\n",
            "Building wheels for collected packages: vall-e, deepspeed, encodec, antlr4-python3-runtime, distance\n",
            "  Building wheel for vall-e (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for vall-e: filename=vall_e-0.0.1.dev20230119012039-py3-none-any.whl size=30768 sha256=5ed3d59139980ef1fb78f4e59c58d22c2d26bcdc5e8ea2632ab1b1bde3df57c6\n",
            "  Stored in directory: /root/.cache/pip/wheels/c0/e2/f3/52410740625c97ef2dc7209314c9039d0cf700c40a373f6c16\n",
            "  Building wheel for deepspeed (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for deepspeed: filename=deepspeed-0.8.0-py3-none-any.whl size=752094 sha256=0183e9b69aa458fdc7c708de9d4200a8d28e1cd7285768d60bf21ef49ac4e1cf\n",
            "  Stored in directory: /root/.cache/pip/wheels/b9/6c/07/8891d25c67f5dff8fb8739ddcf5a38697fd329edfdcd6aa246\n",
            "  Building wheel for encodec (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for encodec: filename=encodec-0.1.1-py3-none-any.whl size=45761 sha256=055e1096204355be98a45fe10df3a638a8175cd4ad303172c2e312038ae465f8\n",
            "  Stored in directory: /root/.cache/pip/wheels/ce/7b/9e/791357a98076b1d06f7113db0cb70a217c3feb31962366ab9d\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144554 sha256=5c78eba821070e5d9de94a61a133eddbd83aa92e77faf79494bc8a60a197d1c6\n",
            "  Stored in directory: /root/.cache/pip/wheels/48/6a/c2/acb58c7afdf57e4cddf5e1513f5a2d62aa8e98f82a00c76d7c\n",
            "  Building wheel for distance (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for distance: filename=Distance-0.1.3-py3-none-any.whl size=16257 sha256=7965a6632857c50ea1f4c7ea1ae92f950542df6f45caf8915e8ed06dcb435347\n",
            "  Stored in directory: /root/.cache/pip/wheels/6a/41/b0/39f403bd1fb459600f3ced3f74e8e88ffd5a409584a043785a\n",
            "Successfully built vall-e deepspeed encodec antlr4-python3-runtime distance\n",
            "Installing collected packages: py-cpuinfo, ninja, hjson, distance, antlr4-python3-runtime, threadpoolctl, scipy, regex, PyYAML, pydantic, pillow, nvidia-cuda-runtime-cu11, nvidia-cuda-nvrtc-cu11, nvidia-cublas-cu11, kiwisolver, joblib, humanize, humanfriendly, fonttools, einops, diskcache, cycler, contourpy, click, soundfile, scikit-learn, omegaconf, nvidia-cudnn-cu11, nltk, matplotlib, inflect, coloredlogs, torch, openTSNE, g2p_en, torchaudio, deepspeed, encodec, vall-e\n",
            "Successfully installed PyYAML-6.0 antlr4-python3-runtime-4.9.3 click-8.1.3 coloredlogs-15.0.1 contourpy-1.0.7 cycler-0.11.0 deepspeed-0.8.0 diskcache-5.4.0 distance-0.1.3 einops-0.6.0 encodec-0.1.1 fonttools-4.38.0 g2p_en-2.1.0 hjson-3.1.0 humanfriendly-10.0 humanize-4.4.0 inflect-6.0.2 joblib-1.2.0 kiwisolver-1.4.4 matplotlib-3.6.3 ninja-1.11.1 nltk-3.8.1 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 omegaconf-2.3.0 openTSNE-0.6.2 pillow-9.4.0 py-cpuinfo-9.0.0 pydantic-1.10.4 regex-2022.10.31 scikit-learn-1.2.0 scipy-1.10.0 soundfile-0.11.0 threadpoolctl-3.1.0 torch-1.13.1 torchaudio-0.13.1 vall-e-0.0.1.dev20230119012039\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "cycler",
                  "kiwisolver"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m vall_e.emb.qnt data/test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6r9kIKXd_PeQ",
        "outputId": "fe2d096c-994c-42e4-ca15-989d4596065d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r  0% 0/1 [00:00<?, ?it/s]\r100% 1/1 [00:00<00:00, 9799.78it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m vall_e.emb.g2p data/test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A_nfRS25_TAk",
        "outputId": "4db022e2-0111-4a21-ac68-0df1accd9e19"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/cmudict.zip.\n",
            "100% 1/1 [00:00<00:00, 12372.58it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m vall_e.train yaml=config/test/ar.yml"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SbWtNBVg_Tfd",
        "outputId": "f8a95cd2-76d0-41bd-e867-9d6cbcec87d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1;30;43mStreaming output truncated to the last 5000 lines.\u001b[0m\n",
            "  \"model.grad_norm\": 0.6686407327651978,\n",
            "  \"model.elapsed_time\": 0.07606005668640137,\n",
            "  \"model.engine_step\": 626,\n",
            "  \"model.loss.nll\": 0.01398468017578125,\n",
            "  \"elapsed_time\": 0.07606005668640137,\n",
            "  \"wall_time\": 1674064267.800759,\n",
            "  \"global_step\": 626\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.024169921875,\n",
            "  \"model.lr\": 0.000125375,\n",
            "  \"model.grad_norm\": 1.4665035009384155,\n",
            "  \"model.elapsed_time\": 0.06312704086303711,\n",
            "  \"model.engine_step\": 627,\n",
            "  \"model.loss.nll\": 0.024169921875,\n",
            "  \"elapsed_time\": 0.06312704086303711,\n",
            "  \"wall_time\": 1674064267.872081,\n",
            "  \"global_step\": 627\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002391815185546875,\n",
            "  \"model.lr\": 0.00012557400000000001,\n",
            "  \"model.grad_norm\": 0.10632999986410141,\n",
            "  \"model.elapsed_time\": 0.0705254077911377,\n",
            "  \"model.engine_step\": 628,\n",
            "  \"model.loss.nll\": 0.002391815185546875,\n",
            "  \"elapsed_time\": 0.0705254077911377,\n",
            "  \"wall_time\": 1674064267.9457426,\n",
            "  \"global_step\": 628\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005985260009765625,\n",
            "  \"model.lr\": 0.000125773,\n",
            "  \"model.grad_norm\": 0.6287567615509033,\n",
            "  \"model.elapsed_time\": 0.0719759464263916,\n",
            "  \"model.engine_step\": 629,\n",
            "  \"model.loss.nll\": 0.005985260009765625,\n",
            "  \"elapsed_time\": 0.0719759464263916,\n",
            "  \"wall_time\": 1674064268.0281825,\n",
            "  \"global_step\": 629\n",
            "}\n",
            "[2023-01-18 17:51:08,112] [INFO] [logging.py:68:log_dist] [Rank 0] step=630, skipped=1, lr=[0.000125972], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:08,113] [INFO] [timer.py:196:stop] epoch=0/micro_step=630/global_step=630, RunningAvgSamplesPerSec=13.465264953979611, CurrSamplesPerSec=11.950401167031364, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.016204833984375,\n",
            "  \"model.lr\": 0.000125972,\n",
            "  \"model.grad_norm\": 1.8782715797424316,\n",
            "  \"model.elapsed_time\": 0.08415484428405762,\n",
            "  \"model.engine_step\": 630,\n",
            "  \"model.loss.nll\": 0.016204833984375,\n",
            "  \"elapsed_time\": 0.08415484428405762,\n",
            "  \"wall_time\": 1674064268.1150744,\n",
            "  \"global_step\": 630\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.015411376953125,\n",
            "  \"model.lr\": 0.000126171,\n",
            "  \"model.grad_norm\": 1.3377455472946167,\n",
            "  \"model.elapsed_time\": 0.06925845146179199,\n",
            "  \"model.engine_step\": 631,\n",
            "  \"model.loss.nll\": 0.015411376953125,\n",
            "  \"elapsed_time\": 0.06925845146179199,\n",
            "  \"wall_time\": 1674064268.1913626,\n",
            "  \"global_step\": 631\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01006317138671875,\n",
            "  \"model.lr\": 0.00012637,\n",
            "  \"model.grad_norm\": 0.6395008563995361,\n",
            "  \"model.elapsed_time\": 0.0704655647277832,\n",
            "  \"model.engine_step\": 632,\n",
            "  \"model.loss.nll\": 0.01006317138671875,\n",
            "  \"elapsed_time\": 0.0704655647277832,\n",
            "  \"wall_time\": 1674064268.263985,\n",
            "  \"global_step\": 632\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0023899078369140625,\n",
            "  \"model.lr\": 0.00012656900000000001,\n",
            "  \"model.grad_norm\": 0.10547526180744171,\n",
            "  \"model.elapsed_time\": 0.08472180366516113,\n",
            "  \"model.engine_step\": 633,\n",
            "  \"model.loss.nll\": 0.0023899078369140625,\n",
            "  \"elapsed_time\": 0.08472180366516113,\n",
            "  \"wall_time\": 1674064268.357052,\n",
            "  \"global_step\": 633\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0191497802734375,\n",
            "  \"model.lr\": 0.000126768,\n",
            "  \"model.grad_norm\": 1.1944607496261597,\n",
            "  \"model.elapsed_time\": 0.06401395797729492,\n",
            "  \"model.engine_step\": 634,\n",
            "  \"model.loss.nll\": 0.0191497802734375,\n",
            "  \"elapsed_time\": 0.06401395797729492,\n",
            "  \"wall_time\": 1674064268.423336,\n",
            "  \"global_step\": 634\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0113677978515625,\n",
            "  \"model.lr\": 0.00012696700000000002,\n",
            "  \"model.grad_norm\": 1.1958808898925781,\n",
            "  \"model.elapsed_time\": 0.06681036949157715,\n",
            "  \"model.engine_step\": 635,\n",
            "  \"model.loss.nll\": 0.0113677978515625,\n",
            "  \"elapsed_time\": 0.06681036949157715,\n",
            "  \"wall_time\": 1674064268.498207,\n",
            "  \"global_step\": 635\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01885986328125,\n",
            "  \"model.lr\": 0.000127166,\n",
            "  \"model.grad_norm\": 1.5206396579742432,\n",
            "  \"model.elapsed_time\": 0.07129502296447754,\n",
            "  \"model.engine_step\": 636,\n",
            "  \"model.loss.nll\": 0.01885986328125,\n",
            "  \"elapsed_time\": 0.07129502296447754,\n",
            "  \"wall_time\": 1674064268.5714104,\n",
            "  \"global_step\": 636\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00997161865234375,\n",
            "  \"model.lr\": 0.000127365,\n",
            "  \"model.grad_norm\": 0.4261948764324188,\n",
            "  \"model.elapsed_time\": 0.07146906852722168,\n",
            "  \"model.engine_step\": 637,\n",
            "  \"model.loss.nll\": 0.00997161865234375,\n",
            "  \"elapsed_time\": 0.07146906852722168,\n",
            "  \"wall_time\": 1674064268.6558902,\n",
            "  \"global_step\": 637\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.031707763671875,\n",
            "  \"model.lr\": 0.00012756400000000001,\n",
            "  \"model.grad_norm\": 1.5593745708465576,\n",
            "  \"model.elapsed_time\": 0.06942558288574219,\n",
            "  \"model.engine_step\": 638,\n",
            "  \"model.loss.nll\": 0.031707763671875,\n",
            "  \"elapsed_time\": 0.06942558288574219,\n",
            "  \"wall_time\": 1674064268.7287722,\n",
            "  \"global_step\": 638\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01439666748046875,\n",
            "  \"model.lr\": 0.000127763,\n",
            "  \"model.grad_norm\": 0.9208853244781494,\n",
            "  \"model.elapsed_time\": 0.07153058052062988,\n",
            "  \"model.engine_step\": 639,\n",
            "  \"model.loss.nll\": 0.01439666748046875,\n",
            "  \"elapsed_time\": 0.07153058052062988,\n",
            "  \"wall_time\": 1674064268.8074877,\n",
            "  \"global_step\": 639\n",
            "}\n",
            "[2023-01-18 17:51:08,875] [INFO] [logging.py:68:log_dist] [Rank 0] step=640, skipped=1, lr=[0.00012796200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:08,876] [INFO] [timer.py:196:stop] epoch=0/micro_step=640/global_step=640, RunningAvgSamplesPerSec=13.475924420897513, CurrSamplesPerSec=14.903330798695254, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0016241073608398438,\n",
            "  \"model.lr\": 0.00012796200000000002,\n",
            "  \"model.grad_norm\": 0.03098682500422001,\n",
            "  \"model.elapsed_time\": 0.06805157661437988,\n",
            "  \"model.engine_step\": 640,\n",
            "  \"model.loss.nll\": 0.0016241073608398438,\n",
            "  \"elapsed_time\": 0.06805157661437988,\n",
            "  \"wall_time\": 1674064268.879614,\n",
            "  \"global_step\": 640\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.027313232421875,\n",
            "  \"model.lr\": 0.000128161,\n",
            "  \"model.grad_norm\": 2.1884028911590576,\n",
            "  \"model.elapsed_time\": 0.07195687294006348,\n",
            "  \"model.engine_step\": 641,\n",
            "  \"model.loss.nll\": 0.027313232421875,\n",
            "  \"elapsed_time\": 0.07195687294006348,\n",
            "  \"wall_time\": 1674064268.9644866,\n",
            "  \"global_step\": 641\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0020656585693359375,\n",
            "  \"model.lr\": 0.00012836,\n",
            "  \"model.grad_norm\": 0.0837344229221344,\n",
            "  \"model.elapsed_time\": 0.06897187232971191,\n",
            "  \"model.engine_step\": 642,\n",
            "  \"model.loss.nll\": 0.0020656585693359375,\n",
            "  \"elapsed_time\": 0.06897187232971191,\n",
            "  \"wall_time\": 1674064269.035666,\n",
            "  \"global_step\": 642\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0027751922607421875,\n",
            "  \"model.lr\": 0.00012855900000000001,\n",
            "  \"model.grad_norm\": 0.11991465091705322,\n",
            "  \"model.elapsed_time\": 0.07442235946655273,\n",
            "  \"model.engine_step\": 643,\n",
            "  \"model.loss.nll\": 0.0027751922607421875,\n",
            "  \"elapsed_time\": 0.07442235946655273,\n",
            "  \"wall_time\": 1674064269.1180182,\n",
            "  \"global_step\": 643\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005382537841796875,\n",
            "  \"model.lr\": 0.000128758,\n",
            "  \"model.grad_norm\": 0.6351223587989807,\n",
            "  \"model.elapsed_time\": 0.06632113456726074,\n",
            "  \"model.engine_step\": 644,\n",
            "  \"model.loss.nll\": 0.005382537841796875,\n",
            "  \"elapsed_time\": 0.06632113456726074,\n",
            "  \"wall_time\": 1674064269.1870081,\n",
            "  \"global_step\": 644\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.003932952880859375,\n",
            "  \"model.lr\": 0.00012895700000000002,\n",
            "  \"model.grad_norm\": 0.31543174386024475,\n",
            "  \"model.elapsed_time\": 0.06653690338134766,\n",
            "  \"model.engine_step\": 645,\n",
            "  \"model.loss.nll\": 0.003932952880859375,\n",
            "  \"elapsed_time\": 0.06653690338134766,\n",
            "  \"wall_time\": 1674064269.2607903,\n",
            "  \"global_step\": 645\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01427459716796875,\n",
            "  \"model.lr\": 0.000129156,\n",
            "  \"model.grad_norm\": 0.7257593870162964,\n",
            "  \"model.elapsed_time\": 0.06513643264770508,\n",
            "  \"model.engine_step\": 646,\n",
            "  \"model.loss.nll\": 0.01427459716796875,\n",
            "  \"elapsed_time\": 0.06513643264770508,\n",
            "  \"wall_time\": 1674064269.3278768,\n",
            "  \"global_step\": 646\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.034393310546875,\n",
            "  \"model.lr\": 0.000129355,\n",
            "  \"model.grad_norm\": 1.836539387702942,\n",
            "  \"model.elapsed_time\": 0.07412242889404297,\n",
            "  \"model.engine_step\": 647,\n",
            "  \"model.loss.nll\": 0.034393310546875,\n",
            "  \"elapsed_time\": 0.07412242889404297,\n",
            "  \"wall_time\": 1674064269.4091756,\n",
            "  \"global_step\": 647\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00788116455078125,\n",
            "  \"model.lr\": 0.000129554,\n",
            "  \"model.grad_norm\": 0.9461675882339478,\n",
            "  \"model.elapsed_time\": 0.0677499771118164,\n",
            "  \"model.engine_step\": 648,\n",
            "  \"model.loss.nll\": 0.00788116455078125,\n",
            "  \"elapsed_time\": 0.0677499771118164,\n",
            "  \"wall_time\": 1674064269.4793925,\n",
            "  \"global_step\": 648\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.040069580078125,\n",
            "  \"model.lr\": 0.000129753,\n",
            "  \"model.grad_norm\": 2.297820806503296,\n",
            "  \"model.elapsed_time\": 0.06391286849975586,\n",
            "  \"model.engine_step\": 649,\n",
            "  \"model.loss.nll\": 0.040069580078125,\n",
            "  \"elapsed_time\": 0.06391286849975586,\n",
            "  \"wall_time\": 1674064269.550972,\n",
            "  \"global_step\": 649\n",
            "}\n",
            "[2023-01-18 17:51:09,621] [INFO] [logging.py:68:log_dist] [Rank 0] step=650, skipped=1, lr=[0.00012995200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:09,623] [INFO] [timer.py:196:stop] epoch=0/micro_step=650/global_step=650, RunningAvgSamplesPerSec=13.490702896899347, CurrSamplesPerSec=14.079144973800725, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0015201568603515625,\n",
            "  \"model.lr\": 0.00012995200000000002,\n",
            "  \"model.grad_norm\": 0.0332970954477787,\n",
            "  \"model.elapsed_time\": 0.07147622108459473,\n",
            "  \"model.engine_step\": 650,\n",
            "  \"model.loss.nll\": 0.0015201568603515625,\n",
            "  \"elapsed_time\": 0.07147622108459473,\n",
            "  \"wall_time\": 1674064269.6244483,\n",
            "  \"global_step\": 650\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013742446899414062,\n",
            "  \"model.lr\": 0.000130151,\n",
            "  \"model.grad_norm\": 0.026595555245876312,\n",
            "  \"model.elapsed_time\": 0.07736396789550781,\n",
            "  \"model.engine_step\": 651,\n",
            "  \"model.loss.nll\": 0.0013742446899414062,\n",
            "  \"elapsed_time\": 0.07736396789550781,\n",
            "  \"wall_time\": 1674064269.7097251,\n",
            "  \"global_step\": 651\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002288818359375,\n",
            "  \"model.lr\": 0.00013035,\n",
            "  \"model.grad_norm\": 0.123663991689682,\n",
            "  \"model.elapsed_time\": 0.0717320442199707,\n",
            "  \"model.engine_step\": 652,\n",
            "  \"model.loss.nll\": 0.002288818359375,\n",
            "  \"elapsed_time\": 0.0717320442199707,\n",
            "  \"wall_time\": 1674064269.7839723,\n",
            "  \"global_step\": 652\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06689453125,\n",
            "  \"model.lr\": 0.000130549,\n",
            "  \"model.grad_norm\": 2.442143440246582,\n",
            "  \"model.elapsed_time\": 0.07742929458618164,\n",
            "  \"model.engine_step\": 653,\n",
            "  \"model.loss.nll\": 0.06689453125,\n",
            "  \"elapsed_time\": 0.07742929458618164,\n",
            "  \"wall_time\": 1674064269.8690145,\n",
            "  \"global_step\": 653\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0266571044921875,\n",
            "  \"model.lr\": 0.000130748,\n",
            "  \"model.grad_norm\": 2.049468517303467,\n",
            "  \"model.elapsed_time\": 0.06497383117675781,\n",
            "  \"model.engine_step\": 654,\n",
            "  \"model.loss.nll\": 0.0266571044921875,\n",
            "  \"elapsed_time\": 0.06497383117675781,\n",
            "  \"wall_time\": 1674064269.9364083,\n",
            "  \"global_step\": 654\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00421905517578125,\n",
            "  \"model.lr\": 0.00013094700000000002,\n",
            "  \"model.grad_norm\": 0.4519415497779846,\n",
            "  \"model.elapsed_time\": 0.07776498794555664,\n",
            "  \"model.engine_step\": 655,\n",
            "  \"model.loss.nll\": 0.00421905517578125,\n",
            "  \"elapsed_time\": 0.07776498794555664,\n",
            "  \"wall_time\": 1674064270.0216136,\n",
            "  \"global_step\": 655\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0018339157104492188,\n",
            "  \"model.lr\": 0.000131146,\n",
            "  \"model.grad_norm\": 0.05764399841427803,\n",
            "  \"model.elapsed_time\": 0.07079362869262695,\n",
            "  \"model.engine_step\": 656,\n",
            "  \"model.loss.nll\": 0.0018339157104492188,\n",
            "  \"elapsed_time\": 0.07079362869262695,\n",
            "  \"wall_time\": 1674064270.095838,\n",
            "  \"global_step\": 656\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0024394989013671875,\n",
            "  \"model.lr\": 0.00013134500000000002,\n",
            "  \"model.grad_norm\": 0.19220390915870667,\n",
            "  \"model.elapsed_time\": 0.07561039924621582,\n",
            "  \"model.engine_step\": 657,\n",
            "  \"model.loss.nll\": 0.0024394989013671875,\n",
            "  \"elapsed_time\": 0.07561039924621582,\n",
            "  \"wall_time\": 1674064270.18258,\n",
            "  \"global_step\": 657\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002384185791015625,\n",
            "  \"model.lr\": 0.000131544,\n",
            "  \"model.grad_norm\": 0.1013735756278038,\n",
            "  \"model.elapsed_time\": 0.0674431324005127,\n",
            "  \"model.engine_step\": 658,\n",
            "  \"model.loss.nll\": 0.002384185791015625,\n",
            "  \"elapsed_time\": 0.0674431324005127,\n",
            "  \"wall_time\": 1674064270.252852,\n",
            "  \"global_step\": 658\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01131439208984375,\n",
            "  \"model.lr\": 0.000131743,\n",
            "  \"model.grad_norm\": 1.229967474937439,\n",
            "  \"model.elapsed_time\": 0.06769084930419922,\n",
            "  \"model.engine_step\": 659,\n",
            "  \"model.loss.nll\": 0.01131439208984375,\n",
            "  \"elapsed_time\": 0.06769084930419922,\n",
            "  \"wall_time\": 1674064270.3286057,\n",
            "  \"global_step\": 659\n",
            "}\n",
            "[2023-01-18 17:51:10,399] [INFO] [logging.py:68:log_dist] [Rank 0] step=660, skipped=1, lr=[0.00013194200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:10,400] [INFO] [timer.py:196:stop] epoch=0/micro_step=660/global_step=660, RunningAvgSamplesPerSec=13.496763080175956, CurrSamplesPerSec=14.3885448863305, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.007015228271484375,\n",
            "  \"model.lr\": 0.00013194200000000002,\n",
            "  \"model.grad_norm\": 0.7661277055740356,\n",
            "  \"model.elapsed_time\": 0.0700836181640625,\n",
            "  \"model.engine_step\": 660,\n",
            "  \"model.loss.nll\": 0.007015228271484375,\n",
            "  \"elapsed_time\": 0.0700836181640625,\n",
            "  \"wall_time\": 1674064270.4016027,\n",
            "  \"global_step\": 660\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.220947265625,\n",
            "  \"model.lr\": 0.000132141,\n",
            "  \"model.grad_norm\": 6.893001079559326,\n",
            "  \"model.elapsed_time\": 0.06570172309875488,\n",
            "  \"model.engine_step\": 661,\n",
            "  \"model.loss.nll\": 0.220947265625,\n",
            "  \"elapsed_time\": 0.06570172309875488,\n",
            "  \"wall_time\": 1674064270.4774594,\n",
            "  \"global_step\": 661\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.007720947265625,\n",
            "  \"model.lr\": 0.00013234000000000002,\n",
            "  \"model.grad_norm\": 0.2997801601886749,\n",
            "  \"model.elapsed_time\": 0.06476736068725586,\n",
            "  \"model.engine_step\": 662,\n",
            "  \"model.loss.nll\": 0.007720947265625,\n",
            "  \"elapsed_time\": 0.06476736068725586,\n",
            "  \"wall_time\": 1674064270.544169,\n",
            "  \"global_step\": 662\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0645751953125,\n",
            "  \"model.lr\": 0.000132539,\n",
            "  \"model.grad_norm\": 2.471350908279419,\n",
            "  \"model.elapsed_time\": 0.06853151321411133,\n",
            "  \"model.engine_step\": 663,\n",
            "  \"model.loss.nll\": 0.0645751953125,\n",
            "  \"elapsed_time\": 0.06853151321411133,\n",
            "  \"wall_time\": 1674064270.6198,\n",
            "  \"global_step\": 663\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0948486328125,\n",
            "  \"model.lr\": 0.000132738,\n",
            "  \"model.grad_norm\": 3.302635908126831,\n",
            "  \"model.elapsed_time\": 0.06984448432922363,\n",
            "  \"model.engine_step\": 664,\n",
            "  \"model.loss.nll\": 0.0948486328125,\n",
            "  \"elapsed_time\": 0.06984448432922363,\n",
            "  \"wall_time\": 1674064270.6921823,\n",
            "  \"global_step\": 664\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1678466796875,\n",
            "  \"model.lr\": 0.00013293700000000002,\n",
            "  \"model.grad_norm\": 3.931987762451172,\n",
            "  \"model.elapsed_time\": 0.0659935474395752,\n",
            "  \"model.engine_step\": 665,\n",
            "  \"model.loss.nll\": 0.1678466796875,\n",
            "  \"elapsed_time\": 0.0659935474395752,\n",
            "  \"wall_time\": 1674064270.7713683,\n",
            "  \"global_step\": 665\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0697021484375,\n",
            "  \"model.lr\": 0.000133136,\n",
            "  \"model.grad_norm\": 2.5635170936584473,\n",
            "  \"model.elapsed_time\": 0.06601858139038086,\n",
            "  \"model.engine_step\": 666,\n",
            "  \"model.loss.nll\": 0.0697021484375,\n",
            "  \"elapsed_time\": 0.06601858139038086,\n",
            "  \"wall_time\": 1674064270.839898,\n",
            "  \"global_step\": 666\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0841064453125,\n",
            "  \"model.lr\": 0.00013333500000000002,\n",
            "  \"model.grad_norm\": 2.320694923400879,\n",
            "  \"model.elapsed_time\": 0.07558250427246094,\n",
            "  \"model.engine_step\": 667,\n",
            "  \"model.loss.nll\": 0.0841064453125,\n",
            "  \"elapsed_time\": 0.07558250427246094,\n",
            "  \"wall_time\": 1674064270.9239588,\n",
            "  \"global_step\": 667\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0596923828125,\n",
            "  \"model.lr\": 0.000133534,\n",
            "  \"model.grad_norm\": 2.459104299545288,\n",
            "  \"model.elapsed_time\": 0.06681346893310547,\n",
            "  \"model.engine_step\": 668,\n",
            "  \"model.loss.nll\": 0.0596923828125,\n",
            "  \"elapsed_time\": 0.06681346893310547,\n",
            "  \"wall_time\": 1674064270.9932415,\n",
            "  \"global_step\": 668\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.051483154296875,\n",
            "  \"model.lr\": 0.000133733,\n",
            "  \"model.grad_norm\": 2.2985012531280518,\n",
            "  \"model.elapsed_time\": 0.07290005683898926,\n",
            "  \"model.engine_step\": 669,\n",
            "  \"model.loss.nll\": 0.051483154296875,\n",
            "  \"elapsed_time\": 0.07290005683898926,\n",
            "  \"wall_time\": 1674064271.0757332,\n",
            "  \"global_step\": 669\n",
            "}\n",
            "[2023-01-18 17:51:11,142] [INFO] [logging.py:68:log_dist] [Rank 0] step=670, skipped=1, lr=[0.00013393200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:11,144] [INFO] [timer.py:196:stop] epoch=0/micro_step=670/global_step=670, RunningAvgSamplesPerSec=13.512782865248033, CurrSamplesPerSec=14.960689126286315, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.163330078125,\n",
            "  \"model.lr\": 0.00013393200000000002,\n",
            "  \"model.grad_norm\": 4.7032036781311035,\n",
            "  \"model.elapsed_time\": 0.06731653213500977,\n",
            "  \"model.engine_step\": 670,\n",
            "  \"model.loss.nll\": 0.163330078125,\n",
            "  \"elapsed_time\": 0.06731653213500977,\n",
            "  \"wall_time\": 1674064271.145495,\n",
            "  \"global_step\": 670\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0138397216796875,\n",
            "  \"model.lr\": 0.000134131,\n",
            "  \"model.grad_norm\": 0.9332157373428345,\n",
            "  \"model.elapsed_time\": 0.07840776443481445,\n",
            "  \"model.engine_step\": 671,\n",
            "  \"model.loss.nll\": 0.0138397216796875,\n",
            "  \"elapsed_time\": 0.07840776443481445,\n",
            "  \"wall_time\": 1674064271.232468,\n",
            "  \"global_step\": 671\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0594482421875,\n",
            "  \"model.lr\": 0.00013433000000000002,\n",
            "  \"model.grad_norm\": 2.5320839881896973,\n",
            "  \"model.elapsed_time\": 0.06598377227783203,\n",
            "  \"model.engine_step\": 672,\n",
            "  \"model.loss.nll\": 0.0594482421875,\n",
            "  \"elapsed_time\": 0.06598377227783203,\n",
            "  \"wall_time\": 1674064271.300476,\n",
            "  \"global_step\": 672\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03338623046875,\n",
            "  \"model.lr\": 0.000134529,\n",
            "  \"model.grad_norm\": 0.979348361492157,\n",
            "  \"model.elapsed_time\": 0.06617188453674316,\n",
            "  \"model.engine_step\": 673,\n",
            "  \"model.loss.nll\": 0.03338623046875,\n",
            "  \"elapsed_time\": 0.06617188453674316,\n",
            "  \"wall_time\": 1674064271.375242,\n",
            "  \"global_step\": 673\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.019744873046875,\n",
            "  \"model.lr\": 0.000134728,\n",
            "  \"model.grad_norm\": 1.05537748336792,\n",
            "  \"model.elapsed_time\": 0.0704953670501709,\n",
            "  \"model.engine_step\": 674,\n",
            "  \"model.loss.nll\": 0.019744873046875,\n",
            "  \"elapsed_time\": 0.0704953670501709,\n",
            "  \"wall_time\": 1674064271.447761,\n",
            "  \"global_step\": 674\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01812744140625,\n",
            "  \"model.lr\": 0.00013492700000000002,\n",
            "  \"model.grad_norm\": 1.1703649759292603,\n",
            "  \"model.elapsed_time\": 0.07603836059570312,\n",
            "  \"model.engine_step\": 675,\n",
            "  \"model.loss.nll\": 0.01812744140625,\n",
            "  \"elapsed_time\": 0.07603836059570312,\n",
            "  \"wall_time\": 1674064271.5318935,\n",
            "  \"global_step\": 675\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005786895751953125,\n",
            "  \"model.lr\": 0.000135126,\n",
            "  \"model.grad_norm\": 0.28133273124694824,\n",
            "  \"model.elapsed_time\": 0.06687545776367188,\n",
            "  \"model.engine_step\": 676,\n",
            "  \"model.loss.nll\": 0.005786895751953125,\n",
            "  \"elapsed_time\": 0.06687545776367188,\n",
            "  \"wall_time\": 1674064271.6012459,\n",
            "  \"global_step\": 676\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.037384033203125,\n",
            "  \"model.lr\": 0.00013532500000000002,\n",
            "  \"model.grad_norm\": 1.4497363567352295,\n",
            "  \"model.elapsed_time\": 0.06707072257995605,\n",
            "  \"model.engine_step\": 677,\n",
            "  \"model.loss.nll\": 0.037384033203125,\n",
            "  \"elapsed_time\": 0.06707072257995605,\n",
            "  \"wall_time\": 1674064271.6771579,\n",
            "  \"global_step\": 677\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0142974853515625,\n",
            "  \"model.lr\": 0.000135524,\n",
            "  \"model.grad_norm\": 1.0101616382598877,\n",
            "  \"model.elapsed_time\": 0.07342267036437988,\n",
            "  \"model.engine_step\": 678,\n",
            "  \"model.loss.nll\": 0.0142974853515625,\n",
            "  \"elapsed_time\": 0.07342267036437988,\n",
            "  \"wall_time\": 1674064271.7526817,\n",
            "  \"global_step\": 678\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04168701171875,\n",
            "  \"model.lr\": 0.000135723,\n",
            "  \"model.grad_norm\": 2.0543336868286133,\n",
            "  \"model.elapsed_time\": 0.06810927391052246,\n",
            "  \"model.engine_step\": 679,\n",
            "  \"model.loss.nll\": 0.04168701171875,\n",
            "  \"elapsed_time\": 0.06810927391052246,\n",
            "  \"wall_time\": 1674064271.829332,\n",
            "  \"global_step\": 679\n",
            "}\n",
            "[2023-01-18 17:51:11,895] [INFO] [logging.py:68:log_dist] [Rank 0] step=680, skipped=1, lr=[0.00013592200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:11,897] [INFO] [timer.py:196:stop] epoch=0/micro_step=680/global_step=680, RunningAvgSamplesPerSec=13.524235664787671, CurrSamplesPerSec=15.205954327437254, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.011474609375,\n",
            "  \"model.lr\": 0.00013592200000000002,\n",
            "  \"model.grad_norm\": 0.8895103335380554,\n",
            "  \"model.elapsed_time\": 0.0662546157836914,\n",
            "  \"model.engine_step\": 680,\n",
            "  \"model.loss.nll\": 0.011474609375,\n",
            "  \"elapsed_time\": 0.0662546157836914,\n",
            "  \"wall_time\": 1674064271.8983476,\n",
            "  \"global_step\": 680\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.006412506103515625,\n",
            "  \"model.lr\": 0.000136121,\n",
            "  \"model.grad_norm\": 0.4512409269809723,\n",
            "  \"model.elapsed_time\": 0.06977963447570801,\n",
            "  \"model.engine_step\": 681,\n",
            "  \"model.loss.nll\": 0.006412506103515625,\n",
            "  \"elapsed_time\": 0.06977963447570801,\n",
            "  \"wall_time\": 1674064271.976189,\n",
            "  \"global_step\": 681\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1268310546875,\n",
            "  \"model.lr\": 0.00013632000000000002,\n",
            "  \"model.grad_norm\": 4.299638748168945,\n",
            "  \"model.elapsed_time\": 0.08397388458251953,\n",
            "  \"model.engine_step\": 682,\n",
            "  \"model.loss.nll\": 0.1268310546875,\n",
            "  \"elapsed_time\": 0.08397388458251953,\n",
            "  \"wall_time\": 1674064272.0628464,\n",
            "  \"global_step\": 682\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01003265380859375,\n",
            "  \"model.lr\": 0.000136519,\n",
            "  \"model.grad_norm\": 0.8245076537132263,\n",
            "  \"model.elapsed_time\": 0.06648921966552734,\n",
            "  \"model.engine_step\": 683,\n",
            "  \"model.loss.nll\": 0.01003265380859375,\n",
            "  \"elapsed_time\": 0.06648921966552734,\n",
            "  \"wall_time\": 1674064272.138134,\n",
            "  \"global_step\": 683\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021484375,\n",
            "  \"model.lr\": 0.00013671800000000003,\n",
            "  \"model.grad_norm\": 1.4962198734283447,\n",
            "  \"model.elapsed_time\": 0.06595349311828613,\n",
            "  \"model.engine_step\": 684,\n",
            "  \"model.loss.nll\": 0.021484375,\n",
            "  \"elapsed_time\": 0.06595349311828613,\n",
            "  \"wall_time\": 1674064272.206653,\n",
            "  \"global_step\": 684\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.062286376953125,\n",
            "  \"model.lr\": 0.00013691700000000002,\n",
            "  \"model.grad_norm\": 3.164285898208618,\n",
            "  \"model.elapsed_time\": 0.0738222599029541,\n",
            "  \"model.engine_step\": 685,\n",
            "  \"model.loss.nll\": 0.062286376953125,\n",
            "  \"elapsed_time\": 0.0738222599029541,\n",
            "  \"wall_time\": 1674064272.2929866,\n",
            "  \"global_step\": 685\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05194091796875,\n",
            "  \"model.lr\": 0.000137116,\n",
            "  \"model.grad_norm\": 3.0367140769958496,\n",
            "  \"model.elapsed_time\": 0.08273482322692871,\n",
            "  \"model.engine_step\": 686,\n",
            "  \"model.loss.nll\": 0.05194091796875,\n",
            "  \"elapsed_time\": 0.08273482322692871,\n",
            "  \"wall_time\": 1674064272.3787816,\n",
            "  \"global_step\": 686\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1414794921875,\n",
            "  \"model.lr\": 0.00013731500000000002,\n",
            "  \"model.grad_norm\": 2.8961355686187744,\n",
            "  \"model.elapsed_time\": 0.07171750068664551,\n",
            "  \"model.engine_step\": 687,\n",
            "  \"model.loss.nll\": 0.1414794921875,\n",
            "  \"elapsed_time\": 0.07171750068664551,\n",
            "  \"wall_time\": 1674064272.4612455,\n",
            "  \"global_step\": 687\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005985260009765625,\n",
            "  \"model.lr\": 0.000137514,\n",
            "  \"model.grad_norm\": 0.3048945665359497,\n",
            "  \"model.elapsed_time\": 0.06536173820495605,\n",
            "  \"model.engine_step\": 688,\n",
            "  \"model.loss.nll\": 0.005985260009765625,\n",
            "  \"elapsed_time\": 0.06536173820495605,\n",
            "  \"wall_time\": 1674064272.5292053,\n",
            "  \"global_step\": 688\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01116943359375,\n",
            "  \"model.lr\": 0.00013771300000000003,\n",
            "  \"model.grad_norm\": 0.9634642601013184,\n",
            "  \"model.elapsed_time\": 0.06457948684692383,\n",
            "  \"model.engine_step\": 689,\n",
            "  \"model.loss.nll\": 0.01116943359375,\n",
            "  \"elapsed_time\": 0.06457948684692383,\n",
            "  \"wall_time\": 1674064272.6013305,\n",
            "  \"global_step\": 689\n",
            "}\n",
            "[2023-01-18 17:51:12,667] [INFO] [logging.py:68:log_dist] [Rank 0] step=690, skipped=1, lr=[0.000137912], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:12,669] [INFO] [timer.py:196:stop] epoch=0/micro_step=690/global_step=690, RunningAvgSamplesPerSec=13.532009775617349, CurrSamplesPerSec=15.01515726467577, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0738525390625,\n",
            "  \"model.lr\": 0.000137912,\n",
            "  \"model.grad_norm\": 4.333977699279785,\n",
            "  \"model.elapsed_time\": 0.06706357002258301,\n",
            "  \"model.engine_step\": 690,\n",
            "  \"model.loss.nll\": 0.0738525390625,\n",
            "  \"elapsed_time\": 0.06706357002258301,\n",
            "  \"wall_time\": 1674064272.6703365,\n",
            "  \"global_step\": 690\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.054229736328125,\n",
            "  \"model.lr\": 0.000138111,\n",
            "  \"model.grad_norm\": 2.340653896331787,\n",
            "  \"model.elapsed_time\": 0.07775449752807617,\n",
            "  \"model.engine_step\": 691,\n",
            "  \"model.loss.nll\": 0.054229736328125,\n",
            "  \"elapsed_time\": 0.07775449752807617,\n",
            "  \"wall_time\": 1674064272.7569895,\n",
            "  \"global_step\": 691\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0361328125,\n",
            "  \"model.lr\": 0.00013831,\n",
            "  \"model.grad_norm\": 1.6978800296783447,\n",
            "  \"model.elapsed_time\": 0.06496381759643555,\n",
            "  \"model.engine_step\": 692,\n",
            "  \"model.loss.nll\": 0.0361328125,\n",
            "  \"elapsed_time\": 0.06496381759643555,\n",
            "  \"wall_time\": 1674064272.8239863,\n",
            "  \"global_step\": 692\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.039642333984375,\n",
            "  \"model.lr\": 0.00013850899999999998,\n",
            "  \"model.grad_norm\": 1.8360949754714966,\n",
            "  \"model.elapsed_time\": 0.0644679069519043,\n",
            "  \"model.engine_step\": 693,\n",
            "  \"model.loss.nll\": 0.039642333984375,\n",
            "  \"elapsed_time\": 0.0644679069519043,\n",
            "  \"wall_time\": 1674064272.8952558,\n",
            "  \"global_step\": 693\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.031005859375,\n",
            "  \"model.lr\": 0.000138708,\n",
            "  \"model.grad_norm\": 1.3711062669754028,\n",
            "  \"model.elapsed_time\": 0.06584024429321289,\n",
            "  \"model.engine_step\": 694,\n",
            "  \"model.loss.nll\": 0.031005859375,\n",
            "  \"elapsed_time\": 0.06584024429321289,\n",
            "  \"wall_time\": 1674064272.963742,\n",
            "  \"global_step\": 694\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.12548828125,\n",
            "  \"model.lr\": 0.000138907,\n",
            "  \"model.grad_norm\": 3.6774659156799316,\n",
            "  \"model.elapsed_time\": 0.07220888137817383,\n",
            "  \"model.engine_step\": 695,\n",
            "  \"model.loss.nll\": 0.12548828125,\n",
            "  \"elapsed_time\": 0.07220888137817383,\n",
            "  \"wall_time\": 1674064273.0451522,\n",
            "  \"global_step\": 695\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.069091796875,\n",
            "  \"model.lr\": 0.000139106,\n",
            "  \"model.grad_norm\": 2.861276626586914,\n",
            "  \"model.elapsed_time\": 0.07130718231201172,\n",
            "  \"model.engine_step\": 696,\n",
            "  \"model.loss.nll\": 0.069091796875,\n",
            "  \"elapsed_time\": 0.07130718231201172,\n",
            "  \"wall_time\": 1674064273.1186924,\n",
            "  \"global_step\": 696\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04345703125,\n",
            "  \"model.lr\": 0.000139305,\n",
            "  \"model.grad_norm\": 2.098349094390869,\n",
            "  \"model.elapsed_time\": 0.06680011749267578,\n",
            "  \"model.engine_step\": 697,\n",
            "  \"model.loss.nll\": 0.04345703125,\n",
            "  \"elapsed_time\": 0.06680011749267578,\n",
            "  \"wall_time\": 1674064273.1921685,\n",
            "  \"global_step\": 697\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.076171875,\n",
            "  \"model.lr\": 0.00013950399999999998,\n",
            "  \"model.grad_norm\": 2.307654857635498,\n",
            "  \"model.elapsed_time\": 0.07197785377502441,\n",
            "  \"model.engine_step\": 698,\n",
            "  \"model.loss.nll\": 0.076171875,\n",
            "  \"elapsed_time\": 0.07197785377502441,\n",
            "  \"wall_time\": 1674064273.2669156,\n",
            "  \"global_step\": 698\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00293731689453125,\n",
            "  \"model.lr\": 0.000139703,\n",
            "  \"model.grad_norm\": 0.10502669960260391,\n",
            "  \"model.elapsed_time\": 0.06783342361450195,\n",
            "  \"model.engine_step\": 699,\n",
            "  \"model.loss.nll\": 0.00293731689453125,\n",
            "  \"elapsed_time\": 0.06783342361450195,\n",
            "  \"wall_time\": 1674064273.3421397,\n",
            "  \"global_step\": 699\n",
            "}\n",
            "[2023-01-18 17:51:13,409] [INFO] [logging.py:68:log_dist] [Rank 0] step=700, skipped=1, lr=[0.000139902], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:13,410] [INFO] [timer.py:196:stop] epoch=0/micro_step=700/global_step=700, RunningAvgSamplesPerSec=13.545190490113047, CurrSamplesPerSec=15.05250049345942, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0171051025390625,\n",
            "  \"model.lr\": 0.000139902,\n",
            "  \"model.grad_norm\": 1.6470047235488892,\n",
            "  \"model.elapsed_time\": 0.06689000129699707,\n",
            "  \"model.engine_step\": 700,\n",
            "  \"model.loss.nll\": 0.0171051025390625,\n",
            "  \"elapsed_time\": 0.06689000129699707,\n",
            "  \"wall_time\": 1674064273.4118605,\n",
            "  \"global_step\": 700\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0215606689453125,\n",
            "  \"model.lr\": 0.000140101,\n",
            "  \"model.grad_norm\": 1.3586033582687378,\n",
            "  \"model.elapsed_time\": 0.0782327651977539,\n",
            "  \"model.engine_step\": 701,\n",
            "  \"model.loss.nll\": 0.0215606689453125,\n",
            "  \"elapsed_time\": 0.0782327651977539,\n",
            "  \"wall_time\": 1674064273.4977155,\n",
            "  \"global_step\": 701\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022064208984375,\n",
            "  \"model.lr\": 0.0001403,\n",
            "  \"model.grad_norm\": 1.5020030736923218,\n",
            "  \"model.elapsed_time\": 0.07325434684753418,\n",
            "  \"model.engine_step\": 702,\n",
            "  \"model.loss.nll\": 0.022064208984375,\n",
            "  \"elapsed_time\": 0.07325434684753418,\n",
            "  \"wall_time\": 1674064273.5737731,\n",
            "  \"global_step\": 702\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.256591796875,\n",
            "  \"model.lr\": 0.00014049899999999998,\n",
            "  \"model.grad_norm\": 4.155454635620117,\n",
            "  \"model.elapsed_time\": 0.06999659538269043,\n",
            "  \"model.engine_step\": 703,\n",
            "  \"model.loss.nll\": 0.256591796875,\n",
            "  \"elapsed_time\": 0.06999659538269043,\n",
            "  \"wall_time\": 1674064273.6521213,\n",
            "  \"global_step\": 703\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0058746337890625,\n",
            "  \"model.lr\": 0.000140698,\n",
            "  \"model.grad_norm\": 0.33834001421928406,\n",
            "  \"model.elapsed_time\": 0.06762027740478516,\n",
            "  \"model.engine_step\": 704,\n",
            "  \"model.loss.nll\": 0.0058746337890625,\n",
            "  \"elapsed_time\": 0.06762027740478516,\n",
            "  \"wall_time\": 1674064273.721799,\n",
            "  \"global_step\": 704\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05023193359375,\n",
            "  \"model.lr\": 0.000140897,\n",
            "  \"model.grad_norm\": 2.0117969512939453,\n",
            "  \"model.elapsed_time\": 0.0649881362915039,\n",
            "  \"model.engine_step\": 705,\n",
            "  \"model.loss.nll\": 0.05023193359375,\n",
            "  \"elapsed_time\": 0.0649881362915039,\n",
            "  \"wall_time\": 1674064273.7935765,\n",
            "  \"global_step\": 705\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0192413330078125,\n",
            "  \"model.lr\": 0.000141096,\n",
            "  \"model.grad_norm\": 1.4188792705535889,\n",
            "  \"model.elapsed_time\": 0.07432150840759277,\n",
            "  \"model.engine_step\": 706,\n",
            "  \"model.loss.nll\": 0.0192413330078125,\n",
            "  \"elapsed_time\": 0.07432150840759277,\n",
            "  \"wall_time\": 1674064273.8703263,\n",
            "  \"global_step\": 706\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0855712890625,\n",
            "  \"model.lr\": 0.000141295,\n",
            "  \"model.grad_norm\": 3.9039056301116943,\n",
            "  \"model.elapsed_time\": 0.06441354751586914,\n",
            "  \"model.engine_step\": 707,\n",
            "  \"model.loss.nll\": 0.0855712890625,\n",
            "  \"elapsed_time\": 0.06441354751586914,\n",
            "  \"wall_time\": 1674064273.9407961,\n",
            "  \"global_step\": 707\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0257568359375,\n",
            "  \"model.lr\": 0.000141494,\n",
            "  \"model.grad_norm\": 1.313428282737732,\n",
            "  \"model.elapsed_time\": 0.06655263900756836,\n",
            "  \"model.engine_step\": 708,\n",
            "  \"model.loss.nll\": 0.0257568359375,\n",
            "  \"elapsed_time\": 0.06655263900756836,\n",
            "  \"wall_time\": 1674064274.009771,\n",
            "  \"global_step\": 708\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0212860107421875,\n",
            "  \"model.lr\": 0.000141693,\n",
            "  \"model.grad_norm\": 1.0259795188903809,\n",
            "  \"model.elapsed_time\": 0.07435798645019531,\n",
            "  \"model.engine_step\": 709,\n",
            "  \"model.loss.nll\": 0.0212860107421875,\n",
            "  \"elapsed_time\": 0.07435798645019531,\n",
            "  \"wall_time\": 1674064274.0903542,\n",
            "  \"global_step\": 709\n",
            "}\n",
            "[2023-01-18 17:51:14,166] [INFO] [logging.py:68:log_dist] [Rank 0] step=710, skipped=1, lr=[0.000141892], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:14,167] [INFO] [timer.py:196:stop] epoch=0/micro_step=710/global_step=710, RunningAvgSamplesPerSec=13.552939465152553, CurrSamplesPerSec=13.26039904774852, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07452392578125,\n",
            "  \"model.lr\": 0.000141892,\n",
            "  \"model.grad_norm\": 1.701620101928711,\n",
            "  \"model.elapsed_time\": 0.07584977149963379,\n",
            "  \"model.engine_step\": 710,\n",
            "  \"model.loss.nll\": 0.07452392578125,\n",
            "  \"elapsed_time\": 0.07584977149963379,\n",
            "  \"wall_time\": 1674064274.1687462,\n",
            "  \"global_step\": 710\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07135009765625,\n",
            "  \"model.lr\": 0.000142091,\n",
            "  \"model.grad_norm\": 2.4637250900268555,\n",
            "  \"model.elapsed_time\": 0.06617498397827148,\n",
            "  \"model.engine_step\": 711,\n",
            "  \"model.loss.nll\": 0.07135009765625,\n",
            "  \"elapsed_time\": 0.06617498397827148,\n",
            "  \"wall_time\": 1674064274.2425516,\n",
            "  \"global_step\": 711\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03204345703125,\n",
            "  \"model.lr\": 0.00014229,\n",
            "  \"model.grad_norm\": 1.687792181968689,\n",
            "  \"model.elapsed_time\": 0.06640410423278809,\n",
            "  \"model.engine_step\": 712,\n",
            "  \"model.loss.nll\": 0.03204345703125,\n",
            "  \"elapsed_time\": 0.06640410423278809,\n",
            "  \"wall_time\": 1674064274.3111293,\n",
            "  \"global_step\": 712\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.019073486328125,\n",
            "  \"model.lr\": 0.000142489,\n",
            "  \"model.grad_norm\": 0.9305726289749146,\n",
            "  \"model.elapsed_time\": 0.07597827911376953,\n",
            "  \"model.engine_step\": 713,\n",
            "  \"model.loss.nll\": 0.019073486328125,\n",
            "  \"elapsed_time\": 0.07597827911376953,\n",
            "  \"wall_time\": 1674064274.394256,\n",
            "  \"global_step\": 713\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.006290435791015625,\n",
            "  \"model.lr\": 0.000142688,\n",
            "  \"model.grad_norm\": 0.33772650361061096,\n",
            "  \"model.elapsed_time\": 0.06847524642944336,\n",
            "  \"model.engine_step\": 714,\n",
            "  \"model.loss.nll\": 0.006290435791015625,\n",
            "  \"elapsed_time\": 0.06847524642944336,\n",
            "  \"wall_time\": 1674064274.4653857,\n",
            "  \"global_step\": 714\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.061187744140625,\n",
            "  \"model.lr\": 0.000142887,\n",
            "  \"model.grad_norm\": 2.312952756881714,\n",
            "  \"model.elapsed_time\": 0.07063555717468262,\n",
            "  \"model.engine_step\": 715,\n",
            "  \"model.loss.nll\": 0.061187744140625,\n",
            "  \"elapsed_time\": 0.07063555717468262,\n",
            "  \"wall_time\": 1674064274.5438464,\n",
            "  \"global_step\": 715\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.053802490234375,\n",
            "  \"model.lr\": 0.000143086,\n",
            "  \"model.grad_norm\": 1.6285432577133179,\n",
            "  \"model.elapsed_time\": 0.07006454467773438,\n",
            "  \"model.engine_step\": 716,\n",
            "  \"model.loss.nll\": 0.053802490234375,\n",
            "  \"elapsed_time\": 0.07006454467773438,\n",
            "  \"wall_time\": 1674064274.6162694,\n",
            "  \"global_step\": 716\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1392822265625,\n",
            "  \"model.lr\": 0.000143285,\n",
            "  \"model.grad_norm\": 3.572406530380249,\n",
            "  \"model.elapsed_time\": 0.06858706474304199,\n",
            "  \"model.engine_step\": 717,\n",
            "  \"model.loss.nll\": 0.1392822265625,\n",
            "  \"elapsed_time\": 0.06858706474304199,\n",
            "  \"wall_time\": 1674064274.692732,\n",
            "  \"global_step\": 717\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0341796875,\n",
            "  \"model.lr\": 0.000143484,\n",
            "  \"model.grad_norm\": 1.349808692932129,\n",
            "  \"model.elapsed_time\": 0.07111144065856934,\n",
            "  \"model.engine_step\": 718,\n",
            "  \"model.loss.nll\": 0.0341796875,\n",
            "  \"elapsed_time\": 0.07111144065856934,\n",
            "  \"wall_time\": 1674064274.76578,\n",
            "  \"global_step\": 718\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005527496337890625,\n",
            "  \"model.lr\": 0.000143683,\n",
            "  \"model.grad_norm\": 0.46401017904281616,\n",
            "  \"model.elapsed_time\": 0.07397723197937012,\n",
            "  \"model.engine_step\": 719,\n",
            "  \"model.loss.nll\": 0.005527496337890625,\n",
            "  \"elapsed_time\": 0.07397723197937012,\n",
            "  \"wall_time\": 1674064274.8484128,\n",
            "  \"global_step\": 719\n",
            "}\n",
            "[2023-01-18 17:51:14,914] [INFO] [logging.py:68:log_dist] [Rank 0] step=720, skipped=1, lr=[0.000143882], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:14,916] [INFO] [timer.py:196:stop] epoch=0/micro_step=720/global_step=720, RunningAvgSamplesPerSec=13.563527393441987, CurrSamplesPerSec=15.182561228996084, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.003589630126953125,\n",
            "  \"model.lr\": 0.000143882,\n",
            "  \"model.grad_norm\": 0.26594850420951843,\n",
            "  \"model.elapsed_time\": 0.06635379791259766,\n",
            "  \"model.engine_step\": 720,\n",
            "  \"model.loss.nll\": 0.003589630126953125,\n",
            "  \"elapsed_time\": 0.06635379791259766,\n",
            "  \"wall_time\": 1674064274.917669,\n",
            "  \"global_step\": 720\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01055908203125,\n",
            "  \"model.lr\": 0.000144081,\n",
            "  \"model.grad_norm\": 0.9271559715270996,\n",
            "  \"model.elapsed_time\": 0.06917071342468262,\n",
            "  \"model.engine_step\": 721,\n",
            "  \"model.loss.nll\": 0.01055908203125,\n",
            "  \"elapsed_time\": 0.06917071342468262,\n",
            "  \"wall_time\": 1674064274.9953854,\n",
            "  \"global_step\": 721\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01215362548828125,\n",
            "  \"model.lr\": 0.00014428,\n",
            "  \"model.grad_norm\": 0.7920917272567749,\n",
            "  \"model.elapsed_time\": 0.07246279716491699,\n",
            "  \"model.engine_step\": 722,\n",
            "  \"model.loss.nll\": 0.01215362548828125,\n",
            "  \"elapsed_time\": 0.07246279716491699,\n",
            "  \"wall_time\": 1674064275.0699162,\n",
            "  \"global_step\": 722\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.020172119140625,\n",
            "  \"model.lr\": 0.000144479,\n",
            "  \"model.grad_norm\": 1.330292820930481,\n",
            "  \"model.elapsed_time\": 0.07569575309753418,\n",
            "  \"model.engine_step\": 723,\n",
            "  \"model.loss.nll\": 0.020172119140625,\n",
            "  \"elapsed_time\": 0.07569575309753418,\n",
            "  \"wall_time\": 1674064275.1539981,\n",
            "  \"global_step\": 723\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0316162109375,\n",
            "  \"model.lr\": 0.000144678,\n",
            "  \"model.grad_norm\": 1.5950610637664795,\n",
            "  \"model.elapsed_time\": 0.07041740417480469,\n",
            "  \"model.engine_step\": 724,\n",
            "  \"model.loss.nll\": 0.0316162109375,\n",
            "  \"elapsed_time\": 0.07041740417480469,\n",
            "  \"wall_time\": 1674064275.2269883,\n",
            "  \"global_step\": 724\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00533294677734375,\n",
            "  \"model.lr\": 0.000144877,\n",
            "  \"model.grad_norm\": 0.3030203878879547,\n",
            "  \"model.elapsed_time\": 0.06799030303955078,\n",
            "  \"model.engine_step\": 725,\n",
            "  \"model.loss.nll\": 0.00533294677734375,\n",
            "  \"elapsed_time\": 0.06799030303955078,\n",
            "  \"wall_time\": 1674064275.3036869,\n",
            "  \"global_step\": 725\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2197265625,\n",
            "  \"model.lr\": 0.000145076,\n",
            "  \"model.grad_norm\": 4.3784499168396,\n",
            "  \"model.elapsed_time\": 0.07369494438171387,\n",
            "  \"model.engine_step\": 726,\n",
            "  \"model.loss.nll\": 0.2197265625,\n",
            "  \"elapsed_time\": 0.07369494438171387,\n",
            "  \"wall_time\": 1674064275.3795843,\n",
            "  \"global_step\": 726\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1014404296875,\n",
            "  \"model.lr\": 0.000145275,\n",
            "  \"model.grad_norm\": 2.6867098808288574,\n",
            "  \"model.elapsed_time\": 0.06444096565246582,\n",
            "  \"model.engine_step\": 727,\n",
            "  \"model.loss.nll\": 0.1014404296875,\n",
            "  \"elapsed_time\": 0.06444096565246582,\n",
            "  \"wall_time\": 1674064275.451949,\n",
            "  \"global_step\": 727\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04364013671875,\n",
            "  \"model.lr\": 0.000145474,\n",
            "  \"model.grad_norm\": 1.4490057229995728,\n",
            "  \"model.elapsed_time\": 0.06700682640075684,\n",
            "  \"model.engine_step\": 728,\n",
            "  \"model.loss.nll\": 0.04364013671875,\n",
            "  \"elapsed_time\": 0.06700682640075684,\n",
            "  \"wall_time\": 1674064275.5214584,\n",
            "  \"global_step\": 728\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07427978515625,\n",
            "  \"model.lr\": 0.000145673,\n",
            "  \"model.grad_norm\": 2.09084153175354,\n",
            "  \"model.elapsed_time\": 0.07384610176086426,\n",
            "  \"model.engine_step\": 729,\n",
            "  \"model.loss.nll\": 0.07427978515625,\n",
            "  \"elapsed_time\": 0.07384610176086426,\n",
            "  \"wall_time\": 1674064275.6032758,\n",
            "  \"global_step\": 729\n",
            "}\n",
            "[2023-01-18 17:51:15,667] [INFO] [logging.py:68:log_dist] [Rank 0] step=730, skipped=1, lr=[0.00014587200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:15,668] [INFO] [timer.py:196:stop] epoch=0/micro_step=730/global_step=730, RunningAvgSamplesPerSec=13.573471951088203, CurrSamplesPerSec=15.632189391454725, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.10552978515625,\n",
            "  \"model.lr\": 0.00014587200000000002,\n",
            "  \"model.grad_norm\": 2.065702438354492,\n",
            "  \"model.elapsed_time\": 0.06440925598144531,\n",
            "  \"model.engine_step\": 730,\n",
            "  \"model.loss.nll\": 0.10552978515625,\n",
            "  \"elapsed_time\": 0.06440925598144531,\n",
            "  \"wall_time\": 1674064275.6696372,\n",
            "  \"global_step\": 730\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0740966796875,\n",
            "  \"model.lr\": 0.000146071,\n",
            "  \"model.grad_norm\": 1.6570626497268677,\n",
            "  \"model.elapsed_time\": 0.07427620887756348,\n",
            "  \"model.engine_step\": 731,\n",
            "  \"model.loss.nll\": 0.0740966796875,\n",
            "  \"elapsed_time\": 0.07427620887756348,\n",
            "  \"wall_time\": 1674064275.7510448,\n",
            "  \"global_step\": 731\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.040313720703125,\n",
            "  \"model.lr\": 0.00014627,\n",
            "  \"model.grad_norm\": 1.5446245670318604,\n",
            "  \"model.elapsed_time\": 0.0666196346282959,\n",
            "  \"model.engine_step\": 732,\n",
            "  \"model.loss.nll\": 0.040313720703125,\n",
            "  \"elapsed_time\": 0.0666196346282959,\n",
            "  \"wall_time\": 1674064275.8204892,\n",
            "  \"global_step\": 732\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.11639404296875,\n",
            "  \"model.lr\": 0.000146469,\n",
            "  \"model.grad_norm\": 3.719604253768921,\n",
            "  \"model.elapsed_time\": 0.07398724555969238,\n",
            "  \"model.engine_step\": 733,\n",
            "  \"model.loss.nll\": 0.11639404296875,\n",
            "  \"elapsed_time\": 0.07398724555969238,\n",
            "  \"wall_time\": 1674064275.9023526,\n",
            "  \"global_step\": 733\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00434112548828125,\n",
            "  \"model.lr\": 0.000146668,\n",
            "  \"model.grad_norm\": 0.32628583908081055,\n",
            "  \"model.elapsed_time\": 0.07251191139221191,\n",
            "  \"model.engine_step\": 734,\n",
            "  \"model.loss.nll\": 0.00434112548828125,\n",
            "  \"elapsed_time\": 0.07251191139221191,\n",
            "  \"wall_time\": 1674064275.9770837,\n",
            "  \"global_step\": 734\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005496978759765625,\n",
            "  \"model.lr\": 0.00014686700000000002,\n",
            "  \"model.grad_norm\": 0.46348094940185547,\n",
            "  \"model.elapsed_time\": 0.07370758056640625,\n",
            "  \"model.engine_step\": 735,\n",
            "  \"model.loss.nll\": 0.005496978759765625,\n",
            "  \"elapsed_time\": 0.07370758056640625,\n",
            "  \"wall_time\": 1674064276.0593543,\n",
            "  \"global_step\": 735\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0132598876953125,\n",
            "  \"model.lr\": 0.000147066,\n",
            "  \"model.grad_norm\": 0.8303768038749695,\n",
            "  \"model.elapsed_time\": 0.10190367698669434,\n",
            "  \"model.engine_step\": 736,\n",
            "  \"model.loss.nll\": 0.0132598876953125,\n",
            "  \"elapsed_time\": 0.10190367698669434,\n",
            "  \"wall_time\": 1674064276.1635518,\n",
            "  \"global_step\": 736\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0771484375,\n",
            "  \"model.lr\": 0.000147265,\n",
            "  \"model.grad_norm\": 2.3493595123291016,\n",
            "  \"model.elapsed_time\": 0.07071328163146973,\n",
            "  \"model.engine_step\": 737,\n",
            "  \"model.loss.nll\": 0.0771484375,\n",
            "  \"elapsed_time\": 0.07071328163146973,\n",
            "  \"wall_time\": 1674064276.2414973,\n",
            "  \"global_step\": 737\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.10906982421875,\n",
            "  \"model.lr\": 0.000147464,\n",
            "  \"model.grad_norm\": 2.4268314838409424,\n",
            "  \"model.elapsed_time\": 0.06653046607971191,\n",
            "  \"model.engine_step\": 738,\n",
            "  \"model.loss.nll\": 0.10906982421875,\n",
            "  \"elapsed_time\": 0.06653046607971191,\n",
            "  \"wall_time\": 1674064276.3112493,\n",
            "  \"global_step\": 738\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.018585205078125,\n",
            "  \"model.lr\": 0.000147663,\n",
            "  \"model.grad_norm\": 1.5054527521133423,\n",
            "  \"model.elapsed_time\": 0.06831812858581543,\n",
            "  \"model.engine_step\": 739,\n",
            "  \"model.loss.nll\": 0.018585205078125,\n",
            "  \"elapsed_time\": 0.06831812858581543,\n",
            "  \"wall_time\": 1674064276.3883624,\n",
            "  \"global_step\": 739\n",
            "}\n",
            "[2023-01-18 17:51:16,459] [INFO] [logging.py:68:log_dist] [Rank 0] step=740, skipped=1, lr=[0.00014786200000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:16,459] [INFO] [timer.py:196:stop] epoch=0/micro_step=740/global_step=740, RunningAvgSamplesPerSec=13.573236524304411, CurrSamplesPerSec=14.27984284459455, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0210113525390625,\n",
            "  \"model.lr\": 0.00014786200000000002,\n",
            "  \"model.grad_norm\": 0.8980631232261658,\n",
            "  \"model.elapsed_time\": 0.07068657875061035,\n",
            "  \"model.engine_step\": 740,\n",
            "  \"model.loss.nll\": 0.0210113525390625,\n",
            "  \"elapsed_time\": 0.07068657875061035,\n",
            "  \"wall_time\": 1674064276.461389,\n",
            "  \"global_step\": 740\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0966796875,\n",
            "  \"model.lr\": 0.000148061,\n",
            "  \"model.grad_norm\": 2.6414995193481445,\n",
            "  \"model.elapsed_time\": 0.06526803970336914,\n",
            "  \"model.engine_step\": 741,\n",
            "  \"model.loss.nll\": 0.0966796875,\n",
            "  \"elapsed_time\": 0.06526803970336914,\n",
            "  \"wall_time\": 1674064276.5363364,\n",
            "  \"global_step\": 741\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.053924560546875,\n",
            "  \"model.lr\": 0.00014826,\n",
            "  \"model.grad_norm\": 2.26761794090271,\n",
            "  \"model.elapsed_time\": 0.06605195999145508,\n",
            "  \"model.engine_step\": 742,\n",
            "  \"model.loss.nll\": 0.053924560546875,\n",
            "  \"elapsed_time\": 0.06605195999145508,\n",
            "  \"wall_time\": 1674064276.6050463,\n",
            "  \"global_step\": 742\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0192108154296875,\n",
            "  \"model.lr\": 0.000148459,\n",
            "  \"model.grad_norm\": 1.0483195781707764,\n",
            "  \"model.elapsed_time\": 0.07774829864501953,\n",
            "  \"model.engine_step\": 743,\n",
            "  \"model.loss.nll\": 0.0192108154296875,\n",
            "  \"elapsed_time\": 0.07774829864501953,\n",
            "  \"wall_time\": 1674064276.6904979,\n",
            "  \"global_step\": 743\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0217132568359375,\n",
            "  \"model.lr\": 0.000148658,\n",
            "  \"model.grad_norm\": 1.3404241800308228,\n",
            "  \"model.elapsed_time\": 0.07208514213562012,\n",
            "  \"model.engine_step\": 744,\n",
            "  \"model.loss.nll\": 0.0217132568359375,\n",
            "  \"elapsed_time\": 0.07208514213562012,\n",
            "  \"wall_time\": 1674064276.7647457,\n",
            "  \"global_step\": 744\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.011688232421875,\n",
            "  \"model.lr\": 0.00014885700000000001,\n",
            "  \"model.grad_norm\": 0.6989189982414246,\n",
            "  \"model.elapsed_time\": 0.06867551803588867,\n",
            "  \"model.engine_step\": 745,\n",
            "  \"model.loss.nll\": 0.011688232421875,\n",
            "  \"elapsed_time\": 0.06867551803588867,\n",
            "  \"wall_time\": 1674064276.840875,\n",
            "  \"global_step\": 745\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01023101806640625,\n",
            "  \"model.lr\": 0.000149056,\n",
            "  \"model.grad_norm\": 0.6005299091339111,\n",
            "  \"model.elapsed_time\": 0.06504631042480469,\n",
            "  \"model.engine_step\": 746,\n",
            "  \"model.loss.nll\": 0.01023101806640625,\n",
            "  \"elapsed_time\": 0.06504631042480469,\n",
            "  \"wall_time\": 1674064276.9084861,\n",
            "  \"global_step\": 746\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04827880859375,\n",
            "  \"model.lr\": 0.000149255,\n",
            "  \"model.grad_norm\": 1.8538258075714111,\n",
            "  \"model.elapsed_time\": 0.07085561752319336,\n",
            "  \"model.engine_step\": 747,\n",
            "  \"model.loss.nll\": 0.04827880859375,\n",
            "  \"elapsed_time\": 0.07085561752319336,\n",
            "  \"wall_time\": 1674064276.9857528,\n",
            "  \"global_step\": 747\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0065460205078125,\n",
            "  \"model.lr\": 0.000149454,\n",
            "  \"model.grad_norm\": 0.5406762361526489,\n",
            "  \"model.elapsed_time\": 0.06711220741271973,\n",
            "  \"model.engine_step\": 748,\n",
            "  \"model.loss.nll\": 0.0065460205078125,\n",
            "  \"elapsed_time\": 0.06711220741271973,\n",
            "  \"wall_time\": 1674064277.055319,\n",
            "  \"global_step\": 748\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0025157928466796875,\n",
            "  \"model.lr\": 0.000149653,\n",
            "  \"model.grad_norm\": 0.09115352481603622,\n",
            "  \"model.elapsed_time\": 0.0677347183227539,\n",
            "  \"model.engine_step\": 749,\n",
            "  \"model.loss.nll\": 0.0025157928466796875,\n",
            "  \"elapsed_time\": 0.0677347183227539,\n",
            "  \"wall_time\": 1674064277.1302354,\n",
            "  \"global_step\": 749\n",
            "}\n",
            "[2023-01-18 17:51:17,207] [INFO] [logging.py:68:log_dist] [Rank 0] step=750, skipped=1, lr=[0.00014985200000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:17,209] [INFO] [timer.py:196:stop] epoch=0/micro_step=750/global_step=750, RunningAvgSamplesPerSec=13.583043829035226, CurrSamplesPerSec=12.954938982768153, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005626678466796875,\n",
            "  \"model.lr\": 0.00014985200000000001,\n",
            "  \"model.grad_norm\": 0.33278509974479675,\n",
            "  \"model.elapsed_time\": 0.07762885093688965,\n",
            "  \"model.engine_step\": 750,\n",
            "  \"model.loss.nll\": 0.005626678466796875,\n",
            "  \"elapsed_time\": 0.07762885093688965,\n",
            "  \"wall_time\": 1674064277.210339,\n",
            "  \"global_step\": 750\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0088653564453125,\n",
            "  \"model.lr\": 0.000150051,\n",
            "  \"model.grad_norm\": 0.9581755995750427,\n",
            "  \"model.elapsed_time\": 0.07430267333984375,\n",
            "  \"model.engine_step\": 751,\n",
            "  \"model.loss.nll\": 0.0088653564453125,\n",
            "  \"elapsed_time\": 0.07430267333984375,\n",
            "  \"wall_time\": 1674064277.293375,\n",
            "  \"global_step\": 751\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.037841796875,\n",
            "  \"model.lr\": 0.00015025000000000002,\n",
            "  \"model.grad_norm\": 0.9778397679328918,\n",
            "  \"model.elapsed_time\": 0.0643761157989502,\n",
            "  \"model.engine_step\": 752,\n",
            "  \"model.loss.nll\": 0.037841796875,\n",
            "  \"elapsed_time\": 0.0643761157989502,\n",
            "  \"wall_time\": 1674064277.359724,\n",
            "  \"global_step\": 752\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0029888153076171875,\n",
            "  \"model.lr\": 0.000150449,\n",
            "  \"model.grad_norm\": 0.16551388800144196,\n",
            "  \"model.elapsed_time\": 0.06527066230773926,\n",
            "  \"model.engine_step\": 753,\n",
            "  \"model.loss.nll\": 0.0029888153076171875,\n",
            "  \"elapsed_time\": 0.06527066230773926,\n",
            "  \"wall_time\": 1674064277.4334438,\n",
            "  \"global_step\": 753\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005401611328125,\n",
            "  \"model.lr\": 0.000150648,\n",
            "  \"model.grad_norm\": 0.518335223197937,\n",
            "  \"model.elapsed_time\": 0.06729412078857422,\n",
            "  \"model.engine_step\": 754,\n",
            "  \"model.loss.nll\": 0.005401611328125,\n",
            "  \"elapsed_time\": 0.06729412078857422,\n",
            "  \"wall_time\": 1674064277.5030758,\n",
            "  \"global_step\": 754\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00199127197265625,\n",
            "  \"model.lr\": 0.00015084700000000001,\n",
            "  \"model.grad_norm\": 0.07197505980730057,\n",
            "  \"model.elapsed_time\": 0.07254242897033691,\n",
            "  \"model.engine_step\": 755,\n",
            "  \"model.loss.nll\": 0.00199127197265625,\n",
            "  \"elapsed_time\": 0.07254242897033691,\n",
            "  \"wall_time\": 1674064277.5842588,\n",
            "  \"global_step\": 755\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00214385986328125,\n",
            "  \"model.lr\": 0.000151046,\n",
            "  \"model.grad_norm\": 0.102442666888237,\n",
            "  \"model.elapsed_time\": 0.06591200828552246,\n",
            "  \"model.engine_step\": 756,\n",
            "  \"model.loss.nll\": 0.00214385986328125,\n",
            "  \"elapsed_time\": 0.06591200828552246,\n",
            "  \"wall_time\": 1674064277.6522095,\n",
            "  \"global_step\": 756\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2099609375,\n",
            "  \"model.lr\": 0.00015124500000000002,\n",
            "  \"model.grad_norm\": 3.9458494186401367,\n",
            "  \"model.elapsed_time\": 0.06554198265075684,\n",
            "  \"model.engine_step\": 757,\n",
            "  \"model.loss.nll\": 0.2099609375,\n",
            "  \"elapsed_time\": 0.06554198265075684,\n",
            "  \"wall_time\": 1674064277.725267,\n",
            "  \"global_step\": 757\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0180816650390625,\n",
            "  \"model.lr\": 0.000151444,\n",
            "  \"model.grad_norm\": 0.7848052382469177,\n",
            "  \"model.elapsed_time\": 0.07838153839111328,\n",
            "  \"model.engine_step\": 758,\n",
            "  \"model.loss.nll\": 0.0180816650390625,\n",
            "  \"elapsed_time\": 0.07838153839111328,\n",
            "  \"wall_time\": 1674064277.8056886,\n",
            "  \"global_step\": 758\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0035915374755859375,\n",
            "  \"model.lr\": 0.000151643,\n",
            "  \"model.grad_norm\": 0.2093065083026886,\n",
            "  \"model.elapsed_time\": 0.06633901596069336,\n",
            "  \"model.engine_step\": 759,\n",
            "  \"model.loss.nll\": 0.0035915374755859375,\n",
            "  \"elapsed_time\": 0.06633901596069336,\n",
            "  \"wall_time\": 1674064277.8800251,\n",
            "  \"global_step\": 759\n",
            "}\n",
            "[2023-01-18 17:51:17,951] [INFO] [logging.py:68:log_dist] [Rank 0] step=760, skipped=1, lr=[0.00015184200000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:17,952] [INFO] [timer.py:196:stop] epoch=0/micro_step=760/global_step=760, RunningAvgSamplesPerSec=13.59435856956746, CurrSamplesPerSec=14.172387810061869, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.018768310546875,\n",
            "  \"model.lr\": 0.00015184200000000001,\n",
            "  \"model.grad_norm\": 1.2461810111999512,\n",
            "  \"model.elapsed_time\": 0.07127118110656738,\n",
            "  \"model.engine_step\": 760,\n",
            "  \"model.loss.nll\": 0.018768310546875,\n",
            "  \"elapsed_time\": 0.07127118110656738,\n",
            "  \"wall_time\": 1674064277.9540162,\n",
            "  \"global_step\": 760\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02081298828125,\n",
            "  \"model.lr\": 0.000152041,\n",
            "  \"model.grad_norm\": 1.2482985258102417,\n",
            "  \"model.elapsed_time\": 0.06709671020507812,\n",
            "  \"model.engine_step\": 761,\n",
            "  \"model.loss.nll\": 0.02081298828125,\n",
            "  \"elapsed_time\": 0.06709671020507812,\n",
            "  \"wall_time\": 1674064278.0299945,\n",
            "  \"global_step\": 761\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0921630859375,\n",
            "  \"model.lr\": 0.00015224000000000002,\n",
            "  \"model.grad_norm\": 2.3487799167633057,\n",
            "  \"model.elapsed_time\": 0.07611584663391113,\n",
            "  \"model.engine_step\": 762,\n",
            "  \"model.loss.nll\": 0.0921630859375,\n",
            "  \"elapsed_time\": 0.07611584663391113,\n",
            "  \"wall_time\": 1674064278.1080894,\n",
            "  \"global_step\": 762\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.040130615234375,\n",
            "  \"model.lr\": 0.000152439,\n",
            "  \"model.grad_norm\": 1.8546298742294312,\n",
            "  \"model.elapsed_time\": 0.07584691047668457,\n",
            "  \"model.engine_step\": 763,\n",
            "  \"model.loss.nll\": 0.040130615234375,\n",
            "  \"elapsed_time\": 0.07584691047668457,\n",
            "  \"wall_time\": 1674064278.1916564,\n",
            "  \"global_step\": 763\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00855255126953125,\n",
            "  \"model.lr\": 0.000152638,\n",
            "  \"model.grad_norm\": 0.30706876516342163,\n",
            "  \"model.elapsed_time\": 0.06374120712280273,\n",
            "  \"model.engine_step\": 764,\n",
            "  \"model.loss.nll\": 0.00855255126953125,\n",
            "  \"elapsed_time\": 0.06374120712280273,\n",
            "  \"wall_time\": 1674064278.2573845,\n",
            "  \"global_step\": 764\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01485443115234375,\n",
            "  \"model.lr\": 0.00015283700000000001,\n",
            "  \"model.grad_norm\": 0.9012272953987122,\n",
            "  \"model.elapsed_time\": 0.07329201698303223,\n",
            "  \"model.engine_step\": 765,\n",
            "  \"model.loss.nll\": 0.01485443115234375,\n",
            "  \"elapsed_time\": 0.07329201698303223,\n",
            "  \"wall_time\": 1674064278.3391807,\n",
            "  \"global_step\": 765\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002346038818359375,\n",
            "  \"model.lr\": 0.000153036,\n",
            "  \"model.grad_norm\": 0.100569948554039,\n",
            "  \"model.elapsed_time\": 0.07148218154907227,\n",
            "  \"model.engine_step\": 766,\n",
            "  \"model.loss.nll\": 0.002346038818359375,\n",
            "  \"elapsed_time\": 0.07148218154907227,\n",
            "  \"wall_time\": 1674064278.412717,\n",
            "  \"global_step\": 766\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00202178955078125,\n",
            "  \"model.lr\": 0.00015323500000000002,\n",
            "  \"model.grad_norm\": 0.07198721170425415,\n",
            "  \"model.elapsed_time\": 0.06771397590637207,\n",
            "  \"model.engine_step\": 767,\n",
            "  \"model.loss.nll\": 0.00202178955078125,\n",
            "  \"elapsed_time\": 0.06771397590637207,\n",
            "  \"wall_time\": 1674064278.488942,\n",
            "  \"global_step\": 767\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002063751220703125,\n",
            "  \"model.lr\": 0.000153434,\n",
            "  \"model.grad_norm\": 0.08900104463100433,\n",
            "  \"model.elapsed_time\": 0.06425261497497559,\n",
            "  \"model.engine_step\": 768,\n",
            "  \"model.loss.nll\": 0.002063751220703125,\n",
            "  \"elapsed_time\": 0.06425261497497559,\n",
            "  \"wall_time\": 1674064278.5550945,\n",
            "  \"global_step\": 768\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06353759765625,\n",
            "  \"model.lr\": 0.000153633,\n",
            "  \"model.grad_norm\": 1.891349196434021,\n",
            "  \"model.elapsed_time\": 0.07287907600402832,\n",
            "  \"model.engine_step\": 769,\n",
            "  \"model.loss.nll\": 0.06353759765625,\n",
            "  \"elapsed_time\": 0.07287907600402832,\n",
            "  \"wall_time\": 1674064278.6351514,\n",
            "  \"global_step\": 769\n",
            "}\n",
            "[2023-01-18 17:51:18,704] [INFO] [logging.py:68:log_dist] [Rank 0] step=770, skipped=1, lr=[0.00015383200000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:18,706] [INFO] [timer.py:196:stop] epoch=0/micro_step=770/global_step=770, RunningAvgSamplesPerSec=13.602594066639474, CurrSamplesPerSec=14.33175925482987, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0057830810546875,\n",
            "  \"model.lr\": 0.00015383200000000001,\n",
            "  \"model.grad_norm\": 0.38281482458114624,\n",
            "  \"model.elapsed_time\": 0.07023406028747559,\n",
            "  \"model.engine_step\": 770,\n",
            "  \"model.loss.nll\": 0.0057830810546875,\n",
            "  \"elapsed_time\": 0.07023406028747559,\n",
            "  \"wall_time\": 1674064278.7073848,\n",
            "  \"global_step\": 770\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0017986297607421875,\n",
            "  \"model.lr\": 0.000154031,\n",
            "  \"model.grad_norm\": 0.06405017524957657,\n",
            "  \"model.elapsed_time\": 0.07048225402832031,\n",
            "  \"model.engine_step\": 771,\n",
            "  \"model.loss.nll\": 0.0017986297607421875,\n",
            "  \"elapsed_time\": 0.07048225402832031,\n",
            "  \"wall_time\": 1674064278.7853181,\n",
            "  \"global_step\": 771\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0018672943115234375,\n",
            "  \"model.lr\": 0.00015423000000000002,\n",
            "  \"model.grad_norm\": 0.05342743173241615,\n",
            "  \"model.elapsed_time\": 0.0678246021270752,\n",
            "  \"model.engine_step\": 772,\n",
            "  \"model.loss.nll\": 0.0018672943115234375,\n",
            "  \"elapsed_time\": 0.0678246021270752,\n",
            "  \"wall_time\": 1674064278.8550885,\n",
            "  \"global_step\": 772\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.004306793212890625,\n",
            "  \"model.lr\": 0.000154429,\n",
            "  \"model.grad_norm\": 0.2815978527069092,\n",
            "  \"model.elapsed_time\": 0.07497620582580566,\n",
            "  \"model.engine_step\": 773,\n",
            "  \"model.loss.nll\": 0.004306793212890625,\n",
            "  \"elapsed_time\": 0.07497620582580566,\n",
            "  \"wall_time\": 1674064278.9373493,\n",
            "  \"global_step\": 773\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0023746490478515625,\n",
            "  \"model.lr\": 0.000154628,\n",
            "  \"model.grad_norm\": 0.0927852988243103,\n",
            "  \"model.elapsed_time\": 0.06779146194458008,\n",
            "  \"model.engine_step\": 774,\n",
            "  \"model.loss.nll\": 0.0023746490478515625,\n",
            "  \"elapsed_time\": 0.06779146194458008,\n",
            "  \"wall_time\": 1674064279.0078154,\n",
            "  \"global_step\": 774\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002597808837890625,\n",
            "  \"model.lr\": 0.00015482700000000001,\n",
            "  \"model.grad_norm\": 0.17345930635929108,\n",
            "  \"model.elapsed_time\": 0.06602883338928223,\n",
            "  \"model.engine_step\": 775,\n",
            "  \"model.loss.nll\": 0.002597808837890625,\n",
            "  \"elapsed_time\": 0.06602883338928223,\n",
            "  \"wall_time\": 1674064279.080798,\n",
            "  \"global_step\": 775\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.033050537109375,\n",
            "  \"model.lr\": 0.000155026,\n",
            "  \"model.grad_norm\": 1.3643585443496704,\n",
            "  \"model.elapsed_time\": 0.06461143493652344,\n",
            "  \"model.engine_step\": 776,\n",
            "  \"model.loss.nll\": 0.033050537109375,\n",
            "  \"elapsed_time\": 0.06461143493652344,\n",
            "  \"wall_time\": 1674064279.1482527,\n",
            "  \"global_step\": 776\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002910614013671875,\n",
            "  \"model.lr\": 0.00015522500000000002,\n",
            "  \"model.grad_norm\": 0.153623566031456,\n",
            "  \"model.elapsed_time\": 0.0873253345489502,\n",
            "  \"model.engine_step\": 777,\n",
            "  \"model.loss.nll\": 0.002910614013671875,\n",
            "  \"elapsed_time\": 0.0873253345489502,\n",
            "  \"wall_time\": 1674064279.2431319,\n",
            "  \"global_step\": 777\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0019130706787109375,\n",
            "  \"model.lr\": 0.000155424,\n",
            "  \"model.grad_norm\": 0.09866287559270859,\n",
            "  \"model.elapsed_time\": 0.06891822814941406,\n",
            "  \"model.engine_step\": 778,\n",
            "  \"model.loss.nll\": 0.0019130706787109375,\n",
            "  \"elapsed_time\": 0.06891822814941406,\n",
            "  \"wall_time\": 1674064279.3150094,\n",
            "  \"global_step\": 778\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02294921875,\n",
            "  \"model.lr\": 0.00015562300000000002,\n",
            "  \"model.grad_norm\": 1.5820597410202026,\n",
            "  \"model.elapsed_time\": 0.07178378105163574,\n",
            "  \"model.engine_step\": 779,\n",
            "  \"model.loss.nll\": 0.02294921875,\n",
            "  \"elapsed_time\": 0.07178378105163574,\n",
            "  \"wall_time\": 1674064279.3964436,\n",
            "  \"global_step\": 779\n",
            "}\n",
            "[2023-01-18 17:51:19,470] [INFO] [logging.py:68:log_dist] [Rank 0] step=780, skipped=1, lr=[0.000155822], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:19,472] [INFO] [timer.py:196:stop] epoch=0/micro_step=780/global_step=780, RunningAvgSamplesPerSec=13.607734915997009, CurrSamplesPerSec=13.388654623922266, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0012331008911132812,\n",
            "  \"model.lr\": 0.000155822,\n",
            "  \"model.grad_norm\": 0.04574034363031387,\n",
            "  \"model.elapsed_time\": 0.07517433166503906,\n",
            "  \"model.engine_step\": 780,\n",
            "  \"model.loss.nll\": 0.0012331008911132812,\n",
            "  \"elapsed_time\": 0.07517433166503906,\n",
            "  \"wall_time\": 1674064279.4736784,\n",
            "  \"global_step\": 780\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002437591552734375,\n",
            "  \"model.lr\": 0.000156021,\n",
            "  \"model.grad_norm\": 0.1758197844028473,\n",
            "  \"model.elapsed_time\": 0.06790685653686523,\n",
            "  \"model.engine_step\": 781,\n",
            "  \"model.loss.nll\": 0.002437591552734375,\n",
            "  \"elapsed_time\": 0.06790685653686523,\n",
            "  \"wall_time\": 1674064279.5491776,\n",
            "  \"global_step\": 781\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010986328125,\n",
            "  \"model.lr\": 0.00015622000000000002,\n",
            "  \"model.grad_norm\": 0.020228909328579903,\n",
            "  \"model.elapsed_time\": 0.07309365272521973,\n",
            "  \"model.engine_step\": 782,\n",
            "  \"model.loss.nll\": 0.0010986328125,\n",
            "  \"elapsed_time\": 0.07309365272521973,\n",
            "  \"wall_time\": 1674064279.6251228,\n",
            "  \"global_step\": 782\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0087432861328125,\n",
            "  \"model.lr\": 0.000156419,\n",
            "  \"model.grad_norm\": 0.3294674754142761,\n",
            "  \"model.elapsed_time\": 0.07824563980102539,\n",
            "  \"model.engine_step\": 783,\n",
            "  \"model.loss.nll\": 0.0087432861328125,\n",
            "  \"elapsed_time\": 0.07824563980102539,\n",
            "  \"wall_time\": 1674064279.710701,\n",
            "  \"global_step\": 783\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0014028549194335938,\n",
            "  \"model.lr\": 0.00015661800000000002,\n",
            "  \"model.grad_norm\": 0.03424568101763725,\n",
            "  \"model.elapsed_time\": 0.07207489013671875,\n",
            "  \"model.engine_step\": 784,\n",
            "  \"model.loss.nll\": 0.0014028549194335938,\n",
            "  \"elapsed_time\": 0.07207489013671875,\n",
            "  \"wall_time\": 1674064279.785735,\n",
            "  \"global_step\": 784\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.007251739501953125,\n",
            "  \"model.lr\": 0.000156817,\n",
            "  \"model.grad_norm\": 0.9611793756484985,\n",
            "  \"model.elapsed_time\": 0.0648951530456543,\n",
            "  \"model.engine_step\": 785,\n",
            "  \"model.loss.nll\": 0.007251739501953125,\n",
            "  \"elapsed_time\": 0.0648951530456543,\n",
            "  \"wall_time\": 1674064279.8582003,\n",
            "  \"global_step\": 785\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0015268325805664062,\n",
            "  \"model.lr\": 0.000157016,\n",
            "  \"model.grad_norm\": 0.06073492765426636,\n",
            "  \"model.elapsed_time\": 0.07234907150268555,\n",
            "  \"model.engine_step\": 786,\n",
            "  \"model.loss.nll\": 0.0015268325805664062,\n",
            "  \"elapsed_time\": 0.07234907150268555,\n",
            "  \"wall_time\": 1674064279.933023,\n",
            "  \"global_step\": 786\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00632476806640625,\n",
            "  \"model.lr\": 0.00015721500000000002,\n",
            "  \"model.grad_norm\": 0.20428283512592316,\n",
            "  \"model.elapsed_time\": 0.0688161849975586,\n",
            "  \"model.engine_step\": 787,\n",
            "  \"model.loss.nll\": 0.00632476806640625,\n",
            "  \"elapsed_time\": 0.0688161849975586,\n",
            "  \"wall_time\": 1674064280.0099955,\n",
            "  \"global_step\": 787\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013055801391601562,\n",
            "  \"model.lr\": 0.000157414,\n",
            "  \"model.grad_norm\": 0.036958128213882446,\n",
            "  \"model.elapsed_time\": 0.06804275512695312,\n",
            "  \"model.engine_step\": 788,\n",
            "  \"model.loss.nll\": 0.0013055801391601562,\n",
            "  \"elapsed_time\": 0.06804275512695312,\n",
            "  \"wall_time\": 1674064280.0799932,\n",
            "  \"global_step\": 788\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010232925415039062,\n",
            "  \"model.lr\": 0.00015761300000000002,\n",
            "  \"model.grad_norm\": 0.024518895894289017,\n",
            "  \"model.elapsed_time\": 0.0664970874786377,\n",
            "  \"model.engine_step\": 789,\n",
            "  \"model.loss.nll\": 0.0010232925415039062,\n",
            "  \"elapsed_time\": 0.0664970874786377,\n",
            "  \"wall_time\": 1674064280.1538918,\n",
            "  \"global_step\": 789\n",
            "}\n",
            "[2023-01-18 17:51:20,238] [INFO] [logging.py:68:log_dist] [Rank 0] step=790, skipped=1, lr=[0.000157812], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:20,239] [INFO] [timer.py:196:stop] epoch=0/micro_step=790/global_step=790, RunningAvgSamplesPerSec=13.612420484061515, CurrSamplesPerSec=11.933197148075863, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.001251220703125,\n",
            "  \"model.lr\": 0.000157812,\n",
            "  \"model.grad_norm\": 0.0350465252995491,\n",
            "  \"model.elapsed_time\": 0.08430933952331543,\n",
            "  \"model.engine_step\": 790,\n",
            "  \"model.loss.nll\": 0.001251220703125,\n",
            "  \"elapsed_time\": 0.08430933952331543,\n",
            "  \"wall_time\": 1674064280.2408926,\n",
            "  \"global_step\": 790\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013494491577148438,\n",
            "  \"model.lr\": 0.000158011,\n",
            "  \"model.grad_norm\": 0.0416562482714653,\n",
            "  \"model.elapsed_time\": 0.07869219779968262,\n",
            "  \"model.engine_step\": 791,\n",
            "  \"model.loss.nll\": 0.0013494491577148438,\n",
            "  \"elapsed_time\": 0.07869219779968262,\n",
            "  \"wall_time\": 1674064280.328165,\n",
            "  \"global_step\": 791\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01708984375,\n",
            "  \"model.lr\": 0.00015821000000000002,\n",
            "  \"model.grad_norm\": 1.1153663396835327,\n",
            "  \"model.elapsed_time\": 0.07093119621276855,\n",
            "  \"model.engine_step\": 792,\n",
            "  \"model.loss.nll\": 0.01708984375,\n",
            "  \"elapsed_time\": 0.07093119621276855,\n",
            "  \"wall_time\": 1674064280.401799,\n",
            "  \"global_step\": 792\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.006099700927734375,\n",
            "  \"model.lr\": 0.000158409,\n",
            "  \"model.grad_norm\": 0.32268407940864563,\n",
            "  \"model.elapsed_time\": 0.06767129898071289,\n",
            "  \"model.engine_step\": 793,\n",
            "  \"model.loss.nll\": 0.006099700927734375,\n",
            "  \"elapsed_time\": 0.06767129898071289,\n",
            "  \"wall_time\": 1674064280.4788332,\n",
            "  \"global_step\": 793\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0011548995971679688,\n",
            "  \"model.lr\": 0.00015860800000000002,\n",
            "  \"model.grad_norm\": 0.03707990050315857,\n",
            "  \"model.elapsed_time\": 0.0693047046661377,\n",
            "  \"model.engine_step\": 794,\n",
            "  \"model.loss.nll\": 0.0011548995971679688,\n",
            "  \"elapsed_time\": 0.0693047046661377,\n",
            "  \"wall_time\": 1674064280.5507624,\n",
            "  \"global_step\": 794\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010700225830078125,\n",
            "  \"model.lr\": 0.000158807,\n",
            "  \"model.grad_norm\": 0.020573850721120834,\n",
            "  \"model.elapsed_time\": 0.07194733619689941,\n",
            "  \"model.engine_step\": 795,\n",
            "  \"model.loss.nll\": 0.0010700225830078125,\n",
            "  \"elapsed_time\": 0.07194733619689941,\n",
            "  \"wall_time\": 1674064280.631436,\n",
            "  \"global_step\": 795\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0038814544677734375,\n",
            "  \"model.lr\": 0.000159006,\n",
            "  \"model.grad_norm\": 0.1043245866894722,\n",
            "  \"model.elapsed_time\": 0.07669997215270996,\n",
            "  \"model.engine_step\": 796,\n",
            "  \"model.loss.nll\": 0.0038814544677734375,\n",
            "  \"elapsed_time\": 0.07669997215270996,\n",
            "  \"wall_time\": 1674064280.7104666,\n",
            "  \"global_step\": 796\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009098052978515625,\n",
            "  \"model.lr\": 0.00015920500000000002,\n",
            "  \"model.grad_norm\": 0.03326129540801048,\n",
            "  \"model.elapsed_time\": 0.07178950309753418,\n",
            "  \"model.engine_step\": 797,\n",
            "  \"model.loss.nll\": 0.0009098052978515625,\n",
            "  \"elapsed_time\": 0.07178950309753418,\n",
            "  \"wall_time\": 1674064280.792885,\n",
            "  \"global_step\": 797\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009026527404785156,\n",
            "  \"model.lr\": 0.000159404,\n",
            "  \"model.grad_norm\": 0.022981589660048485,\n",
            "  \"model.elapsed_time\": 0.06815981864929199,\n",
            "  \"model.engine_step\": 798,\n",
            "  \"model.loss.nll\": 0.0009026527404785156,\n",
            "  \"elapsed_time\": 0.06815981864929199,\n",
            "  \"wall_time\": 1674064280.8632295,\n",
            "  \"global_step\": 798\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010919570922851562,\n",
            "  \"model.lr\": 0.00015960300000000002,\n",
            "  \"model.grad_norm\": 0.02628026157617569,\n",
            "  \"model.elapsed_time\": 0.06559157371520996,\n",
            "  \"model.engine_step\": 799,\n",
            "  \"model.loss.nll\": 0.0010919570922851562,\n",
            "  \"elapsed_time\": 0.06559157371520996,\n",
            "  \"wall_time\": 1674064280.9360704,\n",
            "  \"global_step\": 799\n",
            "}\n",
            "[2023-01-18 17:51:21,009] [INFO] [logging.py:68:log_dist] [Rank 0] step=800, skipped=1, lr=[0.000159802], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:21,011] [INFO] [timer.py:196:stop] epoch=0/micro_step=800/global_step=800, RunningAvgSamplesPerSec=13.617237812326465, CurrSamplesPerSec=13.533112422522578, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0070648193359375,\n",
            "  \"model.lr\": 0.000159802,\n",
            "  \"model.grad_norm\": 0.5371197462081909,\n",
            "  \"model.elapsed_time\": 0.07433724403381348,\n",
            "  \"model.engine_step\": 800,\n",
            "  \"model.loss.nll\": 0.0070648193359375,\n",
            "  \"elapsed_time\": 0.07433724403381348,\n",
            "  \"wall_time\": 1674064281.0123627,\n",
            "  \"global_step\": 800\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013055801391601562,\n",
            "  \"model.lr\": 0.00016000100000000003,\n",
            "  \"model.grad_norm\": 0.06954418122768402,\n",
            "  \"model.elapsed_time\": 0.0697476863861084,\n",
            "  \"model.engine_step\": 801,\n",
            "  \"model.loss.nll\": 0.0013055801391601562,\n",
            "  \"elapsed_time\": 0.0697476863861084,\n",
            "  \"wall_time\": 1674064281.0899992,\n",
            "  \"global_step\": 801\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0011262893676757812,\n",
            "  \"model.lr\": 0.00016020000000000002,\n",
            "  \"model.grad_norm\": 0.03563525900244713,\n",
            "  \"model.elapsed_time\": 0.06627202033996582,\n",
            "  \"model.engine_step\": 802,\n",
            "  \"model.loss.nll\": 0.0011262893676757812,\n",
            "  \"elapsed_time\": 0.06627202033996582,\n",
            "  \"wall_time\": 1674064281.1582747,\n",
            "  \"global_step\": 802\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005039215087890625,\n",
            "  \"model.lr\": 0.000160399,\n",
            "  \"model.grad_norm\": 0.1552809178829193,\n",
            "  \"model.elapsed_time\": 0.07069134712219238,\n",
            "  \"model.engine_step\": 803,\n",
            "  \"model.loss.nll\": 0.005039215087890625,\n",
            "  \"elapsed_time\": 0.07069134712219238,\n",
            "  \"wall_time\": 1674064281.236191,\n",
            "  \"global_step\": 803\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00087738037109375,\n",
            "  \"model.lr\": 0.00016059800000000002,\n",
            "  \"model.grad_norm\": 0.01901860162615776,\n",
            "  \"model.elapsed_time\": 0.06899571418762207,\n",
            "  \"model.engine_step\": 804,\n",
            "  \"model.loss.nll\": 0.00087738037109375,\n",
            "  \"elapsed_time\": 0.06899571418762207,\n",
            "  \"wall_time\": 1674064281.3081958,\n",
            "  \"global_step\": 804\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008182525634765625,\n",
            "  \"model.lr\": 0.000160797,\n",
            "  \"model.grad_norm\": 0.014711734838783741,\n",
            "  \"model.elapsed_time\": 0.0667273998260498,\n",
            "  \"model.engine_step\": 805,\n",
            "  \"model.loss.nll\": 0.0008182525634765625,\n",
            "  \"elapsed_time\": 0.0667273998260498,\n",
            "  \"wall_time\": 1674064281.382005,\n",
            "  \"global_step\": 805\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010576248168945312,\n",
            "  \"model.lr\": 0.00016099600000000003,\n",
            "  \"model.grad_norm\": 0.03535635024309158,\n",
            "  \"model.elapsed_time\": 0.07411551475524902,\n",
            "  \"model.engine_step\": 806,\n",
            "  \"model.loss.nll\": 0.0010576248168945312,\n",
            "  \"elapsed_time\": 0.07411551475524902,\n",
            "  \"wall_time\": 1674064281.4588053,\n",
            "  \"global_step\": 806\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008082389831542969,\n",
            "  \"model.lr\": 0.00016119500000000002,\n",
            "  \"model.grad_norm\": 0.011496180668473244,\n",
            "  \"model.elapsed_time\": 0.06814908981323242,\n",
            "  \"model.engine_step\": 807,\n",
            "  \"model.loss.nll\": 0.0008082389831542969,\n",
            "  \"elapsed_time\": 0.06814908981323242,\n",
            "  \"wall_time\": 1674064281.5353177,\n",
            "  \"global_step\": 807\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008945465087890625,\n",
            "  \"model.lr\": 0.000161394,\n",
            "  \"model.grad_norm\": 0.016027376055717468,\n",
            "  \"model.elapsed_time\": 0.06672525405883789,\n",
            "  \"model.engine_step\": 808,\n",
            "  \"model.loss.nll\": 0.0008945465087890625,\n",
            "  \"elapsed_time\": 0.06672525405883789,\n",
            "  \"wall_time\": 1674064281.604015,\n",
            "  \"global_step\": 808\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008406639099121094,\n",
            "  \"model.lr\": 0.00016159300000000002,\n",
            "  \"model.grad_norm\": 0.013996999710798264,\n",
            "  \"model.elapsed_time\": 0.06987833976745605,\n",
            "  \"model.engine_step\": 809,\n",
            "  \"model.loss.nll\": 0.0008406639099121094,\n",
            "  \"elapsed_time\": 0.06987833976745605,\n",
            "  \"wall_time\": 1674064281.681723,\n",
            "  \"global_step\": 809\n",
            "}\n",
            "[2023-01-18 17:51:21,750] [INFO] [logging.py:68:log_dist] [Rank 0] step=810, skipped=1, lr=[0.000161792], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:21,752] [INFO] [timer.py:196:stop] epoch=0/micro_step=810/global_step=810, RunningAvgSamplesPerSec=13.62749886593918, CurrSamplesPerSec=14.443593487423897, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010013580322265625,\n",
            "  \"model.lr\": 0.000161792,\n",
            "  \"model.grad_norm\": 0.034952033311128616,\n",
            "  \"model.elapsed_time\": 0.0696713924407959,\n",
            "  \"model.engine_step\": 810,\n",
            "  \"model.loss.nll\": 0.0010013580322265625,\n",
            "  \"elapsed_time\": 0.0696713924407959,\n",
            "  \"wall_time\": 1674064281.7533782,\n",
            "  \"global_step\": 810\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007886886596679688,\n",
            "  \"model.lr\": 0.00016199100000000003,\n",
            "  \"model.grad_norm\": 0.009807390160858631,\n",
            "  \"model.elapsed_time\": 0.07273173332214355,\n",
            "  \"model.engine_step\": 811,\n",
            "  \"model.loss.nll\": 0.0007886886596679688,\n",
            "  \"elapsed_time\": 0.07273173332214355,\n",
            "  \"wall_time\": 1674064281.8334498,\n",
            "  \"global_step\": 811\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.014495849609375,\n",
            "  \"model.lr\": 0.00016219000000000002,\n",
            "  \"model.grad_norm\": 1.1735060214996338,\n",
            "  \"model.elapsed_time\": 0.0717160701751709,\n",
            "  \"model.engine_step\": 812,\n",
            "  \"model.loss.nll\": 0.014495849609375,\n",
            "  \"elapsed_time\": 0.0717160701751709,\n",
            "  \"wall_time\": 1674064281.9072526,\n",
            "  \"global_step\": 812\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007963180541992188,\n",
            "  \"model.lr\": 0.000162389,\n",
            "  \"model.grad_norm\": 0.017559044063091278,\n",
            "  \"model.elapsed_time\": 0.07048606872558594,\n",
            "  \"model.engine_step\": 813,\n",
            "  \"model.loss.nll\": 0.0007963180541992188,\n",
            "  \"elapsed_time\": 0.07048606872558594,\n",
            "  \"wall_time\": 1674064281.992002,\n",
            "  \"global_step\": 813\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008382797241210938,\n",
            "  \"model.lr\": 0.00016258800000000002,\n",
            "  \"model.grad_norm\": 0.01942954771220684,\n",
            "  \"model.elapsed_time\": 0.07192468643188477,\n",
            "  \"model.engine_step\": 814,\n",
            "  \"model.loss.nll\": 0.0008382797241210938,\n",
            "  \"elapsed_time\": 0.07192468643188477,\n",
            "  \"wall_time\": 1674064282.0659552,\n",
            "  \"global_step\": 814\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008678436279296875,\n",
            "  \"model.lr\": 0.00016278699999999998,\n",
            "  \"model.grad_norm\": 0.019614966586232185,\n",
            "  \"model.elapsed_time\": 0.07382583618164062,\n",
            "  \"model.engine_step\": 815,\n",
            "  \"model.loss.nll\": 0.0008678436279296875,\n",
            "  \"elapsed_time\": 0.07382583618164062,\n",
            "  \"wall_time\": 1674064282.1472886,\n",
            "  \"global_step\": 815\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0012073516845703125,\n",
            "  \"model.lr\": 0.000162986,\n",
            "  \"model.grad_norm\": 0.04364334046840668,\n",
            "  \"model.elapsed_time\": 0.06787681579589844,\n",
            "  \"model.engine_step\": 816,\n",
            "  \"model.loss.nll\": 0.0012073516845703125,\n",
            "  \"elapsed_time\": 0.06787681579589844,\n",
            "  \"wall_time\": 1674064282.2177093,\n",
            "  \"global_step\": 816\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0033931732177734375,\n",
            "  \"model.lr\": 0.000163185,\n",
            "  \"model.grad_norm\": 0.45527517795562744,\n",
            "  \"model.elapsed_time\": 0.07704520225524902,\n",
            "  \"model.engine_step\": 817,\n",
            "  \"model.loss.nll\": 0.0033931732177734375,\n",
            "  \"elapsed_time\": 0.07704520225524902,\n",
            "  \"wall_time\": 1674064282.302579,\n",
            "  \"global_step\": 817\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00133514404296875,\n",
            "  \"model.lr\": 0.000163384,\n",
            "  \"model.grad_norm\": 0.1283542662858963,\n",
            "  \"model.elapsed_time\": 0.06682682037353516,\n",
            "  \"model.engine_step\": 818,\n",
            "  \"model.loss.nll\": 0.00133514404296875,\n",
            "  \"elapsed_time\": 0.06682682037353516,\n",
            "  \"wall_time\": 1674064282.3721027,\n",
            "  \"global_step\": 818\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00392913818359375,\n",
            "  \"model.lr\": 0.000163583,\n",
            "  \"model.grad_norm\": 0.22094310820102692,\n",
            "  \"model.elapsed_time\": 0.0801999568939209,\n",
            "  \"model.engine_step\": 819,\n",
            "  \"model.loss.nll\": 0.00392913818359375,\n",
            "  \"elapsed_time\": 0.0801999568939209,\n",
            "  \"wall_time\": 1674064282.4612858,\n",
            "  \"global_step\": 819\n",
            "}\n",
            "[2023-01-18 17:51:22,532] [INFO] [logging.py:68:log_dist] [Rank 0] step=820, skipped=1, lr=[0.00016378199999999998], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:22,534] [INFO] [timer.py:196:stop] epoch=0/micro_step=820/global_step=820, RunningAvgSamplesPerSec=13.629880777955245, CurrSamplesPerSec=13.979289220996078, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007567405700683594,\n",
            "  \"model.lr\": 0.00016378199999999998,\n",
            "  \"model.grad_norm\": 0.017056085169315338,\n",
            "  \"model.elapsed_time\": 0.07201290130615234,\n",
            "  \"model.engine_step\": 820,\n",
            "  \"model.loss.nll\": 0.0007567405700683594,\n",
            "  \"elapsed_time\": 0.07201290130615234,\n",
            "  \"wall_time\": 1674064282.5354104,\n",
            "  \"global_step\": 820\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007410049438476562,\n",
            "  \"model.lr\": 0.000163981,\n",
            "  \"model.grad_norm\": 0.016239101067185402,\n",
            "  \"model.elapsed_time\": 0.07158970832824707,\n",
            "  \"model.engine_step\": 821,\n",
            "  \"model.loss.nll\": 0.0007410049438476562,\n",
            "  \"elapsed_time\": 0.07158970832824707,\n",
            "  \"wall_time\": 1674064282.6157715,\n",
            "  \"global_step\": 821\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007371902465820312,\n",
            "  \"model.lr\": 0.00016418,\n",
            "  \"model.grad_norm\": 0.012620486319065094,\n",
            "  \"model.elapsed_time\": 0.07011532783508301,\n",
            "  \"model.engine_step\": 822,\n",
            "  \"model.loss.nll\": 0.0007371902465820312,\n",
            "  \"elapsed_time\": 0.07011532783508301,\n",
            "  \"wall_time\": 1674064282.687958,\n",
            "  \"global_step\": 822\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.005237579345703125,\n",
            "  \"model.lr\": 0.000164379,\n",
            "  \"model.grad_norm\": 0.38691815733909607,\n",
            "  \"model.elapsed_time\": 0.07503795623779297,\n",
            "  \"model.engine_step\": 823,\n",
            "  \"model.loss.nll\": 0.005237579345703125,\n",
            "  \"elapsed_time\": 0.07503795623779297,\n",
            "  \"wall_time\": 1674064282.7704263,\n",
            "  \"global_step\": 823\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007061958312988281,\n",
            "  \"model.lr\": 0.000164578,\n",
            "  \"model.grad_norm\": 0.017608586698770523,\n",
            "  \"model.elapsed_time\": 0.07146215438842773,\n",
            "  \"model.engine_step\": 824,\n",
            "  \"model.loss.nll\": 0.0007061958312988281,\n",
            "  \"elapsed_time\": 0.07146215438842773,\n",
            "  \"wall_time\": 1674064282.8446164,\n",
            "  \"global_step\": 824\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013132095336914062,\n",
            "  \"model.lr\": 0.000164777,\n",
            "  \"model.grad_norm\": 0.14204685389995575,\n",
            "  \"model.elapsed_time\": 0.07453083992004395,\n",
            "  \"model.engine_step\": 825,\n",
            "  \"model.loss.nll\": 0.0013132095336914062,\n",
            "  \"elapsed_time\": 0.07453083992004395,\n",
            "  \"wall_time\": 1674064282.9279044,\n",
            "  \"global_step\": 825\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002452850341796875,\n",
            "  \"model.lr\": 0.000164976,\n",
            "  \"model.grad_norm\": 0.05508125573396683,\n",
            "  \"model.elapsed_time\": 0.08994483947753906,\n",
            "  \"model.engine_step\": 826,\n",
            "  \"model.loss.nll\": 0.002452850341796875,\n",
            "  \"elapsed_time\": 0.08994483947753906,\n",
            "  \"wall_time\": 1674064283.0231462,\n",
            "  \"global_step\": 826\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01678466796875,\n",
            "  \"model.lr\": 0.000165175,\n",
            "  \"model.grad_norm\": 1.0805805921554565,\n",
            "  \"model.elapsed_time\": 0.07024407386779785,\n",
            "  \"model.engine_step\": 827,\n",
            "  \"model.loss.nll\": 0.01678466796875,\n",
            "  \"elapsed_time\": 0.07024407386779785,\n",
            "  \"wall_time\": 1674064283.1021245,\n",
            "  \"global_step\": 827\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007181167602539062,\n",
            "  \"model.lr\": 0.000165374,\n",
            "  \"model.grad_norm\": 0.008976507000625134,\n",
            "  \"model.elapsed_time\": 0.06977391242980957,\n",
            "  \"model.engine_step\": 828,\n",
            "  \"model.loss.nll\": 0.0007181167602539062,\n",
            "  \"elapsed_time\": 0.06977391242980957,\n",
            "  \"wall_time\": 1674064283.174601,\n",
            "  \"global_step\": 828\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0028553009033203125,\n",
            "  \"model.lr\": 0.000165573,\n",
            "  \"model.grad_norm\": 0.07467638701200485,\n",
            "  \"model.elapsed_time\": 0.0795450210571289,\n",
            "  \"model.engine_step\": 829,\n",
            "  \"model.loss.nll\": 0.0028553009033203125,\n",
            "  \"elapsed_time\": 0.0795450210571289,\n",
            "  \"wall_time\": 1674064283.2615137,\n",
            "  \"global_step\": 829\n",
            "}\n",
            "[2023-01-18 17:51:23,339] [INFO] [logging.py:68:log_dist] [Rank 0] step=830, skipped=1, lr=[0.000165772], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:23,340] [INFO] [timer.py:196:stop] epoch=0/micro_step=830/global_step=830, RunningAvgSamplesPerSec=13.626570223469423, CurrSamplesPerSec=12.978028064421308, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006313323974609375,\n",
            "  \"model.lr\": 0.000165772,\n",
            "  \"model.grad_norm\": 0.009306713007390499,\n",
            "  \"model.elapsed_time\": 0.0775308609008789,\n",
            "  \"model.engine_step\": 830,\n",
            "  \"model.loss.nll\": 0.0006313323974609375,\n",
            "  \"elapsed_time\": 0.0775308609008789,\n",
            "  \"wall_time\": 1674064283.342083,\n",
            "  \"global_step\": 830\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00203704833984375,\n",
            "  \"model.lr\": 0.000165971,\n",
            "  \"model.grad_norm\": 0.068873830139637,\n",
            "  \"model.elapsed_time\": 0.07012081146240234,\n",
            "  \"model.engine_step\": 831,\n",
            "  \"model.loss.nll\": 0.00203704833984375,\n",
            "  \"elapsed_time\": 0.07012081146240234,\n",
            "  \"wall_time\": 1674064283.4202359,\n",
            "  \"global_step\": 831\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008220672607421875,\n",
            "  \"model.lr\": 0.00016617,\n",
            "  \"model.grad_norm\": 0.06086619570851326,\n",
            "  \"model.elapsed_time\": 0.07313680648803711,\n",
            "  \"model.engine_step\": 832,\n",
            "  \"model.loss.nll\": 0.0008220672607421875,\n",
            "  \"elapsed_time\": 0.07313680648803711,\n",
            "  \"wall_time\": 1674064283.495886,\n",
            "  \"global_step\": 832\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00077056884765625,\n",
            "  \"model.lr\": 0.000166369,\n",
            "  \"model.grad_norm\": 0.030941225588321686,\n",
            "  \"model.elapsed_time\": 0.0682828426361084,\n",
            "  \"model.engine_step\": 833,\n",
            "  \"model.loss.nll\": 0.00077056884765625,\n",
            "  \"elapsed_time\": 0.0682828426361084,\n",
            "  \"wall_time\": 1674064283.5729575,\n",
            "  \"global_step\": 833\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007162094116210938,\n",
            "  \"model.lr\": 0.000166568,\n",
            "  \"model.grad_norm\": 0.012392725795507431,\n",
            "  \"model.elapsed_time\": 0.06719470024108887,\n",
            "  \"model.engine_step\": 834,\n",
            "  \"model.loss.nll\": 0.0007162094116210938,\n",
            "  \"elapsed_time\": 0.06719470024108887,\n",
            "  \"wall_time\": 1674064283.642434,\n",
            "  \"global_step\": 834\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007119178771972656,\n",
            "  \"model.lr\": 0.000166767,\n",
            "  \"model.grad_norm\": 0.021053556352853775,\n",
            "  \"model.elapsed_time\": 0.06966423988342285,\n",
            "  \"model.engine_step\": 835,\n",
            "  \"model.loss.nll\": 0.0007119178771972656,\n",
            "  \"elapsed_time\": 0.06966423988342285,\n",
            "  \"wall_time\": 1674064283.720007,\n",
            "  \"global_step\": 835\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0011148452758789062,\n",
            "  \"model.lr\": 0.000166966,\n",
            "  \"model.grad_norm\": 0.060294393450021744,\n",
            "  \"model.elapsed_time\": 0.07590484619140625,\n",
            "  \"model.engine_step\": 836,\n",
            "  \"model.loss.nll\": 0.0011148452758789062,\n",
            "  \"elapsed_time\": 0.07590484619140625,\n",
            "  \"wall_time\": 1674064283.799097,\n",
            "  \"global_step\": 836\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008893013000488281,\n",
            "  \"model.lr\": 0.000167165,\n",
            "  \"model.grad_norm\": 0.022920211777091026,\n",
            "  \"model.elapsed_time\": 0.06917691230773926,\n",
            "  \"model.engine_step\": 837,\n",
            "  \"model.loss.nll\": 0.0008893013000488281,\n",
            "  \"elapsed_time\": 0.06917691230773926,\n",
            "  \"wall_time\": 1674064283.8777168,\n",
            "  \"global_step\": 837\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0015497207641601562,\n",
            "  \"model.lr\": 0.000167364,\n",
            "  \"model.grad_norm\": 0.10844112187623978,\n",
            "  \"model.elapsed_time\": 0.07287430763244629,\n",
            "  \"model.engine_step\": 838,\n",
            "  \"model.loss.nll\": 0.0015497207641601562,\n",
            "  \"elapsed_time\": 0.07287430763244629,\n",
            "  \"wall_time\": 1674064283.9530697,\n",
            "  \"global_step\": 838\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009937286376953125,\n",
            "  \"model.lr\": 0.000167563,\n",
            "  \"model.grad_norm\": 0.04099626839160919,\n",
            "  \"model.elapsed_time\": 0.06801152229309082,\n",
            "  \"model.engine_step\": 839,\n",
            "  \"model.loss.nll\": 0.0009937286376953125,\n",
            "  \"elapsed_time\": 0.06801152229309082,\n",
            "  \"wall_time\": 1674064284.029427,\n",
            "  \"global_step\": 839\n",
            "}\n",
            "[2023-01-18 17:51:24,102] [INFO] [logging.py:68:log_dist] [Rank 0] step=840, skipped=1, lr=[0.000167762], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:24,104] [INFO] [timer.py:196:stop] epoch=0/micro_step=840/global_step=840, RunningAvgSamplesPerSec=13.63254683649652, CurrSamplesPerSec=13.62747130283349, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00426483154296875,\n",
            "  \"model.lr\": 0.000167762,\n",
            "  \"model.grad_norm\": 0.3527338206768036,\n",
            "  \"model.elapsed_time\": 0.07384109497070312,\n",
            "  \"model.engine_step\": 840,\n",
            "  \"model.loss.nll\": 0.00426483154296875,\n",
            "  \"elapsed_time\": 0.07384109497070312,\n",
            "  \"wall_time\": 1674064284.1054165,\n",
            "  \"global_step\": 840\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008311271667480469,\n",
            "  \"model.lr\": 0.000167961,\n",
            "  \"model.grad_norm\": 0.034592147916555405,\n",
            "  \"model.elapsed_time\": 0.06382346153259277,\n",
            "  \"model.engine_step\": 841,\n",
            "  \"model.loss.nll\": 0.0008311271667480469,\n",
            "  \"elapsed_time\": 0.06382346153259277,\n",
            "  \"wall_time\": 1674064284.1768017,\n",
            "  \"global_step\": 841\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007457733154296875,\n",
            "  \"model.lr\": 0.00016816,\n",
            "  \"model.grad_norm\": 0.021611960604786873,\n",
            "  \"model.elapsed_time\": 0.06979155540466309,\n",
            "  \"model.engine_step\": 842,\n",
            "  \"model.loss.nll\": 0.0007457733154296875,\n",
            "  \"elapsed_time\": 0.06979155540466309,\n",
            "  \"wall_time\": 1674064284.249303,\n",
            "  \"global_step\": 842\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0020694732666015625,\n",
            "  \"model.lr\": 0.000168359,\n",
            "  \"model.grad_norm\": 0.06032758578658104,\n",
            "  \"model.elapsed_time\": 0.0769498348236084,\n",
            "  \"model.engine_step\": 843,\n",
            "  \"model.loss.nll\": 0.0020694732666015625,\n",
            "  \"elapsed_time\": 0.0769498348236084,\n",
            "  \"wall_time\": 1674064284.3337839,\n",
            "  \"global_step\": 843\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007910728454589844,\n",
            "  \"model.lr\": 0.000168558,\n",
            "  \"model.grad_norm\": 0.027792269363999367,\n",
            "  \"model.elapsed_time\": 0.07233691215515137,\n",
            "  \"model.engine_step\": 844,\n",
            "  \"model.loss.nll\": 0.0007910728454589844,\n",
            "  \"elapsed_time\": 0.07233691215515137,\n",
            "  \"wall_time\": 1674064284.4088926,\n",
            "  \"global_step\": 844\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0025424957275390625,\n",
            "  \"model.lr\": 0.000168757,\n",
            "  \"model.grad_norm\": 0.14954859018325806,\n",
            "  \"model.elapsed_time\": 0.07560014724731445,\n",
            "  \"model.engine_step\": 845,\n",
            "  \"model.loss.nll\": 0.0025424957275390625,\n",
            "  \"elapsed_time\": 0.07560014724731445,\n",
            "  \"wall_time\": 1674064284.491491,\n",
            "  \"global_step\": 845\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00096893310546875,\n",
            "  \"model.lr\": 0.000168956,\n",
            "  \"model.grad_norm\": 0.06962725520133972,\n",
            "  \"model.elapsed_time\": 0.07219696044921875,\n",
            "  \"model.engine_step\": 846,\n",
            "  \"model.loss.nll\": 0.00096893310546875,\n",
            "  \"elapsed_time\": 0.07219696044921875,\n",
            "  \"wall_time\": 1674064284.5663767,\n",
            "  \"global_step\": 846\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002033233642578125,\n",
            "  \"model.lr\": 0.000169155,\n",
            "  \"model.grad_norm\": 0.07206287235021591,\n",
            "  \"model.elapsed_time\": 0.0666191577911377,\n",
            "  \"model.engine_step\": 847,\n",
            "  \"model.loss.nll\": 0.002033233642578125,\n",
            "  \"elapsed_time\": 0.0666191577911377,\n",
            "  \"wall_time\": 1674064284.6397974,\n",
            "  \"global_step\": 847\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0011644363403320312,\n",
            "  \"model.lr\": 0.000169354,\n",
            "  \"model.grad_norm\": 0.1391131430864334,\n",
            "  \"model.elapsed_time\": 0.06461000442504883,\n",
            "  \"model.engine_step\": 848,\n",
            "  \"model.loss.nll\": 0.0011644363403320312,\n",
            "  \"elapsed_time\": 0.06461000442504883,\n",
            "  \"wall_time\": 1674064284.7073956,\n",
            "  \"global_step\": 848\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007104873657226562,\n",
            "  \"model.lr\": 0.000169553,\n",
            "  \"model.grad_norm\": 0.01862257346510887,\n",
            "  \"model.elapsed_time\": 0.07414603233337402,\n",
            "  \"model.engine_step\": 849,\n",
            "  \"model.loss.nll\": 0.0007104873657226562,\n",
            "  \"elapsed_time\": 0.07414603233337402,\n",
            "  \"wall_time\": 1674064284.7903304,\n",
            "  \"global_step\": 849\n",
            "}\n",
            "[2023-01-18 17:51:24,866] [INFO] [logging.py:68:log_dist] [Rank 0] step=850, skipped=1, lr=[0.000169752], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:24,868] [INFO] [timer.py:196:stop] epoch=0/micro_step=850/global_step=850, RunningAvgSamplesPerSec=13.637321838828107, CurrSamplesPerSec=13.066041967801425, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005440711975097656,\n",
            "  \"model.lr\": 0.000169752,\n",
            "  \"model.grad_norm\": 0.006814977619796991,\n",
            "  \"model.elapsed_time\": 0.07701802253723145,\n",
            "  \"model.engine_step\": 850,\n",
            "  \"model.loss.nll\": 0.0005440711975097656,\n",
            "  \"elapsed_time\": 0.07701802253723145,\n",
            "  \"wall_time\": 1674064284.8695214,\n",
            "  \"global_step\": 850\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:24\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00055694580078125,\n",
            "  \"model.lr\": 0.000169951,\n",
            "  \"model.grad_norm\": 0.0075829122215509415,\n",
            "  \"model.elapsed_time\": 0.06945180892944336,\n",
            "  \"model.engine_step\": 851,\n",
            "  \"model.loss.nll\": 0.00055694580078125,\n",
            "  \"elapsed_time\": 0.06945180892944336,\n",
            "  \"wall_time\": 1674064284.9494069,\n",
            "  \"global_step\": 851\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009517669677734375,\n",
            "  \"model.lr\": 0.00017015000000000002,\n",
            "  \"model.grad_norm\": 0.03852885589003563,\n",
            "  \"model.elapsed_time\": 0.07021093368530273,\n",
            "  \"model.engine_step\": 852,\n",
            "  \"model.loss.nll\": 0.0009517669677734375,\n",
            "  \"elapsed_time\": 0.07021093368530273,\n",
            "  \"wall_time\": 1674064285.0219264,\n",
            "  \"global_step\": 852\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.003116607666015625,\n",
            "  \"model.lr\": 0.000170349,\n",
            "  \"model.grad_norm\": 0.21874071657657623,\n",
            "  \"model.elapsed_time\": 0.06890606880187988,\n",
            "  \"model.engine_step\": 853,\n",
            "  \"model.loss.nll\": 0.003116607666015625,\n",
            "  \"elapsed_time\": 0.06890606880187988,\n",
            "  \"wall_time\": 1674064285.0975335,\n",
            "  \"global_step\": 853\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007190704345703125,\n",
            "  \"model.lr\": 0.000170548,\n",
            "  \"model.grad_norm\": 0.013308933936059475,\n",
            "  \"model.elapsed_time\": 0.0708918571472168,\n",
            "  \"model.engine_step\": 854,\n",
            "  \"model.loss.nll\": 0.0007190704345703125,\n",
            "  \"elapsed_time\": 0.0708918571472168,\n",
            "  \"wall_time\": 1674064285.171004,\n",
            "  \"global_step\": 854\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0033416748046875,\n",
            "  \"model.lr\": 0.000170747,\n",
            "  \"model.grad_norm\": 0.23275886476039886,\n",
            "  \"model.elapsed_time\": 0.06683802604675293,\n",
            "  \"model.engine_step\": 855,\n",
            "  \"model.loss.nll\": 0.0033416748046875,\n",
            "  \"elapsed_time\": 0.06683802604675293,\n",
            "  \"wall_time\": 1674064285.245625,\n",
            "  \"global_step\": 855\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006961822509765625,\n",
            "  \"model.lr\": 0.000170946,\n",
            "  \"model.grad_norm\": 0.025966523215174675,\n",
            "  \"model.elapsed_time\": 0.08014392852783203,\n",
            "  \"model.engine_step\": 856,\n",
            "  \"model.loss.nll\": 0.0006961822509765625,\n",
            "  \"elapsed_time\": 0.08014392852783203,\n",
            "  \"wall_time\": 1674064285.329638,\n",
            "  \"global_step\": 856\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0015821456909179688,\n",
            "  \"model.lr\": 0.00017114500000000002,\n",
            "  \"model.grad_norm\": 0.027008548378944397,\n",
            "  \"model.elapsed_time\": 0.06776547431945801,\n",
            "  \"model.engine_step\": 857,\n",
            "  \"model.loss.nll\": 0.0015821456909179688,\n",
            "  \"elapsed_time\": 0.06776547431945801,\n",
            "  \"wall_time\": 1674064285.4065554,\n",
            "  \"global_step\": 857\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01446533203125,\n",
            "  \"model.lr\": 0.000171344,\n",
            "  \"model.grad_norm\": 1.149872899055481,\n",
            "  \"model.elapsed_time\": 0.0633084774017334,\n",
            "  \"model.engine_step\": 858,\n",
            "  \"model.loss.nll\": 0.01446533203125,\n",
            "  \"elapsed_time\": 0.0633084774017334,\n",
            "  \"wall_time\": 1674064285.4723725,\n",
            "  \"global_step\": 858\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007805824279785156,\n",
            "  \"model.lr\": 0.000171543,\n",
            "  \"model.grad_norm\": 0.03495882824063301,\n",
            "  \"model.elapsed_time\": 0.07478952407836914,\n",
            "  \"model.engine_step\": 859,\n",
            "  \"model.loss.nll\": 0.0007805824279785156,\n",
            "  \"elapsed_time\": 0.07478952407836914,\n",
            "  \"wall_time\": 1674064285.556141,\n",
            "  \"global_step\": 859\n",
            "}\n",
            "[2023-01-18 17:51:25,629] [INFO] [logging.py:68:log_dist] [Rank 0] step=860, skipped=1, lr=[0.000171742], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:25,630] [INFO] [timer.py:196:stop] epoch=0/micro_step=860/global_step=860, RunningAvgSamplesPerSec=13.643720422250693, CurrSamplesPerSec=13.717589882293687, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005440711975097656,\n",
            "  \"model.lr\": 0.000171742,\n",
            "  \"model.grad_norm\": 0.007896446622908115,\n",
            "  \"model.elapsed_time\": 0.07340073585510254,\n",
            "  \"model.engine_step\": 860,\n",
            "  \"model.loss.nll\": 0.0005440711975097656,\n",
            "  \"elapsed_time\": 0.07340073585510254,\n",
            "  \"wall_time\": 1674064285.6318054,\n",
            "  \"global_step\": 860\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.006988525390625,\n",
            "  \"model.lr\": 0.000171941,\n",
            "  \"model.grad_norm\": 0.598944365978241,\n",
            "  \"model.elapsed_time\": 0.06722807884216309,\n",
            "  \"model.engine_step\": 861,\n",
            "  \"model.loss.nll\": 0.006988525390625,\n",
            "  \"elapsed_time\": 0.06722807884216309,\n",
            "  \"wall_time\": 1674064285.706393,\n",
            "  \"global_step\": 861\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005645751953125,\n",
            "  \"model.lr\": 0.00017214000000000002,\n",
            "  \"model.grad_norm\": 0.008003951981663704,\n",
            "  \"model.elapsed_time\": 0.06689858436584473,\n",
            "  \"model.engine_step\": 862,\n",
            "  \"model.loss.nll\": 0.0005645751953125,\n",
            "  \"elapsed_time\": 0.06689858436584473,\n",
            "  \"wall_time\": 1674064285.776257,\n",
            "  \"global_step\": 862\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.000560760498046875,\n",
            "  \"model.lr\": 0.000172339,\n",
            "  \"model.grad_norm\": 0.010371891781687737,\n",
            "  \"model.elapsed_time\": 0.0782628059387207,\n",
            "  \"model.engine_step\": 863,\n",
            "  \"model.loss.nll\": 0.000560760498046875,\n",
            "  \"elapsed_time\": 0.0782628059387207,\n",
            "  \"wall_time\": 1674064285.865051,\n",
            "  \"global_step\": 863\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005335807800292969,\n",
            "  \"model.lr\": 0.000172538,\n",
            "  \"model.grad_norm\": 0.00663054408505559,\n",
            "  \"model.elapsed_time\": 0.06750988960266113,\n",
            "  \"model.engine_step\": 864,\n",
            "  \"model.loss.nll\": 0.0005335807800292969,\n",
            "  \"elapsed_time\": 0.06750988960266113,\n",
            "  \"wall_time\": 1674064285.9345987,\n",
            "  \"global_step\": 864\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:25\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0015859603881835938,\n",
            "  \"model.lr\": 0.000172737,\n",
            "  \"model.grad_norm\": 0.027806920930743217,\n",
            "  \"model.elapsed_time\": 0.07413935661315918,\n",
            "  \"model.engine_step\": 865,\n",
            "  \"model.loss.nll\": 0.0015859603881835938,\n",
            "  \"elapsed_time\": 0.07413935661315918,\n",
            "  \"wall_time\": 1674064286.0162876,\n",
            "  \"global_step\": 865\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005745887756347656,\n",
            "  \"model.lr\": 0.000172936,\n",
            "  \"model.grad_norm\": 0.01589694805443287,\n",
            "  \"model.elapsed_time\": 0.06745624542236328,\n",
            "  \"model.engine_step\": 866,\n",
            "  \"model.loss.nll\": 0.0005745887756347656,\n",
            "  \"elapsed_time\": 0.06745624542236328,\n",
            "  \"wall_time\": 1674064286.0857131,\n",
            "  \"global_step\": 866\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02978515625,\n",
            "  \"model.lr\": 0.00017313500000000002,\n",
            "  \"model.grad_norm\": 2.4944300651550293,\n",
            "  \"model.elapsed_time\": 0.07317352294921875,\n",
            "  \"model.engine_step\": 867,\n",
            "  \"model.loss.nll\": 0.02978515625,\n",
            "  \"elapsed_time\": 0.07317352294921875,\n",
            "  \"wall_time\": 1674064286.1659176,\n",
            "  \"global_step\": 867\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005307197570800781,\n",
            "  \"model.lr\": 0.000173334,\n",
            "  \"model.grad_norm\": 0.01576666347682476,\n",
            "  \"model.elapsed_time\": 0.07047915458679199,\n",
            "  \"model.engine_step\": 868,\n",
            "  \"model.loss.nll\": 0.0005307197570800781,\n",
            "  \"elapsed_time\": 0.07047915458679199,\n",
            "  \"wall_time\": 1674064286.2390382,\n",
            "  \"global_step\": 868\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005197525024414062,\n",
            "  \"model.lr\": 0.000173533,\n",
            "  \"model.grad_norm\": 0.010266739875078201,\n",
            "  \"model.elapsed_time\": 0.06500029563903809,\n",
            "  \"model.engine_step\": 869,\n",
            "  \"model.loss.nll\": 0.0005197525024414062,\n",
            "  \"elapsed_time\": 0.06500029563903809,\n",
            "  \"wall_time\": 1674064286.3117518,\n",
            "  \"global_step\": 869\n",
            "}\n",
            "[2023-01-18 17:51:26,394] [INFO] [logging.py:68:log_dist] [Rank 0] step=870, skipped=1, lr=[0.000173732], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:26,396] [INFO] [timer.py:196:stop] epoch=0/micro_step=870/global_step=870, RunningAvgSamplesPerSec=13.648163862785417, CurrSamplesPerSec=12.066675489220183, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005450248718261719,\n",
            "  \"model.lr\": 0.000173732,\n",
            "  \"model.grad_norm\": 0.010488033294677734,\n",
            "  \"model.elapsed_time\": 0.08329987525939941,\n",
            "  \"model.engine_step\": 870,\n",
            "  \"model.loss.nll\": 0.0005450248718261719,\n",
            "  \"elapsed_time\": 0.08329987525939941,\n",
            "  \"wall_time\": 1674064286.3975894,\n",
            "  \"global_step\": 870\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005373954772949219,\n",
            "  \"model.lr\": 0.000173931,\n",
            "  \"model.grad_norm\": 0.011208874173462391,\n",
            "  \"model.elapsed_time\": 0.06979870796203613,\n",
            "  \"model.engine_step\": 871,\n",
            "  \"model.loss.nll\": 0.0005373954772949219,\n",
            "  \"elapsed_time\": 0.06979870796203613,\n",
            "  \"wall_time\": 1674064286.4747846,\n",
            "  \"global_step\": 871\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.033203125,\n",
            "  \"model.lr\": 0.00017413000000000002,\n",
            "  \"model.grad_norm\": 2.010220527648926,\n",
            "  \"model.elapsed_time\": 0.06549358367919922,\n",
            "  \"model.engine_step\": 872,\n",
            "  \"model.loss.nll\": 0.033203125,\n",
            "  \"elapsed_time\": 0.06549358367919922,\n",
            "  \"wall_time\": 1674064286.542892,\n",
            "  \"global_step\": 872\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005106925964355469,\n",
            "  \"model.lr\": 0.000174329,\n",
            "  \"model.grad_norm\": 0.009369616396725178,\n",
            "  \"model.elapsed_time\": 0.0691530704498291,\n",
            "  \"model.engine_step\": 873,\n",
            "  \"model.loss.nll\": 0.0005106925964355469,\n",
            "  \"elapsed_time\": 0.0691530704498291,\n",
            "  \"wall_time\": 1674064286.619186,\n",
            "  \"global_step\": 873\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006775856018066406,\n",
            "  \"model.lr\": 0.00017452800000000002,\n",
            "  \"model.grad_norm\": 0.027183763682842255,\n",
            "  \"model.elapsed_time\": 0.07792425155639648,\n",
            "  \"model.engine_step\": 874,\n",
            "  \"model.loss.nll\": 0.0006775856018066406,\n",
            "  \"elapsed_time\": 0.07792425155639648,\n",
            "  \"wall_time\": 1674064286.6997876,\n",
            "  \"global_step\": 874\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005402565002441406,\n",
            "  \"model.lr\": 0.000174727,\n",
            "  \"model.grad_norm\": 0.00793297216296196,\n",
            "  \"model.elapsed_time\": 0.06878328323364258,\n",
            "  \"model.engine_step\": 875,\n",
            "  \"model.loss.nll\": 0.0005402565002441406,\n",
            "  \"elapsed_time\": 0.06878328323364258,\n",
            "  \"wall_time\": 1674064286.776192,\n",
            "  \"global_step\": 875\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006175041198730469,\n",
            "  \"model.lr\": 0.000174926,\n",
            "  \"model.grad_norm\": 0.022646483033895493,\n",
            "  \"model.elapsed_time\": 0.06995320320129395,\n",
            "  \"model.engine_step\": 876,\n",
            "  \"model.loss.nll\": 0.0006175041198730469,\n",
            "  \"elapsed_time\": 0.06995320320129395,\n",
            "  \"wall_time\": 1674064286.8487716,\n",
            "  \"global_step\": 876\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:26\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010614395141601562,\n",
            "  \"model.lr\": 0.00017512500000000001,\n",
            "  \"model.grad_norm\": 0.07850973308086395,\n",
            "  \"model.elapsed_time\": 0.0737605094909668,\n",
            "  \"model.engine_step\": 877,\n",
            "  \"model.loss.nll\": 0.0010614395141601562,\n",
            "  \"elapsed_time\": 0.0737605094909668,\n",
            "  \"wall_time\": 1674064286.9379873,\n",
            "  \"global_step\": 877\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005755424499511719,\n",
            "  \"model.lr\": 0.000175324,\n",
            "  \"model.grad_norm\": 0.012314419262111187,\n",
            "  \"model.elapsed_time\": 0.07141304016113281,\n",
            "  \"model.engine_step\": 878,\n",
            "  \"model.loss.nll\": 0.0005755424499511719,\n",
            "  \"elapsed_time\": 0.07141304016113281,\n",
            "  \"wall_time\": 1674064287.011571,\n",
            "  \"global_step\": 878\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00119781494140625,\n",
            "  \"model.lr\": 0.00017552300000000002,\n",
            "  \"model.grad_norm\": 0.15410511195659637,\n",
            "  \"model.elapsed_time\": 0.06988239288330078,\n",
            "  \"model.engine_step\": 879,\n",
            "  \"model.loss.nll\": 0.00119781494140625,\n",
            "  \"elapsed_time\": 0.06988239288330078,\n",
            "  \"wall_time\": 1674064287.0893476,\n",
            "  \"global_step\": 879\n",
            "}\n",
            "[2023-01-18 17:51:27,159] [INFO] [logging.py:68:log_dist] [Rank 0] step=880, skipped=1, lr=[0.000175722], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:27,160] [INFO] [timer.py:196:stop] epoch=0/micro_step=880/global_step=880, RunningAvgSamplesPerSec=13.65401405570413, CurrSamplesPerSec=14.32265643141058, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005712509155273438,\n",
            "  \"model.lr\": 0.000175722,\n",
            "  \"model.grad_norm\": 0.022335806861519814,\n",
            "  \"model.elapsed_time\": 0.0705711841583252,\n",
            "  \"model.engine_step\": 880,\n",
            "  \"model.loss.nll\": 0.0005712509155273438,\n",
            "  \"elapsed_time\": 0.0705711841583252,\n",
            "  \"wall_time\": 1674064287.1622252,\n",
            "  \"global_step\": 880\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07305908203125,\n",
            "  \"model.lr\": 0.000175921,\n",
            "  \"model.grad_norm\": 1.6858680248260498,\n",
            "  \"model.elapsed_time\": 0.06593036651611328,\n",
            "  \"model.engine_step\": 881,\n",
            "  \"model.loss.nll\": 0.07305908203125,\n",
            "  \"elapsed_time\": 0.06593036651611328,\n",
            "  \"wall_time\": 1674064287.2365532,\n",
            "  \"global_step\": 881\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006666183471679688,\n",
            "  \"model.lr\": 0.00017612000000000001,\n",
            "  \"model.grad_norm\": 0.025505229830741882,\n",
            "  \"model.elapsed_time\": 0.06923246383666992,\n",
            "  \"model.engine_step\": 882,\n",
            "  \"model.loss.nll\": 0.0006666183471679688,\n",
            "  \"elapsed_time\": 0.06923246383666992,\n",
            "  \"wall_time\": 1674064287.308392,\n",
            "  \"global_step\": 882\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004534721374511719,\n",
            "  \"model.lr\": 0.000176319,\n",
            "  \"model.grad_norm\": 0.006822826340794563,\n",
            "  \"model.elapsed_time\": 0.09254288673400879,\n",
            "  \"model.engine_step\": 883,\n",
            "  \"model.loss.nll\": 0.0004534721374511719,\n",
            "  \"elapsed_time\": 0.09254288673400879,\n",
            "  \"wall_time\": 1674064287.4091966,\n",
            "  \"global_step\": 883\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007576942443847656,\n",
            "  \"model.lr\": 0.00017651800000000002,\n",
            "  \"model.grad_norm\": 0.028434554114937782,\n",
            "  \"model.elapsed_time\": 0.07384872436523438,\n",
            "  \"model.engine_step\": 884,\n",
            "  \"model.loss.nll\": 0.0007576942443847656,\n",
            "  \"elapsed_time\": 0.07384872436523438,\n",
            "  \"wall_time\": 1674064287.4852312,\n",
            "  \"global_step\": 884\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00444793701171875,\n",
            "  \"model.lr\": 0.000176717,\n",
            "  \"model.grad_norm\": 0.3832715153694153,\n",
            "  \"model.elapsed_time\": 0.06593871116638184,\n",
            "  \"model.engine_step\": 885,\n",
            "  \"model.loss.nll\": 0.00444793701171875,\n",
            "  \"elapsed_time\": 0.06593871116638184,\n",
            "  \"wall_time\": 1674064287.559031,\n",
            "  \"global_step\": 885\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0016222000122070312,\n",
            "  \"model.lr\": 0.000176916,\n",
            "  \"model.grad_norm\": 0.04385935142636299,\n",
            "  \"model.elapsed_time\": 0.06800723075866699,\n",
            "  \"model.engine_step\": 886,\n",
            "  \"model.loss.nll\": 0.0016222000122070312,\n",
            "  \"elapsed_time\": 0.06800723075866699,\n",
            "  \"wall_time\": 1674064287.6289396,\n",
            "  \"global_step\": 886\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007653236389160156,\n",
            "  \"model.lr\": 0.00017711500000000001,\n",
            "  \"model.grad_norm\": 0.03596273437142372,\n",
            "  \"model.elapsed_time\": 0.06783151626586914,\n",
            "  \"model.engine_step\": 887,\n",
            "  \"model.loss.nll\": 0.0007653236389160156,\n",
            "  \"elapsed_time\": 0.06783151626586914,\n",
            "  \"wall_time\": 1674064287.7044404,\n",
            "  \"global_step\": 887\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.001201629638671875,\n",
            "  \"model.lr\": 0.000177314,\n",
            "  \"model.grad_norm\": 0.18520301580429077,\n",
            "  \"model.elapsed_time\": 0.07663559913635254,\n",
            "  \"model.engine_step\": 888,\n",
            "  \"model.loss.nll\": 0.001201629638671875,\n",
            "  \"elapsed_time\": 0.07663559913635254,\n",
            "  \"wall_time\": 1674064287.7831178,\n",
            "  \"global_step\": 888\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00154876708984375,\n",
            "  \"model.lr\": 0.00017751300000000002,\n",
            "  \"model.grad_norm\": 0.26260489225387573,\n",
            "  \"model.elapsed_time\": 0.07224178314208984,\n",
            "  \"model.engine_step\": 889,\n",
            "  \"model.loss.nll\": 0.00154876708984375,\n",
            "  \"elapsed_time\": 0.07224178314208984,\n",
            "  \"wall_time\": 1674064287.8633382,\n",
            "  \"global_step\": 889\n",
            "}\n",
            "[2023-01-18 17:51:27,931] [INFO] [logging.py:68:log_dist] [Rank 0] step=890, skipped=1, lr=[0.000177712], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:27,933] [INFO] [timer.py:196:stop] epoch=0/micro_step=890/global_step=890, RunningAvgSamplesPerSec=13.656602834797216, CurrSamplesPerSec=14.57914714347288, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0019178390502929688,\n",
            "  \"model.lr\": 0.000177712,\n",
            "  \"model.grad_norm\": 0.3699924945831299,\n",
            "  \"model.elapsed_time\": 0.0690610408782959,\n",
            "  \"model.engine_step\": 890,\n",
            "  \"model.loss.nll\": 0.0019178390502929688,\n",
            "  \"elapsed_time\": 0.0690610408782959,\n",
            "  \"wall_time\": 1674064287.934508,\n",
            "  \"global_step\": 890\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:27\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005350112915039062,\n",
            "  \"model.lr\": 0.000177911,\n",
            "  \"model.grad_norm\": 0.034479036927223206,\n",
            "  \"model.elapsed_time\": 0.0781090259552002,\n",
            "  \"model.engine_step\": 891,\n",
            "  \"model.loss.nll\": 0.0005350112915039062,\n",
            "  \"elapsed_time\": 0.0781090259552002,\n",
            "  \"wall_time\": 1674064288.0205367,\n",
            "  \"global_step\": 891\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0322265625,\n",
            "  \"model.lr\": 0.00017811000000000001,\n",
            "  \"model.grad_norm\": 1.4621649980545044,\n",
            "  \"model.elapsed_time\": 0.07810020446777344,\n",
            "  \"model.engine_step\": 892,\n",
            "  \"model.loss.nll\": 0.0322265625,\n",
            "  \"elapsed_time\": 0.07810020446777344,\n",
            "  \"wall_time\": 1674064288.1006675,\n",
            "  \"global_step\": 892\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004906654357910156,\n",
            "  \"model.lr\": 0.000178309,\n",
            "  \"model.grad_norm\": 0.005881503690034151,\n",
            "  \"model.elapsed_time\": 0.06893324851989746,\n",
            "  \"model.engine_step\": 893,\n",
            "  \"model.loss.nll\": 0.0004906654357910156,\n",
            "  \"elapsed_time\": 0.06893324851989746,\n",
            "  \"wall_time\": 1674064288.177755,\n",
            "  \"global_step\": 893\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004787445068359375,\n",
            "  \"model.lr\": 0.00017850800000000002,\n",
            "  \"model.grad_norm\": 0.011059905402362347,\n",
            "  \"model.elapsed_time\": 0.07258200645446777,\n",
            "  \"model.engine_step\": 894,\n",
            "  \"model.loss.nll\": 0.0004787445068359375,\n",
            "  \"elapsed_time\": 0.07258200645446777,\n",
            "  \"wall_time\": 1674064288.253115,\n",
            "  \"global_step\": 894\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00677490234375,\n",
            "  \"model.lr\": 0.000178707,\n",
            "  \"model.grad_norm\": 0.80377596616745,\n",
            "  \"model.elapsed_time\": 0.0676267147064209,\n",
            "  \"model.engine_step\": 895,\n",
            "  \"model.loss.nll\": 0.00677490234375,\n",
            "  \"elapsed_time\": 0.0676267147064209,\n",
            "  \"wall_time\": 1674064288.3278196,\n",
            "  \"global_step\": 895\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0033283233642578125,\n",
            "  \"model.lr\": 0.00017890600000000002,\n",
            "  \"model.grad_norm\": 0.2688745856285095,\n",
            "  \"model.elapsed_time\": 0.07814860343933105,\n",
            "  \"model.engine_step\": 896,\n",
            "  \"model.loss.nll\": 0.0033283233642578125,\n",
            "  \"elapsed_time\": 0.07814860343933105,\n",
            "  \"wall_time\": 1674064288.4090106,\n",
            "  \"global_step\": 896\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004322528839111328,\n",
            "  \"model.lr\": 0.00017910500000000001,\n",
            "  \"model.grad_norm\": 0.005015801638364792,\n",
            "  \"model.elapsed_time\": 0.0736386775970459,\n",
            "  \"model.engine_step\": 897,\n",
            "  \"model.loss.nll\": 0.0004322528839111328,\n",
            "  \"elapsed_time\": 0.0736386775970459,\n",
            "  \"wall_time\": 1674064288.4910889,\n",
            "  \"global_step\": 897\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.000728607177734375,\n",
            "  \"model.lr\": 0.000179304,\n",
            "  \"model.grad_norm\": 0.055456068366765976,\n",
            "  \"model.elapsed_time\": 0.06873059272766113,\n",
            "  \"model.engine_step\": 898,\n",
            "  \"model.loss.nll\": 0.000728607177734375,\n",
            "  \"elapsed_time\": 0.06873059272766113,\n",
            "  \"wall_time\": 1674064288.5624952,\n",
            "  \"global_step\": 898\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00043272972106933594,\n",
            "  \"model.lr\": 0.00017950300000000002,\n",
            "  \"model.grad_norm\": 0.005826006177812815,\n",
            "  \"model.elapsed_time\": 0.07320594787597656,\n",
            "  \"model.engine_step\": 899,\n",
            "  \"model.loss.nll\": 0.00043272972106933594,\n",
            "  \"elapsed_time\": 0.07320594787597656,\n",
            "  \"wall_time\": 1674064288.6430938,\n",
            "  \"global_step\": 899\n",
            "}\n",
            "[2023-01-18 17:51:28,719] [INFO] [logging.py:68:log_dist] [Rank 0] step=900, skipped=1, lr=[0.000179702], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:28,721] [INFO] [timer.py:196:stop] epoch=0/micro_step=900/global_step=900, RunningAvgSamplesPerSec=13.65617044836593, CurrSamplesPerSec=13.12779422719391, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005145072937011719,\n",
            "  \"model.lr\": 0.000179702,\n",
            "  \"model.grad_norm\": 0.012866544537246227,\n",
            "  \"model.elapsed_time\": 0.07663297653198242,\n",
            "  \"model.engine_step\": 900,\n",
            "  \"model.loss.nll\": 0.0005145072937011719,\n",
            "  \"elapsed_time\": 0.07663297653198242,\n",
            "  \"wall_time\": 1674064288.722382,\n",
            "  \"global_step\": 900\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007872581481933594,\n",
            "  \"model.lr\": 0.00017990100000000002,\n",
            "  \"model.grad_norm\": 0.04418841749429703,\n",
            "  \"model.elapsed_time\": 0.06649279594421387,\n",
            "  \"model.engine_step\": 901,\n",
            "  \"model.loss.nll\": 0.0007872581481933594,\n",
            "  \"elapsed_time\": 0.06649279594421387,\n",
            "  \"wall_time\": 1674064288.7970102,\n",
            "  \"global_step\": 901\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0029144287109375,\n",
            "  \"model.lr\": 0.00018010000000000001,\n",
            "  \"model.grad_norm\": 0.14507876336574554,\n",
            "  \"model.elapsed_time\": 0.07632112503051758,\n",
            "  \"model.engine_step\": 902,\n",
            "  \"model.loss.nll\": 0.0029144287109375,\n",
            "  \"elapsed_time\": 0.07632112503051758,\n",
            "  \"wall_time\": 1674064288.8758729,\n",
            "  \"global_step\": 902\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:28\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0169219970703125,\n",
            "  \"model.lr\": 0.000180299,\n",
            "  \"model.grad_norm\": 1.214802861213684,\n",
            "  \"model.elapsed_time\": 0.07664179801940918,\n",
            "  \"model.engine_step\": 903,\n",
            "  \"model.loss.nll\": 0.0169219970703125,\n",
            "  \"elapsed_time\": 0.07664179801940918,\n",
            "  \"wall_time\": 1674064288.9608994,\n",
            "  \"global_step\": 903\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0019397735595703125,\n",
            "  \"model.lr\": 0.00018049800000000002,\n",
            "  \"model.grad_norm\": 0.11902536451816559,\n",
            "  \"model.elapsed_time\": 0.06704139709472656,\n",
            "  \"model.engine_step\": 904,\n",
            "  \"model.loss.nll\": 0.0019397735595703125,\n",
            "  \"elapsed_time\": 0.06704139709472656,\n",
            "  \"wall_time\": 1674064289.030386,\n",
            "  \"global_step\": 904\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007195472717285156,\n",
            "  \"model.lr\": 0.000180697,\n",
            "  \"model.grad_norm\": 0.031394559890031815,\n",
            "  \"model.elapsed_time\": 0.07631731033325195,\n",
            "  \"model.engine_step\": 905,\n",
            "  \"model.loss.nll\": 0.0007195472717285156,\n",
            "  \"elapsed_time\": 0.07631731033325195,\n",
            "  \"wall_time\": 1674064289.115102,\n",
            "  \"global_step\": 905\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005183219909667969,\n",
            "  \"model.lr\": 0.00018089600000000002,\n",
            "  \"model.grad_norm\": 0.006867548450827599,\n",
            "  \"model.elapsed_time\": 0.06760859489440918,\n",
            "  \"model.engine_step\": 906,\n",
            "  \"model.loss.nll\": 0.0005183219909667969,\n",
            "  \"elapsed_time\": 0.06760859489440918,\n",
            "  \"wall_time\": 1674064289.1852603,\n",
            "  \"global_step\": 906\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004508495330810547,\n",
            "  \"model.lr\": 0.00018109500000000001,\n",
            "  \"model.grad_norm\": 0.00694676861166954,\n",
            "  \"model.elapsed_time\": 0.07235360145568848,\n",
            "  \"model.engine_step\": 907,\n",
            "  \"model.loss.nll\": 0.0004508495330810547,\n",
            "  \"elapsed_time\": 0.07235360145568848,\n",
            "  \"wall_time\": 1674064289.2692587,\n",
            "  \"global_step\": 907\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004334449768066406,\n",
            "  \"model.lr\": 0.000181294,\n",
            "  \"model.grad_norm\": 0.004834052175283432,\n",
            "  \"model.elapsed_time\": 0.06766891479492188,\n",
            "  \"model.engine_step\": 908,\n",
            "  \"model.loss.nll\": 0.0004334449768066406,\n",
            "  \"elapsed_time\": 0.06766891479492188,\n",
            "  \"wall_time\": 1674064289.3397315,\n",
            "  \"global_step\": 908\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01366424560546875,\n",
            "  \"model.lr\": 0.00018149300000000002,\n",
            "  \"model.grad_norm\": 1.2006261348724365,\n",
            "  \"model.elapsed_time\": 0.07872128486633301,\n",
            "  \"model.engine_step\": 909,\n",
            "  \"model.loss.nll\": 0.01366424560546875,\n",
            "  \"elapsed_time\": 0.07872128486633301,\n",
            "  \"wall_time\": 1674064289.4262977,\n",
            "  \"global_step\": 909\n",
            "}\n",
            "[2023-01-18 17:51:29,497] [INFO] [logging.py:68:log_dist] [Rank 0] step=910, skipped=1, lr=[0.000181692], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:29,497] [INFO] [timer.py:196:stop] epoch=0/micro_step=910/global_step=910, RunningAvgSamplesPerSec=13.659117574275067, CurrSamplesPerSec=14.360022185550633, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.039031982421875,\n",
            "  \"model.lr\": 0.000181692,\n",
            "  \"model.grad_norm\": 1.8731565475463867,\n",
            "  \"model.elapsed_time\": 0.07040739059448242,\n",
            "  \"model.engine_step\": 910,\n",
            "  \"model.loss.nll\": 0.039031982421875,\n",
            "  \"elapsed_time\": 0.07040739059448242,\n",
            "  \"wall_time\": 1674064289.4996645,\n",
            "  \"global_step\": 910\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.001758575439453125,\n",
            "  \"model.lr\": 0.00018189100000000002,\n",
            "  \"model.grad_norm\": 0.06640204787254333,\n",
            "  \"model.elapsed_time\": 0.07465720176696777,\n",
            "  \"model.engine_step\": 911,\n",
            "  \"model.loss.nll\": 0.001758575439453125,\n",
            "  \"elapsed_time\": 0.07465720176696777,\n",
            "  \"wall_time\": 1674064289.5857131,\n",
            "  \"global_step\": 911\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0016508102416992188,\n",
            "  \"model.lr\": 0.00018209,\n",
            "  \"model.grad_norm\": 0.26250049471855164,\n",
            "  \"model.elapsed_time\": 0.07120180130004883,\n",
            "  \"model.engine_step\": 912,\n",
            "  \"model.loss.nll\": 0.0016508102416992188,\n",
            "  \"elapsed_time\": 0.07120180130004883,\n",
            "  \"wall_time\": 1674064289.6590252,\n",
            "  \"global_step\": 912\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00757598876953125,\n",
            "  \"model.lr\": 0.000182289,\n",
            "  \"model.grad_norm\": 1.0890681743621826,\n",
            "  \"model.elapsed_time\": 0.07105541229248047,\n",
            "  \"model.engine_step\": 913,\n",
            "  \"model.loss.nll\": 0.00757598876953125,\n",
            "  \"elapsed_time\": 0.07105541229248047,\n",
            "  \"wall_time\": 1674064289.739074,\n",
            "  \"global_step\": 913\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0074462890625,\n",
            "  \"model.lr\": 0.00018248800000000002,\n",
            "  \"model.grad_norm\": 1.0063672065734863,\n",
            "  \"model.elapsed_time\": 0.06642365455627441,\n",
            "  \"model.engine_step\": 914,\n",
            "  \"model.loss.nll\": 0.0074462890625,\n",
            "  \"elapsed_time\": 0.06642365455627441,\n",
            "  \"wall_time\": 1674064289.8074272,\n",
            "  \"global_step\": 914\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005612373352050781,\n",
            "  \"model.lr\": 0.000182687,\n",
            "  \"model.grad_norm\": 0.040822844952344894,\n",
            "  \"model.elapsed_time\": 0.07392120361328125,\n",
            "  \"model.engine_step\": 915,\n",
            "  \"model.loss.nll\": 0.0005612373352050781,\n",
            "  \"elapsed_time\": 0.07392120361328125,\n",
            "  \"wall_time\": 1674064289.8900063,\n",
            "  \"global_step\": 915\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007257461547851562,\n",
            "  \"model.lr\": 0.00018288600000000002,\n",
            "  \"model.grad_norm\": 0.04130544513463974,\n",
            "  \"model.elapsed_time\": 0.06850099563598633,\n",
            "  \"model.engine_step\": 916,\n",
            "  \"model.loss.nll\": 0.0007257461547851562,\n",
            "  \"elapsed_time\": 0.06850099563598633,\n",
            "  \"wall_time\": 1674064289.9605436,\n",
            "  \"global_step\": 916\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:29\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0014524459838867188,\n",
            "  \"model.lr\": 0.000183085,\n",
            "  \"model.grad_norm\": 0.17521417140960693,\n",
            "  \"model.elapsed_time\": 0.08195877075195312,\n",
            "  \"model.engine_step\": 917,\n",
            "  \"model.loss.nll\": 0.0014524459838867188,\n",
            "  \"elapsed_time\": 0.08195877075195312,\n",
            "  \"wall_time\": 1674064290.0502887,\n",
            "  \"global_step\": 917\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.023834228515625,\n",
            "  \"model.lr\": 0.000183284,\n",
            "  \"model.grad_norm\": 1.6977101564407349,\n",
            "  \"model.elapsed_time\": 0.06749296188354492,\n",
            "  \"model.engine_step\": 918,\n",
            "  \"model.loss.nll\": 0.023834228515625,\n",
            "  \"elapsed_time\": 0.06749296188354492,\n",
            "  \"wall_time\": 1674064290.1198888,\n",
            "  \"global_step\": 918\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00739288330078125,\n",
            "  \"model.lr\": 0.00018348300000000002,\n",
            "  \"model.grad_norm\": 1.1081064939498901,\n",
            "  \"model.elapsed_time\": 0.06743597984313965,\n",
            "  \"model.engine_step\": 919,\n",
            "  \"model.loss.nll\": 0.00739288330078125,\n",
            "  \"elapsed_time\": 0.06743597984313965,\n",
            "  \"wall_time\": 1674064290.1943917,\n",
            "  \"global_step\": 919\n",
            "}\n",
            "[2023-01-18 17:51:30,262] [INFO] [logging.py:68:log_dist] [Rank 0] step=920, skipped=1, lr=[0.000183682], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:30,264] [INFO] [timer.py:196:stop] epoch=0/micro_step=920/global_step=920, RunningAvgSamplesPerSec=13.663635800077802, CurrSamplesPerSec=14.712932691633108, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05938720703125,\n",
            "  \"model.lr\": 0.000183682,\n",
            "  \"model.grad_norm\": 1.3041584491729736,\n",
            "  \"model.elapsed_time\": 0.06841897964477539,\n",
            "  \"model.engine_step\": 920,\n",
            "  \"model.loss.nll\": 0.05938720703125,\n",
            "  \"elapsed_time\": 0.06841897964477539,\n",
            "  \"wall_time\": 1674064290.265486,\n",
            "  \"global_step\": 920\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0036983489990234375,\n",
            "  \"model.lr\": 0.00018388100000000002,\n",
            "  \"model.grad_norm\": 0.20350584387779236,\n",
            "  \"model.elapsed_time\": 0.0728909969329834,\n",
            "  \"model.engine_step\": 921,\n",
            "  \"model.loss.nll\": 0.0036983489990234375,\n",
            "  \"elapsed_time\": 0.0728909969329834,\n",
            "  \"wall_time\": 1674064290.3452582,\n",
            "  \"global_step\": 921\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0294342041015625,\n",
            "  \"model.lr\": 0.00018408,\n",
            "  \"model.grad_norm\": 1.5474631786346436,\n",
            "  \"model.elapsed_time\": 0.06780099868774414,\n",
            "  \"model.engine_step\": 922,\n",
            "  \"model.loss.nll\": 0.0294342041015625,\n",
            "  \"elapsed_time\": 0.06780099868774414,\n",
            "  \"wall_time\": 1674064290.4159436,\n",
            "  \"global_step\": 922\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00296783447265625,\n",
            "  \"model.lr\": 0.00018427900000000003,\n",
            "  \"model.grad_norm\": 0.20965687930583954,\n",
            "  \"model.elapsed_time\": 0.0749814510345459,\n",
            "  \"model.engine_step\": 923,\n",
            "  \"model.loss.nll\": 0.00296783447265625,\n",
            "  \"elapsed_time\": 0.0749814510345459,\n",
            "  \"wall_time\": 1674064290.4978259,\n",
            "  \"global_step\": 923\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0029048919677734375,\n",
            "  \"model.lr\": 0.00018447800000000002,\n",
            "  \"model.grad_norm\": 0.16182228922843933,\n",
            "  \"model.elapsed_time\": 0.06893467903137207,\n",
            "  \"model.engine_step\": 924,\n",
            "  \"model.loss.nll\": 0.0029048919677734375,\n",
            "  \"elapsed_time\": 0.06893467903137207,\n",
            "  \"wall_time\": 1674064290.5692446,\n",
            "  \"global_step\": 924\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006074905395507812,\n",
            "  \"model.lr\": 0.000184677,\n",
            "  \"model.grad_norm\": 0.01769500970840454,\n",
            "  \"model.elapsed_time\": 0.07396554946899414,\n",
            "  \"model.engine_step\": 925,\n",
            "  \"model.loss.nll\": 0.0006074905395507812,\n",
            "  \"elapsed_time\": 0.07396554946899414,\n",
            "  \"wall_time\": 1674064290.650841,\n",
            "  \"global_step\": 925\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010690689086914062,\n",
            "  \"model.lr\": 0.00018487600000000002,\n",
            "  \"model.grad_norm\": 0.06209440156817436,\n",
            "  \"model.elapsed_time\": 0.0699467658996582,\n",
            "  \"model.engine_step\": 926,\n",
            "  \"model.loss.nll\": 0.0010690689086914062,\n",
            "  \"elapsed_time\": 0.0699467658996582,\n",
            "  \"wall_time\": 1674064290.723337,\n",
            "  \"global_step\": 926\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.002155303955078125,\n",
            "  \"model.lr\": 0.000185075,\n",
            "  \"model.grad_norm\": 0.3082036077976227,\n",
            "  \"model.elapsed_time\": 0.06688642501831055,\n",
            "  \"model.engine_step\": 927,\n",
            "  \"model.loss.nll\": 0.002155303955078125,\n",
            "  \"elapsed_time\": 0.06688642501831055,\n",
            "  \"wall_time\": 1674064290.798969,\n",
            "  \"global_step\": 927\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0230712890625,\n",
            "  \"model.lr\": 0.00018527400000000003,\n",
            "  \"model.grad_norm\": 1.5005996227264404,\n",
            "  \"model.elapsed_time\": 0.07114672660827637,\n",
            "  \"model.engine_step\": 928,\n",
            "  \"model.loss.nll\": 0.0230712890625,\n",
            "  \"elapsed_time\": 0.07114672660827637,\n",
            "  \"wall_time\": 1674064290.873689,\n",
            "  \"global_step\": 928\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:30\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0015201568603515625,\n",
            "  \"model.lr\": 0.00018547300000000002,\n",
            "  \"model.grad_norm\": 0.13369594514369965,\n",
            "  \"model.elapsed_time\": 0.06841611862182617,\n",
            "  \"model.engine_step\": 929,\n",
            "  \"model.loss.nll\": 0.0015201568603515625,\n",
            "  \"elapsed_time\": 0.06841611862182617,\n",
            "  \"wall_time\": 1674064290.9543438,\n",
            "  \"global_step\": 929\n",
            "}\n",
            "[2023-01-18 17:51:31,022] [INFO] [logging.py:68:log_dist] [Rank 0] step=930, skipped=1, lr=[0.000185672], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:31,023] [INFO] [timer.py:196:stop] epoch=0/micro_step=930/global_step=930, RunningAvgSamplesPerSec=13.66964868768512, CurrSamplesPerSec=14.732105161482938, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013523101806640625,\n",
            "  \"model.lr\": 0.000185672,\n",
            "  \"model.grad_norm\": 0.11763664335012436,\n",
            "  \"model.elapsed_time\": 0.06837201118469238,\n",
            "  \"model.engine_step\": 930,\n",
            "  \"model.loss.nll\": 0.0013523101806640625,\n",
            "  \"elapsed_time\": 0.06837201118469238,\n",
            "  \"wall_time\": 1674064291.024838,\n",
            "  \"global_step\": 930\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013408660888671875,\n",
            "  \"model.lr\": 0.00018587100000000002,\n",
            "  \"model.grad_norm\": 0.14430618286132812,\n",
            "  \"model.elapsed_time\": 0.06752777099609375,\n",
            "  \"model.engine_step\": 931,\n",
            "  \"model.loss.nll\": 0.0013408660888671875,\n",
            "  \"elapsed_time\": 0.06752777099609375,\n",
            "  \"wall_time\": 1674064291.1009867,\n",
            "  \"global_step\": 931\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005536079406738281,\n",
            "  \"model.lr\": 0.00018607,\n",
            "  \"model.grad_norm\": 0.01506722904741764,\n",
            "  \"model.elapsed_time\": 0.07343292236328125,\n",
            "  \"model.engine_step\": 932,\n",
            "  \"model.loss.nll\": 0.0005536079406738281,\n",
            "  \"elapsed_time\": 0.07343292236328125,\n",
            "  \"wall_time\": 1674064291.177356,\n",
            "  \"global_step\": 932\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00046539306640625,\n",
            "  \"model.lr\": 0.00018626900000000003,\n",
            "  \"model.grad_norm\": 0.009242451749742031,\n",
            "  \"model.elapsed_time\": 0.07019853591918945,\n",
            "  \"model.engine_step\": 933,\n",
            "  \"model.loss.nll\": 0.00046539306640625,\n",
            "  \"elapsed_time\": 0.07019853591918945,\n",
            "  \"wall_time\": 1674064291.2557964,\n",
            "  \"global_step\": 933\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00069427490234375,\n",
            "  \"model.lr\": 0.00018646800000000002,\n",
            "  \"model.grad_norm\": 0.03836260363459587,\n",
            "  \"model.elapsed_time\": 0.06670212745666504,\n",
            "  \"model.engine_step\": 934,\n",
            "  \"model.loss.nll\": 0.00069427490234375,\n",
            "  \"elapsed_time\": 0.06670212745666504,\n",
            "  \"wall_time\": 1674064291.3245735,\n",
            "  \"global_step\": 934\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.004913330078125,\n",
            "  \"model.lr\": 0.000186667,\n",
            "  \"model.grad_norm\": 0.44822484254837036,\n",
            "  \"model.elapsed_time\": 0.06580400466918945,\n",
            "  \"model.engine_step\": 935,\n",
            "  \"model.loss.nll\": 0.004913330078125,\n",
            "  \"elapsed_time\": 0.06580400466918945,\n",
            "  \"wall_time\": 1674064291.3970213,\n",
            "  \"global_step\": 935\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0007071495056152344,\n",
            "  \"model.lr\": 0.00018686600000000002,\n",
            "  \"model.grad_norm\": 0.03190678730607033,\n",
            "  \"model.elapsed_time\": 0.0823204517364502,\n",
            "  \"model.engine_step\": 936,\n",
            "  \"model.loss.nll\": 0.0007071495056152344,\n",
            "  \"elapsed_time\": 0.0823204517364502,\n",
            "  \"wall_time\": 1674064291.4819539,\n",
            "  \"global_step\": 936\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013427734375,\n",
            "  \"model.lr\": 0.000187065,\n",
            "  \"model.grad_norm\": 0.2031809687614441,\n",
            "  \"model.elapsed_time\": 0.06984114646911621,\n",
            "  \"model.engine_step\": 937,\n",
            "  \"model.loss.nll\": 0.0013427734375,\n",
            "  \"elapsed_time\": 0.06984114646911621,\n",
            "  \"wall_time\": 1674064291.5595074,\n",
            "  \"global_step\": 937\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0014314651489257812,\n",
            "  \"model.lr\": 0.00018726400000000003,\n",
            "  \"model.grad_norm\": 0.1282922327518463,\n",
            "  \"model.elapsed_time\": 0.06524157524108887,\n",
            "  \"model.engine_step\": 938,\n",
            "  \"model.loss.nll\": 0.0014314651489257812,\n",
            "  \"elapsed_time\": 0.06524157524108887,\n",
            "  \"wall_time\": 1674064291.6272423,\n",
            "  \"global_step\": 938\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0011777877807617188,\n",
            "  \"model.lr\": 0.00018746300000000002,\n",
            "  \"model.grad_norm\": 0.10131219029426575,\n",
            "  \"model.elapsed_time\": 0.06609559059143066,\n",
            "  \"model.engine_step\": 939,\n",
            "  \"model.loss.nll\": 0.0011777877807617188,\n",
            "  \"elapsed_time\": 0.06609559059143066,\n",
            "  \"wall_time\": 1674064291.7009695,\n",
            "  \"global_step\": 939\n",
            "}\n",
            "[2023-01-18 17:51:31,778] [INFO] [logging.py:68:log_dist] [Rank 0] step=940, skipped=1, lr=[0.000187662], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:31,779] [INFO] [timer.py:196:stop] epoch=0/micro_step=940/global_step=940, RunningAvgSamplesPerSec=13.675244488803937, CurrSamplesPerSec=12.953498642668091, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0014410018920898438,\n",
            "  \"model.lr\": 0.000187662,\n",
            "  \"model.grad_norm\": 0.13800153136253357,\n",
            "  \"model.elapsed_time\": 0.07770180702209473,\n",
            "  \"model.engine_step\": 940,\n",
            "  \"model.loss.nll\": 0.0014410018920898438,\n",
            "  \"elapsed_time\": 0.07770180702209473,\n",
            "  \"wall_time\": 1674064291.7806923,\n",
            "  \"global_step\": 940\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.004611968994140625,\n",
            "  \"model.lr\": 0.000187861,\n",
            "  \"model.grad_norm\": 0.3466826379299164,\n",
            "  \"model.elapsed_time\": 0.0756065845489502,\n",
            "  \"model.engine_step\": 941,\n",
            "  \"model.loss.nll\": 0.004611968994140625,\n",
            "  \"elapsed_time\": 0.0756065845489502,\n",
            "  \"wall_time\": 1674064291.8639805,\n",
            "  \"global_step\": 941\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0016469955444335938,\n",
            "  \"model.lr\": 0.00018805999999999998,\n",
            "  \"model.grad_norm\": 0.06158750131726265,\n",
            "  \"model.elapsed_time\": 0.07655572891235352,\n",
            "  \"model.engine_step\": 942,\n",
            "  \"model.loss.nll\": 0.0016469955444335938,\n",
            "  \"elapsed_time\": 0.07655572891235352,\n",
            "  \"wall_time\": 1674064291.942649,\n",
            "  \"global_step\": 942\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:31\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00200653076171875,\n",
            "  \"model.lr\": 0.000188259,\n",
            "  \"model.grad_norm\": 0.07700418680906296,\n",
            "  \"model.elapsed_time\": 0.06708455085754395,\n",
            "  \"model.engine_step\": 943,\n",
            "  \"model.loss.nll\": 0.00200653076171875,\n",
            "  \"elapsed_time\": 0.06708455085754395,\n",
            "  \"wall_time\": 1674064292.0162606,\n",
            "  \"global_step\": 943\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0014820098876953125,\n",
            "  \"model.lr\": 0.000188458,\n",
            "  \"model.grad_norm\": 0.044551633298397064,\n",
            "  \"model.elapsed_time\": 0.0656282901763916,\n",
            "  \"model.engine_step\": 944,\n",
            "  \"model.loss.nll\": 0.0014820098876953125,\n",
            "  \"elapsed_time\": 0.0656282901763916,\n",
            "  \"wall_time\": 1674064292.0843523,\n",
            "  \"global_step\": 944\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008573532104492188,\n",
            "  \"model.lr\": 0.000188657,\n",
            "  \"model.grad_norm\": 0.05751385912299156,\n",
            "  \"model.elapsed_time\": 0.07296323776245117,\n",
            "  \"model.engine_step\": 945,\n",
            "  \"model.loss.nll\": 0.0008573532104492188,\n",
            "  \"elapsed_time\": 0.07296323776245117,\n",
            "  \"wall_time\": 1674064292.1646585,\n",
            "  \"global_step\": 945\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004596710205078125,\n",
            "  \"model.lr\": 0.000188856,\n",
            "  \"model.grad_norm\": 0.008119890466332436,\n",
            "  \"model.elapsed_time\": 0.06977105140686035,\n",
            "  \"model.engine_step\": 946,\n",
            "  \"model.loss.nll\": 0.0004596710205078125,\n",
            "  \"elapsed_time\": 0.06977105140686035,\n",
            "  \"wall_time\": 1674064292.2370152,\n",
            "  \"global_step\": 946\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0019121170043945312,\n",
            "  \"model.lr\": 0.000189055,\n",
            "  \"model.grad_norm\": 0.09327071905136108,\n",
            "  \"model.elapsed_time\": 0.07088375091552734,\n",
            "  \"model.engine_step\": 947,\n",
            "  \"model.loss.nll\": 0.0019121170043945312,\n",
            "  \"elapsed_time\": 0.07088375091552734,\n",
            "  \"wall_time\": 1674064292.3159957,\n",
            "  \"global_step\": 947\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0024013519287109375,\n",
            "  \"model.lr\": 0.000189254,\n",
            "  \"model.grad_norm\": 0.07510319352149963,\n",
            "  \"model.elapsed_time\": 0.07642531394958496,\n",
            "  \"model.engine_step\": 948,\n",
            "  \"model.loss.nll\": 0.0024013519287109375,\n",
            "  \"elapsed_time\": 0.07642531394958496,\n",
            "  \"wall_time\": 1674064292.395012,\n",
            "  \"global_step\": 948\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00208282470703125,\n",
            "  \"model.lr\": 0.000189453,\n",
            "  \"model.grad_norm\": 0.308651328086853,\n",
            "  \"model.elapsed_time\": 0.07923507690429688,\n",
            "  \"model.engine_step\": 949,\n",
            "  \"model.loss.nll\": 0.00208282470703125,\n",
            "  \"elapsed_time\": 0.07923507690429688,\n",
            "  \"wall_time\": 1674064292.4835126,\n",
            "  \"global_step\": 949\n",
            "}\n",
            "[2023-01-18 17:51:32,559] [INFO] [logging.py:68:log_dist] [Rank 0] step=950, skipped=1, lr=[0.000189652], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:32,560] [INFO] [timer.py:196:stop] epoch=0/micro_step=950/global_step=950, RunningAvgSamplesPerSec=13.675684067403722, CurrSamplesPerSec=13.208367842443213, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0024166107177734375,\n",
            "  \"model.lr\": 0.000189652,\n",
            "  \"model.grad_norm\": 0.11844455450773239,\n",
            "  \"model.elapsed_time\": 0.07618927955627441,\n",
            "  \"model.engine_step\": 950,\n",
            "  \"model.loss.nll\": 0.0024166107177734375,\n",
            "  \"elapsed_time\": 0.07618927955627441,\n",
            "  \"wall_time\": 1674064292.5622063,\n",
            "  \"global_step\": 950\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0005025863647460938,\n",
            "  \"model.lr\": 0.000189851,\n",
            "  \"model.grad_norm\": 0.011852174997329712,\n",
            "  \"model.elapsed_time\": 0.07542300224304199,\n",
            "  \"model.engine_step\": 951,\n",
            "  \"model.loss.nll\": 0.0005025863647460938,\n",
            "  \"elapsed_time\": 0.07542300224304199,\n",
            "  \"wall_time\": 1674064292.6455176,\n",
            "  \"global_step\": 951\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00047326087951660156,\n",
            "  \"model.lr\": 0.00019005,\n",
            "  \"model.grad_norm\": 0.009286964312195778,\n",
            "  \"model.elapsed_time\": 0.06689739227294922,\n",
            "  \"model.engine_step\": 952,\n",
            "  \"model.loss.nll\": 0.00047326087951660156,\n",
            "  \"elapsed_time\": 0.06689739227294922,\n",
            "  \"wall_time\": 1674064292.7144902,\n",
            "  \"global_step\": 952\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00041937828063964844,\n",
            "  \"model.lr\": 0.000190249,\n",
            "  \"model.grad_norm\": 0.008374388329684734,\n",
            "  \"model.elapsed_time\": 0.07251524925231934,\n",
            "  \"model.engine_step\": 953,\n",
            "  \"model.loss.nll\": 0.00041937828063964844,\n",
            "  \"elapsed_time\": 0.07251524925231934,\n",
            "  \"wall_time\": 1674064292.7949154,\n",
            "  \"global_step\": 953\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0017833709716796875,\n",
            "  \"model.lr\": 0.000190448,\n",
            "  \"model.grad_norm\": 0.06208634749054909,\n",
            "  \"model.elapsed_time\": 0.07620048522949219,\n",
            "  \"model.engine_step\": 954,\n",
            "  \"model.loss.nll\": 0.0017833709716796875,\n",
            "  \"elapsed_time\": 0.07620048522949219,\n",
            "  \"wall_time\": 1674064292.8732877,\n",
            "  \"global_step\": 954\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:32\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00042128562927246094,\n",
            "  \"model.lr\": 0.000190647,\n",
            "  \"model.grad_norm\": 0.007351769134402275,\n",
            "  \"model.elapsed_time\": 0.0733039379119873,\n",
            "  \"model.engine_step\": 955,\n",
            "  \"model.loss.nll\": 0.00042128562927246094,\n",
            "  \"elapsed_time\": 0.0733039379119873,\n",
            "  \"wall_time\": 1674064292.9549417,\n",
            "  \"global_step\": 955\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004162788391113281,\n",
            "  \"model.lr\": 0.000190846,\n",
            "  \"model.grad_norm\": 0.006876017432659864,\n",
            "  \"model.elapsed_time\": 0.06561112403869629,\n",
            "  \"model.engine_step\": 956,\n",
            "  \"model.loss.nll\": 0.0004162788391113281,\n",
            "  \"elapsed_time\": 0.06561112403869629,\n",
            "  \"wall_time\": 1674064293.0234363,\n",
            "  \"global_step\": 956\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003960132598876953,\n",
            "  \"model.lr\": 0.000191045,\n",
            "  \"model.grad_norm\": 0.004890585783869028,\n",
            "  \"model.elapsed_time\": 0.07240962982177734,\n",
            "  \"model.engine_step\": 957,\n",
            "  \"model.loss.nll\": 0.0003960132598876953,\n",
            "  \"elapsed_time\": 0.07240962982177734,\n",
            "  \"wall_time\": 1674064293.1048808,\n",
            "  \"global_step\": 957\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0021800994873046875,\n",
            "  \"model.lr\": 0.000191244,\n",
            "  \"model.grad_norm\": 0.06355362385511398,\n",
            "  \"model.elapsed_time\": 0.0674431324005127,\n",
            "  \"model.engine_step\": 958,\n",
            "  \"model.loss.nll\": 0.0021800994873046875,\n",
            "  \"elapsed_time\": 0.0674431324005127,\n",
            "  \"wall_time\": 1674064293.1755219,\n",
            "  \"global_step\": 958\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0024566650390625,\n",
            "  \"model.lr\": 0.000191443,\n",
            "  \"model.grad_norm\": 0.08831003308296204,\n",
            "  \"model.elapsed_time\": 0.0723714828491211,\n",
            "  \"model.engine_step\": 959,\n",
            "  \"model.loss.nll\": 0.0024566650390625,\n",
            "  \"elapsed_time\": 0.0723714828491211,\n",
            "  \"wall_time\": 1674064293.2580786,\n",
            "  \"global_step\": 959\n",
            "}\n",
            "[2023-01-18 17:51:33,327] [INFO] [logging.py:68:log_dist] [Rank 0] step=960, skipped=1, lr=[0.000191642], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:33,328] [INFO] [timer.py:196:stop] epoch=0/micro_step=960/global_step=960, RunningAvgSamplesPerSec=13.67985103207307, CurrSamplesPerSec=14.571701541486734, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00035309791564941406,\n",
            "  \"model.lr\": 0.000191642,\n",
            "  \"model.grad_norm\": 0.003611976746469736,\n",
            "  \"model.elapsed_time\": 0.0690925121307373,\n",
            "  \"model.engine_step\": 960,\n",
            "  \"model.loss.nll\": 0.00035309791564941406,\n",
            "  \"elapsed_time\": 0.0690925121307373,\n",
            "  \"wall_time\": 1674064293.329951,\n",
            "  \"global_step\": 960\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.000457763671875,\n",
            "  \"model.lr\": 0.000191841,\n",
            "  \"model.grad_norm\": 0.012122291140258312,\n",
            "  \"model.elapsed_time\": 0.06999611854553223,\n",
            "  \"model.engine_step\": 961,\n",
            "  \"model.loss.nll\": 0.000457763671875,\n",
            "  \"elapsed_time\": 0.06999611854553223,\n",
            "  \"wall_time\": 1674064293.4080112,\n",
            "  \"global_step\": 961\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004382133483886719,\n",
            "  \"model.lr\": 0.00019204,\n",
            "  \"model.grad_norm\": 0.008845568634569645,\n",
            "  \"model.elapsed_time\": 0.08048582077026367,\n",
            "  \"model.engine_step\": 962,\n",
            "  \"model.loss.nll\": 0.0004382133483886719,\n",
            "  \"elapsed_time\": 0.08048582077026367,\n",
            "  \"wall_time\": 1674064293.4909368,\n",
            "  \"global_step\": 962\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00035762786865234375,\n",
            "  \"model.lr\": 0.000192239,\n",
            "  \"model.grad_norm\": 0.0046011595986783504,\n",
            "  \"model.elapsed_time\": 0.07885503768920898,\n",
            "  \"model.engine_step\": 963,\n",
            "  \"model.loss.nll\": 0.00035762786865234375,\n",
            "  \"elapsed_time\": 0.07885503768920898,\n",
            "  \"wall_time\": 1674064293.578258,\n",
            "  \"global_step\": 963\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003597736358642578,\n",
            "  \"model.lr\": 0.000192438,\n",
            "  \"model.grad_norm\": 0.006454636808484793,\n",
            "  \"model.elapsed_time\": 0.06833100318908691,\n",
            "  \"model.engine_step\": 964,\n",
            "  \"model.loss.nll\": 0.0003597736358642578,\n",
            "  \"elapsed_time\": 0.06833100318908691,\n",
            "  \"wall_time\": 1674064293.6486905,\n",
            "  \"global_step\": 964\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0023097991943359375,\n",
            "  \"model.lr\": 0.000192637,\n",
            "  \"model.grad_norm\": 0.05615047737956047,\n",
            "  \"model.elapsed_time\": 0.06780099868774414,\n",
            "  \"model.engine_step\": 965,\n",
            "  \"model.loss.nll\": 0.0023097991943359375,\n",
            "  \"elapsed_time\": 0.06780099868774414,\n",
            "  \"wall_time\": 1674064293.7322557,\n",
            "  \"global_step\": 965\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00162506103515625,\n",
            "  \"model.lr\": 0.000192836,\n",
            "  \"model.grad_norm\": 0.0827396884560585,\n",
            "  \"model.elapsed_time\": 0.06823301315307617,\n",
            "  \"model.engine_step\": 966,\n",
            "  \"model.loss.nll\": 0.00162506103515625,\n",
            "  \"elapsed_time\": 0.06823301315307617,\n",
            "  \"wall_time\": 1674064293.802731,\n",
            "  \"global_step\": 966\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004036426544189453,\n",
            "  \"model.lr\": 0.000193035,\n",
            "  \"model.grad_norm\": 0.008038957603275776,\n",
            "  \"model.elapsed_time\": 0.07450652122497559,\n",
            "  \"model.engine_step\": 967,\n",
            "  \"model.loss.nll\": 0.0004036426544189453,\n",
            "  \"elapsed_time\": 0.07450652122497559,\n",
            "  \"wall_time\": 1674064293.8873525,\n",
            "  \"global_step\": 967\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0006184577941894531,\n",
            "  \"model.lr\": 0.000193234,\n",
            "  \"model.grad_norm\": 0.025529924780130386,\n",
            "  \"model.elapsed_time\": 0.0757746696472168,\n",
            "  \"model.engine_step\": 968,\n",
            "  \"model.loss.nll\": 0.0006184577941894531,\n",
            "  \"elapsed_time\": 0.0757746696472168,\n",
            "  \"wall_time\": 1674064293.9658358,\n",
            "  \"global_step\": 968\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:33\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003604888916015625,\n",
            "  \"model.lr\": 0.00019343300000000002,\n",
            "  \"model.grad_norm\": 0.004911195486783981,\n",
            "  \"model.elapsed_time\": 0.08045148849487305,\n",
            "  \"model.engine_step\": 969,\n",
            "  \"model.loss.nll\": 0.0003604888916015625,\n",
            "  \"elapsed_time\": 0.08045148849487305,\n",
            "  \"wall_time\": 1674064294.0557127,\n",
            "  \"global_step\": 969\n",
            "}\n",
            "[2023-01-18 17:51:34,122] [INFO] [logging.py:68:log_dist] [Rank 0] step=970, skipped=1, lr=[0.000193632], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:34,123] [INFO] [timer.py:196:stop] epoch=0/micro_step=970/global_step=970, RunningAvgSamplesPerSec=13.680038034549565, CurrSamplesPerSec=15.04979996053033, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0013713836669921875,\n",
            "  \"model.lr\": 0.000193632,\n",
            "  \"model.grad_norm\": 0.08516436070203781,\n",
            "  \"model.elapsed_time\": 0.06691455841064453,\n",
            "  \"model.engine_step\": 970,\n",
            "  \"model.loss.nll\": 0.0013713836669921875,\n",
            "  \"elapsed_time\": 0.06691455841064453,\n",
            "  \"wall_time\": 1674064294.1247704,\n",
            "  \"global_step\": 970\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004565715789794922,\n",
            "  \"model.lr\": 0.000193831,\n",
            "  \"model.grad_norm\": 0.00968612264841795,\n",
            "  \"model.elapsed_time\": 0.06865072250366211,\n",
            "  \"model.engine_step\": 971,\n",
            "  \"model.loss.nll\": 0.0004565715789794922,\n",
            "  \"elapsed_time\": 0.06865072250366211,\n",
            "  \"wall_time\": 1674064294.201754,\n",
            "  \"global_step\": 971\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003535747528076172,\n",
            "  \"model.lr\": 0.00019403,\n",
            "  \"model.grad_norm\": 0.005711346864700317,\n",
            "  \"model.elapsed_time\": 0.08006882667541504,\n",
            "  \"model.engine_step\": 972,\n",
            "  \"model.loss.nll\": 0.0003535747528076172,\n",
            "  \"elapsed_time\": 0.08006882667541504,\n",
            "  \"wall_time\": 1674064294.2844718,\n",
            "  \"global_step\": 972\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00165557861328125,\n",
            "  \"model.lr\": 0.000194229,\n",
            "  \"model.grad_norm\": 0.03102588653564453,\n",
            "  \"model.elapsed_time\": 0.06449222564697266,\n",
            "  \"model.engine_step\": 973,\n",
            "  \"model.loss.nll\": 0.00165557861328125,\n",
            "  \"elapsed_time\": 0.06449222564697266,\n",
            "  \"wall_time\": 1674064294.3566256,\n",
            "  \"global_step\": 973\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00038695335388183594,\n",
            "  \"model.lr\": 0.00019442800000000002,\n",
            "  \"model.grad_norm\": 0.005201127380132675,\n",
            "  \"model.elapsed_time\": 0.07183003425598145,\n",
            "  \"model.engine_step\": 974,\n",
            "  \"model.loss.nll\": 0.00038695335388183594,\n",
            "  \"elapsed_time\": 0.07183003425598145,\n",
            "  \"wall_time\": 1674064294.4306254,\n",
            "  \"global_step\": 974\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0008726119995117188,\n",
            "  \"model.lr\": 0.000194627,\n",
            "  \"model.grad_norm\": 0.016863064840435982,\n",
            "  \"model.elapsed_time\": 0.07288074493408203,\n",
            "  \"model.engine_step\": 975,\n",
            "  \"model.loss.nll\": 0.0008726119995117188,\n",
            "  \"elapsed_time\": 0.07288074493408203,\n",
            "  \"wall_time\": 1674064294.510013,\n",
            "  \"global_step\": 975\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00038361549377441406,\n",
            "  \"model.lr\": 0.000194826,\n",
            "  \"model.grad_norm\": 0.004849079996347427,\n",
            "  \"model.elapsed_time\": 0.07889032363891602,\n",
            "  \"model.engine_step\": 976,\n",
            "  \"model.loss.nll\": 0.00038361549377441406,\n",
            "  \"elapsed_time\": 0.07889032363891602,\n",
            "  \"wall_time\": 1674064294.5916765,\n",
            "  \"global_step\": 976\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00040078163146972656,\n",
            "  \"model.lr\": 0.000195025,\n",
            "  \"model.grad_norm\": 0.0071145398542284966,\n",
            "  \"model.elapsed_time\": 0.07874727249145508,\n",
            "  \"model.engine_step\": 977,\n",
            "  \"model.loss.nll\": 0.00040078163146972656,\n",
            "  \"elapsed_time\": 0.07874727249145508,\n",
            "  \"wall_time\": 1674064294.6801414,\n",
            "  \"global_step\": 977\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00043010711669921875,\n",
            "  \"model.lr\": 0.000195224,\n",
            "  \"model.grad_norm\": 0.00910128653049469,\n",
            "  \"model.elapsed_time\": 0.06860971450805664,\n",
            "  \"model.engine_step\": 978,\n",
            "  \"model.loss.nll\": 0.00043010711669921875,\n",
            "  \"elapsed_time\": 0.06860971450805664,\n",
            "  \"wall_time\": 1674064294.750863,\n",
            "  \"global_step\": 978\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0004265308380126953,\n",
            "  \"model.lr\": 0.00019542300000000002,\n",
            "  \"model.grad_norm\": 0.023968026041984558,\n",
            "  \"model.elapsed_time\": 0.07851791381835938,\n",
            "  \"model.engine_step\": 979,\n",
            "  \"model.loss.nll\": 0.0004265308380126953,\n",
            "  \"elapsed_time\": 0.07851791381835938,\n",
            "  \"wall_time\": 1674064294.8376343,\n",
            "  \"global_step\": 979\n",
            "}\n",
            "[2023-01-18 17:51:34,911] [INFO] [logging.py:68:log_dist] [Rank 0] step=980, skipped=1, lr=[0.000195622], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:34,913] [INFO] [timer.py:196:stop] epoch=0/micro_step=980/global_step=980, RunningAvgSamplesPerSec=13.679194084467786, CurrSamplesPerSec=13.584262312072081, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.001216888427734375,\n",
            "  \"model.lr\": 0.000195622,\n",
            "  \"model.grad_norm\": 0.031152023002505302,\n",
            "  \"model.elapsed_time\": 0.07408404350280762,\n",
            "  \"model.engine_step\": 980,\n",
            "  \"model.loss.nll\": 0.001216888427734375,\n",
            "  \"elapsed_time\": 0.07408404350280762,\n",
            "  \"wall_time\": 1674064294.914476,\n",
            "  \"global_step\": 980\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:34\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00038814544677734375,\n",
            "  \"model.lr\": 0.000195821,\n",
            "  \"model.grad_norm\": 0.006433330476284027,\n",
            "  \"model.elapsed_time\": 0.07074713706970215,\n",
            "  \"model.engine_step\": 981,\n",
            "  \"model.loss.nll\": 0.00038814544677734375,\n",
            "  \"elapsed_time\": 0.07074713706970215,\n",
            "  \"wall_time\": 1674064294.9959462,\n",
            "  \"global_step\": 981\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003476142883300781,\n",
            "  \"model.lr\": 0.00019602,\n",
            "  \"model.grad_norm\": 0.005304727237671614,\n",
            "  \"model.elapsed_time\": 0.07941675186157227,\n",
            "  \"model.engine_step\": 982,\n",
            "  \"model.loss.nll\": 0.0003476142883300781,\n",
            "  \"elapsed_time\": 0.07941675186157227,\n",
            "  \"wall_time\": 1674064295.077677,\n",
            "  \"global_step\": 982\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00034356117248535156,\n",
            "  \"model.lr\": 0.000196219,\n",
            "  \"model.grad_norm\": 0.007520612329244614,\n",
            "  \"model.elapsed_time\": 0.06870818138122559,\n",
            "  \"model.engine_step\": 983,\n",
            "  \"model.loss.nll\": 0.00034356117248535156,\n",
            "  \"elapsed_time\": 0.06870818138122559,\n",
            "  \"wall_time\": 1674064295.1553497,\n",
            "  \"global_step\": 983\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003445148468017578,\n",
            "  \"model.lr\": 0.00019641800000000002,\n",
            "  \"model.grad_norm\": 0.004065616987645626,\n",
            "  \"model.elapsed_time\": 0.07415318489074707,\n",
            "  \"model.engine_step\": 984,\n",
            "  \"model.loss.nll\": 0.0003445148468017578,\n",
            "  \"elapsed_time\": 0.07415318489074707,\n",
            "  \"wall_time\": 1674064295.2316904,\n",
            "  \"global_step\": 984\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003714561462402344,\n",
            "  \"model.lr\": 0.000196617,\n",
            "  \"model.grad_norm\": 0.005580452736467123,\n",
            "  \"model.elapsed_time\": 0.07380270957946777,\n",
            "  \"model.engine_step\": 985,\n",
            "  \"model.loss.nll\": 0.0003714561462402344,\n",
            "  \"elapsed_time\": 0.07380270957946777,\n",
            "  \"wall_time\": 1674064295.3143694,\n",
            "  \"global_step\": 985\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.000335693359375,\n",
            "  \"model.lr\": 0.000196816,\n",
            "  \"model.grad_norm\": 0.004571422003209591,\n",
            "  \"model.elapsed_time\": 0.07337760925292969,\n",
            "  \"model.engine_step\": 986,\n",
            "  \"model.loss.nll\": 0.000335693359375,\n",
            "  \"elapsed_time\": 0.07337760925292969,\n",
            "  \"wall_time\": 1674064295.3898025,\n",
            "  \"global_step\": 986\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009112358093261719,\n",
            "  \"model.lr\": 0.000197015,\n",
            "  \"model.grad_norm\": 0.02469298616051674,\n",
            "  \"model.elapsed_time\": 0.06678247451782227,\n",
            "  \"model.engine_step\": 987,\n",
            "  \"model.loss.nll\": 0.0009112358093261719,\n",
            "  \"elapsed_time\": 0.06678247451782227,\n",
            "  \"wall_time\": 1674064295.4641123,\n",
            "  \"global_step\": 987\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010919570922851562,\n",
            "  \"model.lr\": 0.000197214,\n",
            "  \"model.grad_norm\": 0.02809477411210537,\n",
            "  \"model.elapsed_time\": 0.0663154125213623,\n",
            "  \"model.engine_step\": 988,\n",
            "  \"model.loss.nll\": 0.0010919570922851562,\n",
            "  \"elapsed_time\": 0.0663154125213623,\n",
            "  \"wall_time\": 1674064295.5329916,\n",
            "  \"global_step\": 988\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0016736984252929688,\n",
            "  \"model.lr\": 0.00019741300000000002,\n",
            "  \"model.grad_norm\": 0.041600875556468964,\n",
            "  \"model.elapsed_time\": 0.07596373558044434,\n",
            "  \"model.engine_step\": 989,\n",
            "  \"model.loss.nll\": 0.0016736984252929688,\n",
            "  \"elapsed_time\": 0.07596373558044434,\n",
            "  \"wall_time\": 1674064295.6173427,\n",
            "  \"global_step\": 989\n",
            "}\n",
            "[2023-01-18 17:51:35,698] [INFO] [logging.py:68:log_dist] [Rank 0] step=990, skipped=1, lr=[0.000197612], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:35,699] [INFO] [timer.py:196:stop] epoch=0/micro_step=990/global_step=990, RunningAvgSamplesPerSec=13.679550470599153, CurrSamplesPerSec=12.397006490665973, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009589195251464844,\n",
            "  \"model.lr\": 0.000197612,\n",
            "  \"model.grad_norm\": 0.015789667144417763,\n",
            "  \"model.elapsed_time\": 0.08114933967590332,\n",
            "  \"model.engine_step\": 990,\n",
            "  \"model.loss.nll\": 0.0009589195251464844,\n",
            "  \"elapsed_time\": 0.08114933967590332,\n",
            "  \"wall_time\": 1674064295.7007182,\n",
            "  \"global_step\": 990\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00035119056701660156,\n",
            "  \"model.lr\": 0.00019781100000000002,\n",
            "  \"model.grad_norm\": 0.008238237351179123,\n",
            "  \"model.elapsed_time\": 0.07364726066589355,\n",
            "  \"model.engine_step\": 991,\n",
            "  \"model.loss.nll\": 0.00035119056701660156,\n",
            "  \"elapsed_time\": 0.07364726066589355,\n",
            "  \"wall_time\": 1674064295.782783,\n",
            "  \"global_step\": 991\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0010585784912109375,\n",
            "  \"model.lr\": 0.00019801,\n",
            "  \"model.grad_norm\": 0.022551221773028374,\n",
            "  \"model.elapsed_time\": 0.06811094284057617,\n",
            "  \"model.engine_step\": 992,\n",
            "  \"model.loss.nll\": 0.0010585784912109375,\n",
            "  \"elapsed_time\": 0.06811094284057617,\n",
            "  \"wall_time\": 1674064295.8529034,\n",
            "  \"global_step\": 992\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:35\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0009045600891113281,\n",
            "  \"model.lr\": 0.000198209,\n",
            "  \"model.grad_norm\": 0.012984487228095531,\n",
            "  \"model.elapsed_time\": 0.06597042083740234,\n",
            "  \"model.engine_step\": 993,\n",
            "  \"model.loss.nll\": 0.0009045600891113281,\n",
            "  \"elapsed_time\": 0.06597042083740234,\n",
            "  \"wall_time\": 1674064295.9265378,\n",
            "  \"global_step\": 993\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00037169456481933594,\n",
            "  \"model.lr\": 0.00019840800000000002,\n",
            "  \"model.grad_norm\": 0.010395930148661137,\n",
            "  \"model.elapsed_time\": 0.08343172073364258,\n",
            "  \"model.engine_step\": 994,\n",
            "  \"model.loss.nll\": 0.00037169456481933594,\n",
            "  \"elapsed_time\": 0.08343172073364258,\n",
            "  \"wall_time\": 1674064296.0119958,\n",
            "  \"global_step\": 994\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00032329559326171875,\n",
            "  \"model.lr\": 0.000198607,\n",
            "  \"model.grad_norm\": 0.005221412982791662,\n",
            "  \"model.elapsed_time\": 0.07179784774780273,\n",
            "  \"model.engine_step\": 995,\n",
            "  \"model.loss.nll\": 0.00032329559326171875,\n",
            "  \"elapsed_time\": 0.07179784774780273,\n",
            "  \"wall_time\": 1674064296.0912716,\n",
            "  \"global_step\": 995\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0012111663818359375,\n",
            "  \"model.lr\": 0.00019880600000000002,\n",
            "  \"model.grad_norm\": 0.054647572338581085,\n",
            "  \"model.elapsed_time\": 0.06981897354125977,\n",
            "  \"model.engine_step\": 996,\n",
            "  \"model.loss.nll\": 0.0012111663818359375,\n",
            "  \"elapsed_time\": 0.06981897354125977,\n",
            "  \"wall_time\": 1674064296.1630049,\n",
            "  \"global_step\": 996\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003018379211425781,\n",
            "  \"model.lr\": 0.000199005,\n",
            "  \"model.grad_norm\": 0.0034441754687577486,\n",
            "  \"model.elapsed_time\": 0.07388901710510254,\n",
            "  \"model.engine_step\": 997,\n",
            "  \"model.loss.nll\": 0.0003018379211425781,\n",
            "  \"elapsed_time\": 0.07388901710510254,\n",
            "  \"wall_time\": 1674064296.2448382,\n",
            "  \"global_step\": 997\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0003120899200439453,\n",
            "  \"model.lr\": 0.000199204,\n",
            "  \"model.grad_norm\": 0.00481032719835639,\n",
            "  \"model.elapsed_time\": 0.06789016723632812,\n",
            "  \"model.engine_step\": 998,\n",
            "  \"model.loss.nll\": 0.0003120899200439453,\n",
            "  \"elapsed_time\": 0.06789016723632812,\n",
            "  \"wall_time\": 1674064296.3148885,\n",
            "  \"global_step\": 998\n",
            "}\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00032639503479003906,\n",
            "  \"model.lr\": 0.00019940300000000002,\n",
            "  \"model.grad_norm\": 0.004049413837492466,\n",
            "  \"model.elapsed_time\": 0.06686139106750488,\n",
            "  \"model.engine_step\": 999,\n",
            "  \"model.loss.nll\": 0.00032639503479003906,\n",
            "  \"elapsed_time\": 0.06686139106750488,\n",
            "  \"wall_time\": 1674064296.389542,\n",
            "  \"global_step\": 999\n",
            "}\n",
            "[2023-01-18 17:51:36,456] [INFO] [logging.py:68:log_dist] [Rank 0] step=1000, skipped=1, lr=[0.000199602], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:51:36,458] [INFO] [timer.py:196:stop] epoch=0/micro_step=1000/global_step=1000, RunningAvgSamplesPerSec=13.683929868461531, CurrSamplesPerSec=14.865353194897803, MemAllocated=0.16GB, MaxMemAllocated=0.3GB\n",
            "\u001b[32m2023-01-18 17:51:36\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.00038695335388183594,\n",
            "  \"model.lr\": 0.000199602,\n",
            "  \"model.grad_norm\": 0.012455814518034458,\n",
            "  \"model.elapsed_time\": 0.06771302223205566,\n",
            "  \"model.engine_step\": 1000,\n",
            "  \"model.loss.nll\": 0.00038695335388183594,\n",
            "  \"elapsed_time\": 0.06771302223205566,\n",
            "  \"wall_time\": 1674064296.4592206,\n",
            "  \"global_step\": 1000\n",
            "}\n",
            "[2023-01-18 17:51:36,460] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint default is begin to save!\n",
            "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
            "  warnings.warn(\n",
            "[2023-01-18 17:51:36,462] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ckpts/test/ar/model/default/mp_rank_00_model_states.pt\n",
            "[2023-01-18 17:51:36,462] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ckpts/test/ar/model/default/mp_rank_00_model_states.pt...\n",
            "[2023-01-18 17:51:36,884] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ckpts/test/ar/model/default/mp_rank_00_model_states.pt.\n",
            "[2023-01-18 17:51:36,884] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint default is ready now!\n",
            "  0% 0/2 [00:00<?, ?it/s]\n",
            "  0% 0/300 [00:00<?, ?it/s]\u001b[A\n",
            "  3% 8/300 [00:00<00:03, 79.54it/s]\u001b[A\n",
            "  6% 17/300 [00:00<00:03, 81.60it/s]\u001b[A\n",
            "  9% 26/300 [00:00<00:03, 83.37it/s]\u001b[A\n",
            " 12% 35/300 [00:00<00:03, 80.98it/s]\u001b[A\n",
            " 15% 44/300 [00:00<00:03, 82.90it/s]\u001b[A\n",
            " 18% 53/300 [00:00<00:02, 84.05it/s]\u001b[A\n",
            " 21% 62/300 [00:00<00:03, 78.79it/s]\u001b[A\n",
            " 24% 71/300 [00:00<00:02, 81.18it/s]\u001b[A\n",
            " 27% 80/300 [00:00<00:02, 82.20it/s]\u001b[A\n",
            " 30% 89/300 [00:01<00:02, 81.43it/s]\u001b[A\n",
            " 33% 99/300 [00:01<00:02, 84.30it/s]\u001b[A\n",
            " 36% 109/300 [00:01<00:02, 85.77it/s]\u001b[A\n",
            " 39% 118/300 [00:01<00:02, 81.71it/s]\u001b[A\n",
            " 43% 128/300 [00:01<00:02, 84.45it/s]\u001b[A\n",
            " 46% 137/300 [00:01<00:01, 84.00it/s]\u001b[A\n",
            " 50% 149/300 [00:01<00:01, 82.65it/s]\n",
            " 50% 1/2 [00:02<00:02,  2.17s/it]\n",
            "  0% 0/300 [00:00<?, ?it/s]\u001b[A\n",
            "  3% 9/300 [00:00<00:03, 84.87it/s]\u001b[A\n",
            "  6% 18/300 [00:00<00:03, 85.95it/s]\u001b[A\n",
            "  9% 27/300 [00:00<00:03, 85.09it/s]\u001b[A\n",
            " 12% 37/300 [00:00<00:02, 88.36it/s]\u001b[A\n",
            " 16% 47/300 [00:00<00:02, 89.22it/s]\u001b[A\n",
            " 19% 56/300 [00:00<00:02, 83.63it/s]\u001b[A\n",
            " 22% 66/300 [00:00<00:02, 86.20it/s]\u001b[A\n",
            " 25% 76/300 [00:00<00:02, 88.63it/s]\u001b[A\n",
            " 28% 85/300 [00:00<00:02, 84.91it/s]\u001b[A\n",
            " 32% 95/300 [00:01<00:02, 86.93it/s]\u001b[A\n",
            " 35% 105/300 [00:01<00:02, 88.79it/s]\u001b[A\n",
            " 38% 114/300 [00:01<00:02, 87.84it/s]\u001b[A\n",
            " 41% 123/300 [00:01<00:02, 87.94it/s]\u001b[A\n",
            " 44% 133/300 [00:01<00:01, 90.74it/s]\u001b[A\n",
            " 50% 149/300 [00:01<00:01, 86.98it/s]\n",
            "100% 2/2 [00:03<00:00,  1.96s/it]\n",
            "\u001b[32m2023-01-18 17:51:40\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "Eval: {'global_step': 1000, 'name': 'subtrain'}.\n",
            "\u001b[32m2023-01-18 17:51:40\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\"global_step\": 1000, \"name\": \"subtrain\"}.\n",
            "0it [00:00, ?it/s]\n",
            "\u001b[32m2023-01-18 17:51:40\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "Eval: {'global_step': 1000, 'name': 'val'}.\n",
            "\u001b[32m2023-01-18 17:51:40\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\"global_step\": 1000, \"name\": \"val\"}.\n",
            "\u001b[32m2023-01-18 17:51:40\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m vall_e.train yaml=config/test/nar.yml"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hEd3Rrsk_WT6",
        "outputId": "0c8ade10-7d57-4ef1-9110-f71e3d2af817"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1;30;43mStreaming output truncated to the last 5000 lines.\u001b[0m\n",
            "  \"model.loss.nll\": 0.0823974609375,\n",
            "  \"elapsed_time\": 0.11907958984375,\n",
            "  \"wall_time\": 1674064368.5964453,\n",
            "  \"global_step\": 623\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:48\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1510009765625,\n",
            "  \"model.lr\": 0.000124181,\n",
            "  \"model.grad_norm\": 3.0945584774017334,\n",
            "  \"model.elapsed_time\": 0.11585307121276855,\n",
            "  \"model.engine_step\": 624,\n",
            "  \"model.loss.nll\": 0.1510009765625,\n",
            "  \"elapsed_time\": 0.11585307121276855,\n",
            "  \"wall_time\": 1674064368.7154737,\n",
            "  \"global_step\": 624\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:48\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:48\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1009521484375,\n",
            "  \"model.lr\": 0.00012438,\n",
            "  \"model.grad_norm\": 2.300785779953003,\n",
            "  \"model.elapsed_time\": 0.11938118934631348,\n",
            "  \"model.engine_step\": 625,\n",
            "  \"model.loss.nll\": 0.1009521484375,\n",
            "  \"elapsed_time\": 0.11938118934631348,\n",
            "  \"wall_time\": 1674064368.843749,\n",
            "  \"global_step\": 625\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:48\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.23681640625,\n",
            "  \"model.lr\": 0.00012457900000000001,\n",
            "  \"model.grad_norm\": 3.840081214904785,\n",
            "  \"model.elapsed_time\": 0.11678767204284668,\n",
            "  \"model.engine_step\": 626,\n",
            "  \"model.loss.nll\": 0.23681640625,\n",
            "  \"elapsed_time\": 0.11678767204284668,\n",
            "  \"wall_time\": 1674064368.9635103,\n",
            "  \"global_step\": 626\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:48\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.378173828125,\n",
            "  \"model.lr\": 0.000124778,\n",
            "  \"model.grad_norm\": 5.766915798187256,\n",
            "  \"model.elapsed_time\": 0.11444497108459473,\n",
            "  \"model.engine_step\": 627,\n",
            "  \"model.loss.nll\": 0.378173828125,\n",
            "  \"elapsed_time\": 0.11444497108459473,\n",
            "  \"wall_time\": 1674064369.0872524,\n",
            "  \"global_step\": 627\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 3.333984375,\n",
            "  \"model.lr\": 0.000124977,\n",
            "  \"model.grad_norm\": 17.463035583496094,\n",
            "  \"model.elapsed_time\": 0.11387491226196289,\n",
            "  \"model.engine_step\": 628,\n",
            "  \"model.loss.nll\": 3.333984375,\n",
            "  \"elapsed_time\": 0.11387491226196289,\n",
            "  \"wall_time\": 1674064369.2049048,\n",
            "  \"global_step\": 628\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.102783203125,\n",
            "  \"model.lr\": 0.000125176,\n",
            "  \"model.grad_norm\": 1.9385052919387817,\n",
            "  \"model.elapsed_time\": 0.11134624481201172,\n",
            "  \"model.engine_step\": 629,\n",
            "  \"model.loss.nll\": 0.102783203125,\n",
            "  \"elapsed_time\": 0.11134624481201172,\n",
            "  \"wall_time\": 1674064369.327032,\n",
            "  \"global_step\": 629\n",
            "}\n",
            "[2023-01-18 17:52:49,441] [INFO] [logging.py:68:log_dist] [Rank 0] step=630, skipped=4, lr=[0.000125375], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:49,442] [INFO] [timer.py:196:stop] epoch=0/micro_step=630/global_step=630, RunningAvgSamplesPerSec=12.617538181444964, CurrSamplesPerSec=8.785943509262905, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.12335205078125,\n",
            "  \"model.lr\": 0.000125375,\n",
            "  \"model.grad_norm\": 2.057163715362549,\n",
            "  \"model.elapsed_time\": 0.11456537246704102,\n",
            "  \"model.engine_step\": 630,\n",
            "  \"model.loss.nll\": 0.12335205078125,\n",
            "  \"elapsed_time\": 0.11456537246704102,\n",
            "  \"wall_time\": 1674064369.4446495,\n",
            "  \"global_step\": 630\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.030548095703125,\n",
            "  \"model.lr\": 0.00012557400000000001,\n",
            "  \"model.grad_norm\": 0.7149236798286438,\n",
            "  \"model.elapsed_time\": 0.11058616638183594,\n",
            "  \"model.engine_step\": 631,\n",
            "  \"model.loss.nll\": 0.030548095703125,\n",
            "  \"elapsed_time\": 0.11058616638183594,\n",
            "  \"wall_time\": 1674064369.5653365,\n",
            "  \"global_step\": 631\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.97265625,\n",
            "  \"model.lr\": 0.000125773,\n",
            "  \"model.grad_norm\": 10.635979652404785,\n",
            "  \"model.elapsed_time\": 0.11872982978820801,\n",
            "  \"model.engine_step\": 632,\n",
            "  \"model.loss.nll\": 0.97265625,\n",
            "  \"elapsed_time\": 0.11872982978820801,\n",
            "  \"wall_time\": 1674064369.6900933,\n",
            "  \"global_step\": 632\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1790771484375,\n",
            "  \"model.lr\": 0.000125972,\n",
            "  \"model.grad_norm\": 2.702669143676758,\n",
            "  \"model.elapsed_time\": 0.12814593315124512,\n",
            "  \"model.engine_step\": 633,\n",
            "  \"model.loss.nll\": 0.1790771484375,\n",
            "  \"elapsed_time\": 0.12814593315124512,\n",
            "  \"wall_time\": 1674064369.8297532,\n",
            "  \"global_step\": 633\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07452392578125,\n",
            "  \"model.lr\": 0.000126171,\n",
            "  \"model.grad_norm\": 1.3674843311309814,\n",
            "  \"model.elapsed_time\": 0.12691450119018555,\n",
            "  \"model.engine_step\": 634,\n",
            "  \"model.loss.nll\": 0.07452392578125,\n",
            "  \"elapsed_time\": 0.12691450119018555,\n",
            "  \"wall_time\": 1674064369.9605267,\n",
            "  \"global_step\": 634\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:49\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.250732421875,\n",
            "  \"model.lr\": 0.00012637,\n",
            "  \"model.grad_norm\": 5.070903301239014,\n",
            "  \"model.elapsed_time\": 0.11677670478820801,\n",
            "  \"model.engine_step\": 635,\n",
            "  \"model.loss.nll\": 0.250732421875,\n",
            "  \"elapsed_time\": 0.11677670478820801,\n",
            "  \"wall_time\": 1674064370.086722,\n",
            "  \"global_step\": 635\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 1.19140625,\n",
            "  \"model.lr\": 0.00012656900000000001,\n",
            "  \"model.grad_norm\": 9.696891784667969,\n",
            "  \"model.elapsed_time\": 0.11373114585876465,\n",
            "  \"model.engine_step\": 636,\n",
            "  \"model.loss.nll\": 1.19140625,\n",
            "  \"elapsed_time\": 0.11373114585876465,\n",
            "  \"wall_time\": 1674064370.2037055,\n",
            "  \"global_step\": 636\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.31591796875,\n",
            "  \"model.lr\": 0.000126768,\n",
            "  \"model.grad_norm\": 5.768166542053223,\n",
            "  \"model.elapsed_time\": 0.11435961723327637,\n",
            "  \"model.engine_step\": 637,\n",
            "  \"model.loss.nll\": 0.31591796875,\n",
            "  \"elapsed_time\": 0.11435961723327637,\n",
            "  \"wall_time\": 1674064370.3268862,\n",
            "  \"global_step\": 637\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07672119140625,\n",
            "  \"model.lr\": 0.00012696700000000002,\n",
            "  \"model.grad_norm\": 1.7858718633651733,\n",
            "  \"model.elapsed_time\": 0.11450481414794922,\n",
            "  \"model.engine_step\": 638,\n",
            "  \"model.loss.nll\": 0.07672119140625,\n",
            "  \"elapsed_time\": 0.11450481414794922,\n",
            "  \"wall_time\": 1674064370.445113,\n",
            "  \"global_step\": 638\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.09588623046875,\n",
            "  \"model.lr\": 0.000127166,\n",
            "  \"model.grad_norm\": 1.8557199239730835,\n",
            "  \"model.elapsed_time\": 0.12663626670837402,\n",
            "  \"model.engine_step\": 639,\n",
            "  \"model.loss.nll\": 0.09588623046875,\n",
            "  \"elapsed_time\": 0.12663626670837402,\n",
            "  \"wall_time\": 1674064370.5883145,\n",
            "  \"global_step\": 639\n",
            "}\n",
            "[2023-01-18 17:52:50,709] [INFO] [logging.py:68:log_dist] [Rank 0] step=640, skipped=4, lr=[0.000127365], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:50,710] [INFO] [timer.py:196:stop] epoch=0/micro_step=640/global_step=640, RunningAvgSamplesPerSec=12.519373940249114, CurrSamplesPerSec=8.32773227705473, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1080322265625,\n",
            "  \"model.lr\": 0.000127365,\n",
            "  \"model.grad_norm\": 1.8936245441436768,\n",
            "  \"model.elapsed_time\": 0.12080836296081543,\n",
            "  \"model.engine_step\": 640,\n",
            "  \"model.loss.nll\": 0.1080322265625,\n",
            "  \"elapsed_time\": 0.12080836296081543,\n",
            "  \"wall_time\": 1674064370.7123237,\n",
            "  \"global_step\": 640\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 1.4267578125,\n",
            "  \"model.lr\": 0.00012756400000000001,\n",
            "  \"model.grad_norm\": 11.696706771850586,\n",
            "  \"model.elapsed_time\": 0.12431168556213379,\n",
            "  \"model.engine_step\": 641,\n",
            "  \"model.loss.nll\": 1.4267578125,\n",
            "  \"elapsed_time\": 0.12431168556213379,\n",
            "  \"wall_time\": 1674064370.8471107,\n",
            "  \"global_step\": 641\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0948486328125,\n",
            "  \"model.lr\": 0.000127763,\n",
            "  \"model.grad_norm\": 1.744415283203125,\n",
            "  \"model.elapsed_time\": 0.11296916007995605,\n",
            "  \"model.engine_step\": 642,\n",
            "  \"model.loss.nll\": 0.0948486328125,\n",
            "  \"elapsed_time\": 0.11296916007995605,\n",
            "  \"wall_time\": 1674064370.9635139,\n",
            "  \"global_step\": 642\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:50\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1131591796875,\n",
            "  \"model.lr\": 0.00012796200000000002,\n",
            "  \"model.grad_norm\": 1.8425102233886719,\n",
            "  \"model.elapsed_time\": 0.12436032295227051,\n",
            "  \"model.engine_step\": 643,\n",
            "  \"model.loss.nll\": 0.1131591796875,\n",
            "  \"elapsed_time\": 0.12436032295227051,\n",
            "  \"wall_time\": 1674064371.0999377,\n",
            "  \"global_step\": 643\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1357421875,\n",
            "  \"model.lr\": 0.000128161,\n",
            "  \"model.grad_norm\": 2.8465635776519775,\n",
            "  \"model.elapsed_time\": 0.12711453437805176,\n",
            "  \"model.engine_step\": 644,\n",
            "  \"model.loss.nll\": 0.1357421875,\n",
            "  \"elapsed_time\": 0.12711453437805176,\n",
            "  \"wall_time\": 1674064371.2304587,\n",
            "  \"global_step\": 644\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.052490234375,\n",
            "  \"model.lr\": 0.00012836,\n",
            "  \"model.grad_norm\": 1.0294458866119385,\n",
            "  \"model.elapsed_time\": 0.12840056419372559,\n",
            "  \"model.engine_step\": 645,\n",
            "  \"model.loss.nll\": 0.052490234375,\n",
            "  \"elapsed_time\": 0.12840056419372559,\n",
            "  \"wall_time\": 1674064371.3702137,\n",
            "  \"global_step\": 645\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.037353515625,\n",
            "  \"model.lr\": 0.00012855900000000001,\n",
            "  \"model.grad_norm\": 0.651430070400238,\n",
            "  \"model.elapsed_time\": 0.11570572853088379,\n",
            "  \"model.engine_step\": 646,\n",
            "  \"model.loss.nll\": 0.037353515625,\n",
            "  \"elapsed_time\": 0.11570572853088379,\n",
            "  \"wall_time\": 1674064371.4890378,\n",
            "  \"global_step\": 646\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.043548583984375,\n",
            "  \"model.lr\": 0.000128758,\n",
            "  \"model.grad_norm\": 0.816012442111969,\n",
            "  \"model.elapsed_time\": 0.1113436222076416,\n",
            "  \"model.engine_step\": 647,\n",
            "  \"model.loss.nll\": 0.043548583984375,\n",
            "  \"elapsed_time\": 0.1113436222076416,\n",
            "  \"wall_time\": 1674064371.61225,\n",
            "  \"global_step\": 647\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.515625,\n",
            "  \"model.lr\": 0.00012895700000000002,\n",
            "  \"model.grad_norm\": 7.381099224090576,\n",
            "  \"model.elapsed_time\": 0.1391746997833252,\n",
            "  \"model.engine_step\": 648,\n",
            "  \"model.loss.nll\": 0.515625,\n",
            "  \"elapsed_time\": 0.1391746997833252,\n",
            "  \"wall_time\": 1674064371.7576914,\n",
            "  \"global_step\": 648\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:51\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.410888671875,\n",
            "  \"model.lr\": 0.000129156,\n",
            "  \"model.grad_norm\": 5.764431476593018,\n",
            "  \"model.elapsed_time\": 0.12972640991210938,\n",
            "  \"model.engine_step\": 649,\n",
            "  \"model.loss.nll\": 0.410888671875,\n",
            "  \"elapsed_time\": 0.12972640991210938,\n",
            "  \"wall_time\": 1674064371.9012094,\n",
            "  \"global_step\": 649\n",
            "}\n",
            "[2023-01-18 17:52:52,024] [INFO] [logging.py:68:log_dist] [Rank 0] step=650, skipped=4, lr=[0.000129355], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:52,025] [INFO] [timer.py:196:stop] epoch=0/micro_step=650/global_step=650, RunningAvgSamplesPerSec=12.415070761626648, CurrSamplesPerSec=8.202557588771485, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.056549072265625,\n",
            "  \"model.lr\": 0.000129355,\n",
            "  \"model.grad_norm\": 1.3892923593521118,\n",
            "  \"model.elapsed_time\": 0.1226503849029541,\n",
            "  \"model.engine_step\": 650,\n",
            "  \"model.loss.nll\": 0.056549072265625,\n",
            "  \"elapsed_time\": 0.1226503849029541,\n",
            "  \"wall_time\": 1674064372.0274878,\n",
            "  \"global_step\": 650\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.172607421875,\n",
            "  \"model.lr\": 0.000129554,\n",
            "  \"model.grad_norm\": 3.9538214206695557,\n",
            "  \"model.elapsed_time\": 0.11802315711975098,\n",
            "  \"model.engine_step\": 651,\n",
            "  \"model.loss.nll\": 0.172607421875,\n",
            "  \"elapsed_time\": 0.11802315711975098,\n",
            "  \"wall_time\": 1674064372.1571743,\n",
            "  \"global_step\": 651\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.030303955078125,\n",
            "  \"model.lr\": 0.000129753,\n",
            "  \"model.grad_norm\": 0.4927171766757965,\n",
            "  \"model.elapsed_time\": 0.1160285472869873,\n",
            "  \"model.engine_step\": 652,\n",
            "  \"model.loss.nll\": 0.030303955078125,\n",
            "  \"elapsed_time\": 0.1160285472869873,\n",
            "  \"wall_time\": 1674064372.279199,\n",
            "  \"global_step\": 652\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.08514404296875,\n",
            "  \"model.lr\": 0.00012995200000000002,\n",
            "  \"model.grad_norm\": 2.0143702030181885,\n",
            "  \"model.elapsed_time\": 0.112884521484375,\n",
            "  \"model.engine_step\": 653,\n",
            "  \"model.loss.nll\": 0.08514404296875,\n",
            "  \"elapsed_time\": 0.112884521484375,\n",
            "  \"wall_time\": 1674064372.4035032,\n",
            "  \"global_step\": 653\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0322265625,\n",
            "  \"model.lr\": 0.000130151,\n",
            "  \"model.grad_norm\": 0.5714519619941711,\n",
            "  \"model.elapsed_time\": 0.12351012229919434,\n",
            "  \"model.engine_step\": 654,\n",
            "  \"model.loss.nll\": 0.0322265625,\n",
            "  \"elapsed_time\": 0.12351012229919434,\n",
            "  \"wall_time\": 1674064372.5301764,\n",
            "  \"global_step\": 654\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03265380859375,\n",
            "  \"model.lr\": 0.00013035,\n",
            "  \"model.grad_norm\": 0.6087841987609863,\n",
            "  \"model.elapsed_time\": 0.11495447158813477,\n",
            "  \"model.engine_step\": 655,\n",
            "  \"model.loss.nll\": 0.03265380859375,\n",
            "  \"elapsed_time\": 0.11495447158813477,\n",
            "  \"wall_time\": 1674064372.6570265,\n",
            "  \"global_step\": 655\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.078857421875,\n",
            "  \"model.lr\": 0.000130549,\n",
            "  \"model.grad_norm\": 1.3936022520065308,\n",
            "  \"model.elapsed_time\": 0.11446189880371094,\n",
            "  \"model.engine_step\": 656,\n",
            "  \"model.loss.nll\": 0.078857421875,\n",
            "  \"elapsed_time\": 0.11446189880371094,\n",
            "  \"wall_time\": 1674064372.7754045,\n",
            "  \"global_step\": 656\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:52\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03936767578125,\n",
            "  \"model.lr\": 0.000130748,\n",
            "  \"model.grad_norm\": 0.7403634190559387,\n",
            "  \"model.elapsed_time\": 0.11318659782409668,\n",
            "  \"model.engine_step\": 657,\n",
            "  \"model.loss.nll\": 0.03936767578125,\n",
            "  \"elapsed_time\": 0.11318659782409668,\n",
            "  \"wall_time\": 1674064372.8992555,\n",
            "  \"global_step\": 657\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.5087890625,\n",
            "  \"model.lr\": 0.00013094700000000002,\n",
            "  \"model.grad_norm\": 7.566792964935303,\n",
            "  \"model.elapsed_time\": 0.12001752853393555,\n",
            "  \"model.engine_step\": 658,\n",
            "  \"model.loss.nll\": 0.5087890625,\n",
            "  \"elapsed_time\": 0.12001752853393555,\n",
            "  \"wall_time\": 1674064373.0229452,\n",
            "  \"global_step\": 658\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.09686279296875,\n",
            "  \"model.lr\": 0.000131146,\n",
            "  \"model.grad_norm\": 1.7615442276000977,\n",
            "  \"model.elapsed_time\": 0.10713887214660645,\n",
            "  \"model.engine_step\": 659,\n",
            "  \"model.loss.nll\": 0.09686279296875,\n",
            "  \"elapsed_time\": 0.10713887214660645,\n",
            "  \"wall_time\": 1674064373.1415439,\n",
            "  \"global_step\": 659\n",
            "}\n",
            "[2023-01-18 17:52:53,258] [INFO] [logging.py:68:log_dist] [Rank 0] step=660, skipped=4, lr=[0.00013134500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:53,260] [INFO] [timer.py:196:stop] epoch=0/micro_step=660/global_step=660, RunningAvgSamplesPerSec=12.33389170126628, CurrSamplesPerSec=8.609899640561718, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0802001953125,\n",
            "  \"model.lr\": 0.00013134500000000002,\n",
            "  \"model.grad_norm\": 2.007503032684326,\n",
            "  \"model.elapsed_time\": 0.11678957939147949,\n",
            "  \"model.engine_step\": 660,\n",
            "  \"model.loss.nll\": 0.0802001953125,\n",
            "  \"elapsed_time\": 0.11678957939147949,\n",
            "  \"wall_time\": 1674064373.2618284,\n",
            "  \"global_step\": 660\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1357421875,\n",
            "  \"model.lr\": 0.000131544,\n",
            "  \"model.grad_norm\": 1.9817140102386475,\n",
            "  \"model.elapsed_time\": 0.11427855491638184,\n",
            "  \"model.engine_step\": 661,\n",
            "  \"model.loss.nll\": 0.1357421875,\n",
            "  \"elapsed_time\": 0.11427855491638184,\n",
            "  \"wall_time\": 1674064373.3844335,\n",
            "  \"global_step\": 661\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.041015625,\n",
            "  \"model.lr\": 0.000131743,\n",
            "  \"model.grad_norm\": 0.7355904579162598,\n",
            "  \"model.elapsed_time\": 0.1170649528503418,\n",
            "  \"model.engine_step\": 662,\n",
            "  \"model.loss.nll\": 0.041015625,\n",
            "  \"elapsed_time\": 0.1170649528503418,\n",
            "  \"wall_time\": 1674064373.5050704,\n",
            "  \"global_step\": 662\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0784912109375,\n",
            "  \"model.lr\": 0.00013194200000000002,\n",
            "  \"model.grad_norm\": 1.3013123273849487,\n",
            "  \"model.elapsed_time\": 0.1302030086517334,\n",
            "  \"model.engine_step\": 663,\n",
            "  \"model.loss.nll\": 0.0784912109375,\n",
            "  \"elapsed_time\": 0.1302030086517334,\n",
            "  \"wall_time\": 1674064373.6445699,\n",
            "  \"global_step\": 663\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06756591796875,\n",
            "  \"model.lr\": 0.000132141,\n",
            "  \"model.grad_norm\": 1.437017798423767,\n",
            "  \"model.elapsed_time\": 0.11867046356201172,\n",
            "  \"model.engine_step\": 664,\n",
            "  \"model.loss.nll\": 0.06756591796875,\n",
            "  \"elapsed_time\": 0.11867046356201172,\n",
            "  \"wall_time\": 1674064373.7669687,\n",
            "  \"global_step\": 664\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:53\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.050384521484375,\n",
            "  \"model.lr\": 0.00013234000000000002,\n",
            "  \"model.grad_norm\": 0.8263425827026367,\n",
            "  \"model.elapsed_time\": 0.11809945106506348,\n",
            "  \"model.engine_step\": 665,\n",
            "  \"model.loss.nll\": 0.050384521484375,\n",
            "  \"elapsed_time\": 0.11809945106506348,\n",
            "  \"wall_time\": 1674064373.8951561,\n",
            "  \"global_step\": 665\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.14404296875,\n",
            "  \"model.lr\": 0.000132539,\n",
            "  \"model.grad_norm\": 2.8939321041107178,\n",
            "  \"model.elapsed_time\": 0.11957836151123047,\n",
            "  \"model.engine_step\": 666,\n",
            "  \"model.loss.nll\": 0.14404296875,\n",
            "  \"elapsed_time\": 0.11957836151123047,\n",
            "  \"wall_time\": 1674064374.0184526,\n",
            "  \"global_step\": 666\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0288543701171875,\n",
            "  \"model.lr\": 0.000132738,\n",
            "  \"model.grad_norm\": 0.5606870651245117,\n",
            "  \"model.elapsed_time\": 0.11268949508666992,\n",
            "  \"model.engine_step\": 667,\n",
            "  \"model.loss.nll\": 0.0288543701171875,\n",
            "  \"elapsed_time\": 0.11268949508666992,\n",
            "  \"wall_time\": 1674064374.141529,\n",
            "  \"global_step\": 667\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.036041259765625,\n",
            "  \"model.lr\": 0.00013293700000000002,\n",
            "  \"model.grad_norm\": 0.5579750537872314,\n",
            "  \"model.elapsed_time\": 0.11490774154663086,\n",
            "  \"model.engine_step\": 668,\n",
            "  \"model.loss.nll\": 0.036041259765625,\n",
            "  \"elapsed_time\": 0.11490774154663086,\n",
            "  \"wall_time\": 1674064374.2597282,\n",
            "  \"global_step\": 668\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03662109375,\n",
            "  \"model.lr\": 0.000133136,\n",
            "  \"model.grad_norm\": 0.6455947756767273,\n",
            "  \"model.elapsed_time\": 0.1101076602935791,\n",
            "  \"model.engine_step\": 669,\n",
            "  \"model.loss.nll\": 0.03662109375,\n",
            "  \"elapsed_time\": 0.1101076602935791,\n",
            "  \"wall_time\": 1674064374.378168,\n",
            "  \"global_step\": 669\n",
            "}\n",
            "[2023-01-18 17:52:54,491] [INFO] [logging.py:68:log_dist] [Rank 0] step=670, skipped=4, lr=[0.00013333500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:54,492] [INFO] [timer.py:196:stop] epoch=0/micro_step=670/global_step=670, RunningAvgSamplesPerSec=12.253433052318353, CurrSamplesPerSec=8.915116978517274, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0340576171875,\n",
            "  \"model.lr\": 0.00013333500000000002,\n",
            "  \"model.grad_norm\": 0.48490044474601746,\n",
            "  \"model.elapsed_time\": 0.11286592483520508,\n",
            "  \"model.engine_step\": 670,\n",
            "  \"model.loss.nll\": 0.0340576171875,\n",
            "  \"elapsed_time\": 0.11286592483520508,\n",
            "  \"wall_time\": 1674064374.4946227,\n",
            "  \"global_step\": 670\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.063232421875,\n",
            "  \"model.lr\": 0.000133534,\n",
            "  \"model.grad_norm\": 1.0209977626800537,\n",
            "  \"model.elapsed_time\": 0.11385107040405273,\n",
            "  \"model.engine_step\": 671,\n",
            "  \"model.loss.nll\": 0.063232421875,\n",
            "  \"elapsed_time\": 0.11385107040405273,\n",
            "  \"wall_time\": 1674064374.6187465,\n",
            "  \"global_step\": 671\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.8779296875,\n",
            "  \"model.lr\": 0.000133733,\n",
            "  \"model.grad_norm\": 10.669273376464844,\n",
            "  \"model.elapsed_time\": 0.11508011817932129,\n",
            "  \"model.engine_step\": 672,\n",
            "  \"model.loss.nll\": 0.8779296875,\n",
            "  \"elapsed_time\": 0.11508011817932129,\n",
            "  \"wall_time\": 1674064374.7369876,\n",
            "  \"global_step\": 672\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0479736328125,\n",
            "  \"model.lr\": 0.00013393200000000002,\n",
            "  \"model.grad_norm\": 0.746871292591095,\n",
            "  \"model.elapsed_time\": 0.11342573165893555,\n",
            "  \"model.engine_step\": 673,\n",
            "  \"model.loss.nll\": 0.0479736328125,\n",
            "  \"elapsed_time\": 0.11342573165893555,\n",
            "  \"wall_time\": 1674064374.8609574,\n",
            "  \"global_step\": 673\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0238037109375,\n",
            "  \"model.lr\": 0.000134131,\n",
            "  \"model.grad_norm\": 0.48043760657310486,\n",
            "  \"model.elapsed_time\": 0.11962342262268066,\n",
            "  \"model.engine_step\": 674,\n",
            "  \"model.loss.nll\": 0.0238037109375,\n",
            "  \"elapsed_time\": 0.11962342262268066,\n",
            "  \"wall_time\": 1674064374.984262,\n",
            "  \"global_step\": 674\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:54\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.027099609375,\n",
            "  \"model.lr\": 0.00013433000000000002,\n",
            "  \"model.grad_norm\": 0.4595232605934143,\n",
            "  \"model.elapsed_time\": 0.11283493041992188,\n",
            "  \"model.engine_step\": 675,\n",
            "  \"model.loss.nll\": 0.027099609375,\n",
            "  \"elapsed_time\": 0.11283493041992188,\n",
            "  \"wall_time\": 1674064375.107099,\n",
            "  \"global_step\": 675\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.259033203125,\n",
            "  \"model.lr\": 0.000134529,\n",
            "  \"model.grad_norm\": 4.378366470336914,\n",
            "  \"model.elapsed_time\": 0.11282682418823242,\n",
            "  \"model.engine_step\": 676,\n",
            "  \"model.loss.nll\": 0.259033203125,\n",
            "  \"elapsed_time\": 0.11282682418823242,\n",
            "  \"wall_time\": 1674064375.2231748,\n",
            "  \"global_step\": 676\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.491943359375,\n",
            "  \"model.lr\": 0.000134728,\n",
            "  \"model.grad_norm\": 5.773825168609619,\n",
            "  \"model.elapsed_time\": 0.12477374076843262,\n",
            "  \"model.engine_step\": 677,\n",
            "  \"model.loss.nll\": 0.491943359375,\n",
            "  \"elapsed_time\": 0.12477374076843262,\n",
            "  \"wall_time\": 1674064375.3572335,\n",
            "  \"global_step\": 677\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05780029296875,\n",
            "  \"model.lr\": 0.00013492700000000002,\n",
            "  \"model.grad_norm\": 1.0345447063446045,\n",
            "  \"model.elapsed_time\": 0.11710929870605469,\n",
            "  \"model.engine_step\": 678,\n",
            "  \"model.loss.nll\": 0.05780029296875,\n",
            "  \"elapsed_time\": 0.11710929870605469,\n",
            "  \"wall_time\": 1674064375.4783752,\n",
            "  \"global_step\": 678\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.024078369140625,\n",
            "  \"model.lr\": 0.000135126,\n",
            "  \"model.grad_norm\": 0.5314279198646545,\n",
            "  \"model.elapsed_time\": 0.12695741653442383,\n",
            "  \"model.engine_step\": 679,\n",
            "  \"model.loss.nll\": 0.024078369140625,\n",
            "  \"elapsed_time\": 0.12695741653442383,\n",
            "  \"wall_time\": 1674064375.6153617,\n",
            "  \"global_step\": 679\n",
            "}\n",
            "[2023-01-18 17:52:55,726] [INFO] [logging.py:68:log_dist] [Rank 0] step=680, skipped=4, lr=[0.00013532500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:55,728] [INFO] [timer.py:196:stop] epoch=0/micro_step=680/global_step=680, RunningAvgSamplesPerSec=12.176419633129997, CurrSamplesPerSec=9.011152480250592, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05413818359375,\n",
            "  \"model.lr\": 0.00013532500000000002,\n",
            "  \"model.grad_norm\": 1.0340583324432373,\n",
            "  \"model.elapsed_time\": 0.11171126365661621,\n",
            "  \"model.engine_step\": 680,\n",
            "  \"model.loss.nll\": 0.05413818359375,\n",
            "  \"elapsed_time\": 0.11171126365661621,\n",
            "  \"wall_time\": 1674064375.7302139,\n",
            "  \"global_step\": 680\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.10052490234375,\n",
            "  \"model.lr\": 0.000135524,\n",
            "  \"model.grad_norm\": 2.2734243869781494,\n",
            "  \"model.elapsed_time\": 0.11413836479187012,\n",
            "  \"model.engine_step\": 681,\n",
            "  \"model.loss.nll\": 0.10052490234375,\n",
            "  \"elapsed_time\": 0.11413836479187012,\n",
            "  \"wall_time\": 1674064375.853545,\n",
            "  \"global_step\": 681\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.057647705078125,\n",
            "  \"model.lr\": 0.000135723,\n",
            "  \"model.grad_norm\": 1.1362894773483276,\n",
            "  \"model.elapsed_time\": 0.11435389518737793,\n",
            "  \"model.engine_step\": 682,\n",
            "  \"model.loss.nll\": 0.057647705078125,\n",
            "  \"elapsed_time\": 0.11435389518737793,\n",
            "  \"wall_time\": 1674064375.9709878,\n",
            "  \"global_step\": 682\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:55\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01800537109375,\n",
            "  \"model.lr\": 0.00013592200000000002,\n",
            "  \"model.grad_norm\": 0.3440203070640564,\n",
            "  \"model.elapsed_time\": 0.11673927307128906,\n",
            "  \"model.engine_step\": 683,\n",
            "  \"model.loss.nll\": 0.01800537109375,\n",
            "  \"elapsed_time\": 0.11673927307128906,\n",
            "  \"wall_time\": 1674064376.09812,\n",
            "  \"global_step\": 683\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0618896484375,\n",
            "  \"model.lr\": 0.000136121,\n",
            "  \"model.grad_norm\": 1.1162879467010498,\n",
            "  \"model.elapsed_time\": 0.1067805290222168,\n",
            "  \"model.engine_step\": 684,\n",
            "  \"model.loss.nll\": 0.0618896484375,\n",
            "  \"elapsed_time\": 0.1067805290222168,\n",
            "  \"wall_time\": 1674064376.2080328,\n",
            "  \"global_step\": 684\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04296875,\n",
            "  \"model.lr\": 0.00013632000000000002,\n",
            "  \"model.grad_norm\": 0.69312584400177,\n",
            "  \"model.elapsed_time\": 0.11638212203979492,\n",
            "  \"model.engine_step\": 685,\n",
            "  \"model.loss.nll\": 0.04296875,\n",
            "  \"elapsed_time\": 0.11638212203979492,\n",
            "  \"wall_time\": 1674064376.3343756,\n",
            "  \"global_step\": 685\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.213134765625,\n",
            "  \"model.lr\": 0.000136519,\n",
            "  \"model.grad_norm\": 3.9220223426818848,\n",
            "  \"model.elapsed_time\": 0.1257476806640625,\n",
            "  \"model.engine_step\": 686,\n",
            "  \"model.loss.nll\": 0.213134765625,\n",
            "  \"elapsed_time\": 0.1257476806640625,\n",
            "  \"wall_time\": 1674064376.4642582,\n",
            "  \"global_step\": 686\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 1.0302734375,\n",
            "  \"model.lr\": 0.00013671800000000003,\n",
            "  \"model.grad_norm\": 10.621912956237793,\n",
            "  \"model.elapsed_time\": 0.11622214317321777,\n",
            "  \"model.engine_step\": 687,\n",
            "  \"model.loss.nll\": 1.0302734375,\n",
            "  \"elapsed_time\": 0.11622214317321777,\n",
            "  \"wall_time\": 1674064376.5919905,\n",
            "  \"global_step\": 687\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04296875,\n",
            "  \"model.lr\": 0.00013691700000000002,\n",
            "  \"model.grad_norm\": 0.7292379140853882,\n",
            "  \"model.elapsed_time\": 0.10260939598083496,\n",
            "  \"model.engine_step\": 688,\n",
            "  \"model.loss.nll\": 0.04296875,\n",
            "  \"elapsed_time\": 0.10260939598083496,\n",
            "  \"wall_time\": 1674064376.6993763,\n",
            "  \"global_step\": 688\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05877685546875,\n",
            "  \"model.lr\": 0.000137116,\n",
            "  \"model.grad_norm\": 1.1574654579162598,\n",
            "  \"model.elapsed_time\": 0.11321043968200684,\n",
            "  \"model.engine_step\": 689,\n",
            "  \"model.loss.nll\": 0.05877685546875,\n",
            "  \"elapsed_time\": 0.11321043968200684,\n",
            "  \"wall_time\": 1674064376.822343,\n",
            "  \"global_step\": 689\n",
            "}\n",
            "[2023-01-18 17:52:56,950] [INFO] [logging.py:68:log_dist] [Rank 0] step=690, skipped=4, lr=[0.00013731500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:56,952] [INFO] [timer.py:196:stop] epoch=0/micro_step=690/global_step=690, RunningAvgSamplesPerSec=12.10542677783199, CurrSamplesPerSec=7.824756077085238, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.014617919921875,\n",
            "  \"model.lr\": 0.00013731500000000002,\n",
            "  \"model.grad_norm\": 0.3471162021160126,\n",
            "  \"model.elapsed_time\": 0.12852787971496582,\n",
            "  \"model.engine_step\": 690,\n",
            "  \"model.loss.nll\": 0.014617919921875,\n",
            "  \"elapsed_time\": 0.12852787971496582,\n",
            "  \"wall_time\": 1674064376.953946,\n",
            "  \"global_step\": 690\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:56\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.314697265625,\n",
            "  \"model.lr\": 0.000137514,\n",
            "  \"model.grad_norm\": 6.114553451538086,\n",
            "  \"model.elapsed_time\": 0.10628294944763184,\n",
            "  \"model.engine_step\": 691,\n",
            "  \"model.loss.nll\": 0.314697265625,\n",
            "  \"elapsed_time\": 0.10628294944763184,\n",
            "  \"wall_time\": 1674064377.071737,\n",
            "  \"global_step\": 691\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03863525390625,\n",
            "  \"model.lr\": 0.00013771300000000003,\n",
            "  \"model.grad_norm\": 0.6861824989318848,\n",
            "  \"model.elapsed_time\": 0.1094048023223877,\n",
            "  \"model.engine_step\": 692,\n",
            "  \"model.loss.nll\": 0.03863525390625,\n",
            "  \"elapsed_time\": 0.1094048023223877,\n",
            "  \"wall_time\": 1674064377.1841595,\n",
            "  \"global_step\": 692\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0625,\n",
            "  \"model.lr\": 0.000137912,\n",
            "  \"model.grad_norm\": 1.1809064149856567,\n",
            "  \"model.elapsed_time\": 0.10338020324707031,\n",
            "  \"model.engine_step\": 693,\n",
            "  \"model.loss.nll\": 0.0625,\n",
            "  \"elapsed_time\": 0.10338020324707031,\n",
            "  \"wall_time\": 1674064377.2971656,\n",
            "  \"global_step\": 693\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.064208984375,\n",
            "  \"model.lr\": 0.000138111,\n",
            "  \"model.grad_norm\": 1.0810827016830444,\n",
            "  \"model.elapsed_time\": 0.09251046180725098,\n",
            "  \"model.engine_step\": 694,\n",
            "  \"model.loss.nll\": 0.064208984375,\n",
            "  \"elapsed_time\": 0.09251046180725098,\n",
            "  \"wall_time\": 1674064377.3926907,\n",
            "  \"global_step\": 694\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.11798095703125,\n",
            "  \"model.lr\": 0.00013831,\n",
            "  \"model.grad_norm\": 2.545905113220215,\n",
            "  \"model.elapsed_time\": 0.11573934555053711,\n",
            "  \"model.engine_step\": 695,\n",
            "  \"model.loss.nll\": 0.11798095703125,\n",
            "  \"elapsed_time\": 0.11573934555053711,\n",
            "  \"wall_time\": 1674064377.5173612,\n",
            "  \"global_step\": 695\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.061492919921875,\n",
            "  \"model.lr\": 0.00013850899999999998,\n",
            "  \"model.grad_norm\": 1.624436616897583,\n",
            "  \"model.elapsed_time\": 0.10390567779541016,\n",
            "  \"model.engine_step\": 696,\n",
            "  \"model.loss.nll\": 0.061492919921875,\n",
            "  \"elapsed_time\": 0.10390567779541016,\n",
            "  \"wall_time\": 1674064377.6249018,\n",
            "  \"global_step\": 696\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0438232421875,\n",
            "  \"model.lr\": 0.000138708,\n",
            "  \"model.grad_norm\": 1.1032193899154663,\n",
            "  \"model.elapsed_time\": 0.11275267601013184,\n",
            "  \"model.engine_step\": 697,\n",
            "  \"model.loss.nll\": 0.0438232421875,\n",
            "  \"elapsed_time\": 0.11275267601013184,\n",
            "  \"wall_time\": 1674064377.7470386,\n",
            "  \"global_step\": 697\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1279296875,\n",
            "  \"model.lr\": 0.000138907,\n",
            "  \"model.grad_norm\": 2.0786383152008057,\n",
            "  \"model.elapsed_time\": 0.09842658042907715,\n",
            "  \"model.engine_step\": 698,\n",
            "  \"model.loss.nll\": 0.1279296875,\n",
            "  \"elapsed_time\": 0.09842658042907715,\n",
            "  \"wall_time\": 1674064377.848785,\n",
            "  \"global_step\": 698\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:57\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 1.1865234375,\n",
            "  \"model.lr\": 0.000139106,\n",
            "  \"model.grad_norm\": 11.361309051513672,\n",
            "  \"model.elapsed_time\": 0.08111166954040527,\n",
            "  \"model.engine_step\": 699,\n",
            "  \"model.loss.nll\": 1.1865234375,\n",
            "  \"elapsed_time\": 0.08111166954040527,\n",
            "  \"wall_time\": 1674064377.938008,\n",
            "  \"global_step\": 699\n",
            "}\n",
            "[2023-01-18 17:52:58,019] [INFO] [logging.py:68:log_dist] [Rank 0] step=700, skipped=4, lr=[0.000139305], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:58,021] [INFO] [timer.py:196:stop] epoch=0/micro_step=700/global_step=700, RunningAvgSamplesPerSec=12.0681589110129, CurrSamplesPerSec=12.227150819748594, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03289794921875,\n",
            "  \"model.lr\": 0.000139305,\n",
            "  \"model.grad_norm\": 0.627068281173706,\n",
            "  \"model.elapsed_time\": 0.08223676681518555,\n",
            "  \"model.engine_step\": 700,\n",
            "  \"model.loss.nll\": 0.03289794921875,\n",
            "  \"elapsed_time\": 0.08223676681518555,\n",
            "  \"wall_time\": 1674064378.023002,\n",
            "  \"global_step\": 700\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.382080078125,\n",
            "  \"model.lr\": 0.00013950399999999998,\n",
            "  \"model.grad_norm\": 4.706935405731201,\n",
            "  \"model.elapsed_time\": 0.07583141326904297,\n",
            "  \"model.engine_step\": 701,\n",
            "  \"model.loss.nll\": 0.382080078125,\n",
            "  \"elapsed_time\": 0.07583141326904297,\n",
            "  \"wall_time\": 1674064378.106637,\n",
            "  \"global_step\": 701\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05157470703125,\n",
            "  \"model.lr\": 0.000139703,\n",
            "  \"model.grad_norm\": 1.0376116037368774,\n",
            "  \"model.elapsed_time\": 0.07083582878112793,\n",
            "  \"model.engine_step\": 702,\n",
            "  \"model.loss.nll\": 0.05157470703125,\n",
            "  \"elapsed_time\": 0.07083582878112793,\n",
            "  \"wall_time\": 1674064378.1800196,\n",
            "  \"global_step\": 702\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01499176025390625,\n",
            "  \"model.lr\": 0.000139902,\n",
            "  \"model.grad_norm\": 0.288755863904953,\n",
            "  \"model.elapsed_time\": 0.07755661010742188,\n",
            "  \"model.engine_step\": 703,\n",
            "  \"model.loss.nll\": 0.01499176025390625,\n",
            "  \"elapsed_time\": 0.07755661010742188,\n",
            "  \"wall_time\": 1674064378.264583,\n",
            "  \"global_step\": 703\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1041259765625,\n",
            "  \"model.lr\": 0.000140101,\n",
            "  \"model.grad_norm\": 2.024627923965454,\n",
            "  \"model.elapsed_time\": 0.07611441612243652,\n",
            "  \"model.engine_step\": 704,\n",
            "  \"model.loss.nll\": 0.1041259765625,\n",
            "  \"elapsed_time\": 0.07611441612243652,\n",
            "  \"wall_time\": 1674064378.3431184,\n",
            "  \"global_step\": 704\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0433349609375,\n",
            "  \"model.lr\": 0.0001403,\n",
            "  \"model.grad_norm\": 0.9229255318641663,\n",
            "  \"model.elapsed_time\": 0.07125020027160645,\n",
            "  \"model.engine_step\": 705,\n",
            "  \"model.loss.nll\": 0.0433349609375,\n",
            "  \"elapsed_time\": 0.07125020027160645,\n",
            "  \"wall_time\": 1674064378.4247,\n",
            "  \"global_step\": 705\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03857421875,\n",
            "  \"model.lr\": 0.00014049899999999998,\n",
            "  \"model.grad_norm\": 1.0672274827957153,\n",
            "  \"model.elapsed_time\": 0.07431292533874512,\n",
            "  \"model.engine_step\": 706,\n",
            "  \"model.loss.nll\": 0.03857421875,\n",
            "  \"elapsed_time\": 0.07431292533874512,\n",
            "  \"wall_time\": 1674064378.5024009,\n",
            "  \"global_step\": 706\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0938720703125,\n",
            "  \"model.lr\": 0.000140698,\n",
            "  \"model.grad_norm\": 1.726409673690796,\n",
            "  \"model.elapsed_time\": 0.07466506958007812,\n",
            "  \"model.engine_step\": 707,\n",
            "  \"model.loss.nll\": 0.0938720703125,\n",
            "  \"elapsed_time\": 0.07466506958007812,\n",
            "  \"wall_time\": 1674064378.586472,\n",
            "  \"global_step\": 707\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.09722900390625,\n",
            "  \"model.lr\": 0.000140897,\n",
            "  \"model.grad_norm\": 2.808590888977051,\n",
            "  \"model.elapsed_time\": 0.07276034355163574,\n",
            "  \"model.engine_step\": 708,\n",
            "  \"model.loss.nll\": 0.09722900390625,\n",
            "  \"elapsed_time\": 0.07276034355163574,\n",
            "  \"wall_time\": 1674064378.6618276,\n",
            "  \"global_step\": 708\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0246124267578125,\n",
            "  \"model.lr\": 0.000141096,\n",
            "  \"model.grad_norm\": 0.49904167652130127,\n",
            "  \"model.elapsed_time\": 0.07420754432678223,\n",
            "  \"model.engine_step\": 709,\n",
            "  \"model.loss.nll\": 0.0246124267578125,\n",
            "  \"elapsed_time\": 0.07420754432678223,\n",
            "  \"wall_time\": 1674064378.7444935,\n",
            "  \"global_step\": 709\n",
            "}\n",
            "[2023-01-18 17:52:58,832] [INFO] [logging.py:68:log_dist] [Rank 0] step=710, skipped=4, lr=[0.000141295], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:58,834] [INFO] [timer.py:196:stop] epoch=0/micro_step=710/global_step=710, RunningAvgSamplesPerSec=12.083208309172381, CurrSamplesPerSec=11.253592624789379, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.029083251953125,\n",
            "  \"model.lr\": 0.000141295,\n",
            "  \"model.grad_norm\": 0.5369359254837036,\n",
            "  \"model.elapsed_time\": 0.08936381340026855,\n",
            "  \"model.engine_step\": 710,\n",
            "  \"model.loss.nll\": 0.029083251953125,\n",
            "  \"elapsed_time\": 0.08936381340026855,\n",
            "  \"wall_time\": 1674064378.836101,\n",
            "  \"global_step\": 710\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:58\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0780029296875,\n",
            "  \"model.lr\": 0.000141494,\n",
            "  \"model.grad_norm\": 1.5230294466018677,\n",
            "  \"model.elapsed_time\": 0.07379484176635742,\n",
            "  \"model.engine_step\": 711,\n",
            "  \"model.loss.nll\": 0.0780029296875,\n",
            "  \"elapsed_time\": 0.07379484176635742,\n",
            "  \"wall_time\": 1674064378.9180708,\n",
            "  \"global_step\": 711\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.5654296875,\n",
            "  \"model.lr\": 0.000141693,\n",
            "  \"model.grad_norm\": 7.864599227905273,\n",
            "  \"model.elapsed_time\": 0.08341860771179199,\n",
            "  \"model.engine_step\": 712,\n",
            "  \"model.loss.nll\": 0.5654296875,\n",
            "  \"elapsed_time\": 0.08341860771179199,\n",
            "  \"wall_time\": 1674064379.0041761,\n",
            "  \"global_step\": 712\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.062255859375,\n",
            "  \"model.lr\": 0.000141892,\n",
            "  \"model.grad_norm\": 1.4590790271759033,\n",
            "  \"model.elapsed_time\": 0.07357954978942871,\n",
            "  \"model.engine_step\": 713,\n",
            "  \"model.loss.nll\": 0.062255859375,\n",
            "  \"elapsed_time\": 0.07357954978942871,\n",
            "  \"wall_time\": 1674064379.0857239,\n",
            "  \"global_step\": 713\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04876708984375,\n",
            "  \"model.lr\": 0.000142091,\n",
            "  \"model.grad_norm\": 1.2258907556533813,\n",
            "  \"model.elapsed_time\": 0.07909250259399414,\n",
            "  \"model.engine_step\": 714,\n",
            "  \"model.loss.nll\": 0.04876708984375,\n",
            "  \"elapsed_time\": 0.07909250259399414,\n",
            "  \"wall_time\": 1674064379.1673753,\n",
            "  \"global_step\": 714\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.055999755859375,\n",
            "  \"model.lr\": 0.00014229,\n",
            "  \"model.grad_norm\": 1.3118464946746826,\n",
            "  \"model.elapsed_time\": 0.08111429214477539,\n",
            "  \"model.engine_step\": 715,\n",
            "  \"model.loss.nll\": 0.055999755859375,\n",
            "  \"elapsed_time\": 0.08111429214477539,\n",
            "  \"wall_time\": 1674064379.2558513,\n",
            "  \"global_step\": 715\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.039154052734375,\n",
            "  \"model.lr\": 0.000142489,\n",
            "  \"model.grad_norm\": 0.724927544593811,\n",
            "  \"model.elapsed_time\": 0.07141494750976562,\n",
            "  \"model.engine_step\": 716,\n",
            "  \"model.loss.nll\": 0.039154052734375,\n",
            "  \"elapsed_time\": 0.07141494750976562,\n",
            "  \"wall_time\": 1674064379.3301682,\n",
            "  \"global_step\": 716\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1925048828125,\n",
            "  \"model.lr\": 0.000142688,\n",
            "  \"model.grad_norm\": 3.228087902069092,\n",
            "  \"model.elapsed_time\": 0.07045793533325195,\n",
            "  \"model.engine_step\": 717,\n",
            "  \"model.loss.nll\": 0.1925048828125,\n",
            "  \"elapsed_time\": 0.07045793533325195,\n",
            "  \"wall_time\": 1674064379.409702,\n",
            "  \"global_step\": 717\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0540771484375,\n",
            "  \"model.lr\": 0.000142887,\n",
            "  \"model.grad_norm\": 1.7343578338623047,\n",
            "  \"model.elapsed_time\": 0.082489013671875,\n",
            "  \"model.engine_step\": 718,\n",
            "  \"model.loss.nll\": 0.0540771484375,\n",
            "  \"elapsed_time\": 0.082489013671875,\n",
            "  \"wall_time\": 1674064379.495137,\n",
            "  \"global_step\": 718\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.044219970703125,\n",
            "  \"model.lr\": 0.000143086,\n",
            "  \"model.grad_norm\": 0.7741522789001465,\n",
            "  \"model.elapsed_time\": 0.08192849159240723,\n",
            "  \"model.engine_step\": 719,\n",
            "  \"model.loss.nll\": 0.044219970703125,\n",
            "  \"elapsed_time\": 0.08192849159240723,\n",
            "  \"wall_time\": 1674064379.5849106,\n",
            "  \"global_step\": 719\n",
            "}\n",
            "[2023-01-18 17:52:59,665] [INFO] [logging.py:68:log_dist] [Rank 0] step=720, skipped=4, lr=[0.000143285], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:52:59,666] [INFO] [timer.py:196:stop] epoch=0/micro_step=720/global_step=720, RunningAvgSamplesPerSec=12.0935861512153, CurrSamplesPerSec=12.463462178481434, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.046112060546875,\n",
            "  \"model.lr\": 0.000143285,\n",
            "  \"model.grad_norm\": 0.7713238000869751,\n",
            "  \"model.elapsed_time\": 0.08073639869689941,\n",
            "  \"model.engine_step\": 720,\n",
            "  \"model.loss.nll\": 0.046112060546875,\n",
            "  \"elapsed_time\": 0.08073639869689941,\n",
            "  \"wall_time\": 1674064379.6680133,\n",
            "  \"global_step\": 720\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022796630859375,\n",
            "  \"model.lr\": 0.000143484,\n",
            "  \"model.grad_norm\": 0.4596768319606781,\n",
            "  \"model.elapsed_time\": 0.08136987686157227,\n",
            "  \"model.engine_step\": 721,\n",
            "  \"model.loss.nll\": 0.022796630859375,\n",
            "  \"elapsed_time\": 0.08136987686157227,\n",
            "  \"wall_time\": 1674064379.7576928,\n",
            "  \"global_step\": 721\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.055572509765625,\n",
            "  \"model.lr\": 0.000143683,\n",
            "  \"model.grad_norm\": 1.271830439567566,\n",
            "  \"model.elapsed_time\": 0.07014894485473633,\n",
            "  \"model.engine_step\": 722,\n",
            "  \"model.loss.nll\": 0.055572509765625,\n",
            "  \"elapsed_time\": 0.07014894485473633,\n",
            "  \"wall_time\": 1674064379.83066,\n",
            "  \"global_step\": 722\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.028045654296875,\n",
            "  \"model.lr\": 0.000143882,\n",
            "  \"model.grad_norm\": 0.5721228122711182,\n",
            "  \"model.elapsed_time\": 0.0740044116973877,\n",
            "  \"model.engine_step\": 723,\n",
            "  \"model.loss.nll\": 0.028045654296875,\n",
            "  \"elapsed_time\": 0.0740044116973877,\n",
            "  \"wall_time\": 1674064379.9122524,\n",
            "  \"global_step\": 723\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.035064697265625,\n",
            "  \"model.lr\": 0.000144081,\n",
            "  \"model.grad_norm\": 1.2769612073898315,\n",
            "  \"model.elapsed_time\": 0.08057022094726562,\n",
            "  \"model.engine_step\": 724,\n",
            "  \"model.loss.nll\": 0.035064697265625,\n",
            "  \"elapsed_time\": 0.08057022094726562,\n",
            "  \"wall_time\": 1674064379.9948318,\n",
            "  \"global_step\": 724\n",
            "}\n",
            "\u001b[32m2023-01-18 17:52:59\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0302581787109375,\n",
            "  \"model.lr\": 0.00014428,\n",
            "  \"model.grad_norm\": 0.6177506446838379,\n",
            "  \"model.elapsed_time\": 0.08031296730041504,\n",
            "  \"model.engine_step\": 725,\n",
            "  \"model.loss.nll\": 0.0302581787109375,\n",
            "  \"elapsed_time\": 0.08031296730041504,\n",
            "  \"wall_time\": 1674064380.0817635,\n",
            "  \"global_step\": 725\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06732177734375,\n",
            "  \"model.lr\": 0.000144479,\n",
            "  \"model.grad_norm\": 1.5162644386291504,\n",
            "  \"model.elapsed_time\": 0.07534313201904297,\n",
            "  \"model.engine_step\": 726,\n",
            "  \"model.loss.nll\": 0.06732177734375,\n",
            "  \"elapsed_time\": 0.07534313201904297,\n",
            "  \"wall_time\": 1674064380.1598415,\n",
            "  \"global_step\": 726\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0210723876953125,\n",
            "  \"model.lr\": 0.000144678,\n",
            "  \"model.grad_norm\": 0.41649046540260315,\n",
            "  \"model.elapsed_time\": 0.07425546646118164,\n",
            "  \"model.engine_step\": 727,\n",
            "  \"model.loss.nll\": 0.0210723876953125,\n",
            "  \"elapsed_time\": 0.07425546646118164,\n",
            "  \"wall_time\": 1674064380.2409494,\n",
            "  \"global_step\": 727\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.222900390625,\n",
            "  \"model.lr\": 0.000144877,\n",
            "  \"model.grad_norm\": 4.824894905090332,\n",
            "  \"model.elapsed_time\": 0.07241153717041016,\n",
            "  \"model.engine_step\": 728,\n",
            "  \"model.loss.nll\": 0.222900390625,\n",
            "  \"elapsed_time\": 0.07241153717041016,\n",
            "  \"wall_time\": 1674064380.3165486,\n",
            "  \"global_step\": 728\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.039398193359375,\n",
            "  \"model.lr\": 0.000145076,\n",
            "  \"model.grad_norm\": 0.634131908416748,\n",
            "  \"model.elapsed_time\": 0.07146573066711426,\n",
            "  \"model.engine_step\": 729,\n",
            "  \"model.loss.nll\": 0.039398193359375,\n",
            "  \"elapsed_time\": 0.07146573066711426,\n",
            "  \"wall_time\": 1674064380.397547,\n",
            "  \"global_step\": 729\n",
            "}\n",
            "[2023-01-18 17:53:00,471] [INFO] [logging.py:68:log_dist] [Rank 0] step=730, skipped=4, lr=[0.000145275], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:00,473] [INFO] [timer.py:196:stop] epoch=0/micro_step=730/global_step=730, RunningAvgSamplesPerSec=12.108655772249765, CurrSamplesPerSec=13.57986414644728, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0250244140625,\n",
            "  \"model.lr\": 0.000145275,\n",
            "  \"model.grad_norm\": 0.47213950753211975,\n",
            "  \"model.elapsed_time\": 0.0740804672241211,\n",
            "  \"model.engine_step\": 730,\n",
            "  \"model.loss.nll\": 0.0250244140625,\n",
            "  \"elapsed_time\": 0.0740804672241211,\n",
            "  \"wall_time\": 1674064380.4741423,\n",
            "  \"global_step\": 730\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.068115234375,\n",
            "  \"model.lr\": 0.000145474,\n",
            "  \"model.grad_norm\": 1.8285444974899292,\n",
            "  \"model.elapsed_time\": 0.07164120674133301,\n",
            "  \"model.engine_step\": 731,\n",
            "  \"model.loss.nll\": 0.068115234375,\n",
            "  \"elapsed_time\": 0.07164120674133301,\n",
            "  \"wall_time\": 1674064380.5519593,\n",
            "  \"global_step\": 731\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03155517578125,\n",
            "  \"model.lr\": 0.000145673,\n",
            "  \"model.grad_norm\": 0.5358395576477051,\n",
            "  \"model.elapsed_time\": 0.07505106925964355,\n",
            "  \"model.engine_step\": 732,\n",
            "  \"model.loss.nll\": 0.03155517578125,\n",
            "  \"elapsed_time\": 0.07505106925964355,\n",
            "  \"wall_time\": 1674064380.6298575,\n",
            "  \"global_step\": 732\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.016448974609375,\n",
            "  \"model.lr\": 0.00014587200000000002,\n",
            "  \"model.grad_norm\": 0.3226730525493622,\n",
            "  \"model.elapsed_time\": 0.07281994819641113,\n",
            "  \"model.engine_step\": 733,\n",
            "  \"model.loss.nll\": 0.016448974609375,\n",
            "  \"elapsed_time\": 0.07281994819641113,\n",
            "  \"wall_time\": 1674064380.710771,\n",
            "  \"global_step\": 733\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.072265625,\n",
            "  \"model.lr\": 0.000146071,\n",
            "  \"model.grad_norm\": 1.349251627922058,\n",
            "  \"model.elapsed_time\": 0.07221055030822754,\n",
            "  \"model.engine_step\": 734,\n",
            "  \"model.loss.nll\": 0.072265625,\n",
            "  \"elapsed_time\": 0.07221055030822754,\n",
            "  \"wall_time\": 1674064380.784958,\n",
            "  \"global_step\": 734\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1275634765625,\n",
            "  \"model.lr\": 0.00014627,\n",
            "  \"model.grad_norm\": 2.54626727104187,\n",
            "  \"model.elapsed_time\": 0.08482027053833008,\n",
            "  \"model.engine_step\": 735,\n",
            "  \"model.loss.nll\": 0.1275634765625,\n",
            "  \"elapsed_time\": 0.08482027053833008,\n",
            "  \"wall_time\": 1674064380.878225,\n",
            "  \"global_step\": 735\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.08636474609375,\n",
            "  \"model.lr\": 0.000146469,\n",
            "  \"model.grad_norm\": 1.765177607536316,\n",
            "  \"model.elapsed_time\": 0.07121133804321289,\n",
            "  \"model.engine_step\": 736,\n",
            "  \"model.loss.nll\": 0.08636474609375,\n",
            "  \"elapsed_time\": 0.07121133804321289,\n",
            "  \"wall_time\": 1674064380.951926,\n",
            "  \"global_step\": 736\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:00\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0660400390625,\n",
            "  \"model.lr\": 0.000146668,\n",
            "  \"model.grad_norm\": 1.1858711242675781,\n",
            "  \"model.elapsed_time\": 0.08453130722045898,\n",
            "  \"model.engine_step\": 737,\n",
            "  \"model.loss.nll\": 0.0660400390625,\n",
            "  \"elapsed_time\": 0.08453130722045898,\n",
            "  \"wall_time\": 1674064381.044107,\n",
            "  \"global_step\": 737\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.100830078125,\n",
            "  \"model.lr\": 0.00014686700000000002,\n",
            "  \"model.grad_norm\": 2.187021255493164,\n",
            "  \"model.elapsed_time\": 0.06897854804992676,\n",
            "  \"model.engine_step\": 738,\n",
            "  \"model.loss.nll\": 0.100830078125,\n",
            "  \"elapsed_time\": 0.06897854804992676,\n",
            "  \"wall_time\": 1674064381.1155262,\n",
            "  \"global_step\": 738\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021575927734375,\n",
            "  \"model.lr\": 0.000147066,\n",
            "  \"model.grad_norm\": 0.4676228165626526,\n",
            "  \"model.elapsed_time\": 0.07744646072387695,\n",
            "  \"model.engine_step\": 739,\n",
            "  \"model.loss.nll\": 0.021575927734375,\n",
            "  \"elapsed_time\": 0.07744646072387695,\n",
            "  \"wall_time\": 1674064381.2023787,\n",
            "  \"global_step\": 739\n",
            "}\n",
            "[2023-01-18 17:53:01,273] [INFO] [logging.py:68:log_dist] [Rank 0] step=740, skipped=4, lr=[0.000147265], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:01,274] [INFO] [timer.py:196:stop] epoch=0/micro_step=740/global_step=740, RunningAvgSamplesPerSec=12.123976465451545, CurrSamplesPerSec=14.077160338444912, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0212860107421875,\n",
            "  \"model.lr\": 0.000147265,\n",
            "  \"model.grad_norm\": 0.38854628801345825,\n",
            "  \"model.elapsed_time\": 0.07172083854675293,\n",
            "  \"model.engine_step\": 740,\n",
            "  \"model.loss.nll\": 0.0212860107421875,\n",
            "  \"elapsed_time\": 0.07172083854675293,\n",
            "  \"wall_time\": 1674064381.276945,\n",
            "  \"global_step\": 740\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04046630859375,\n",
            "  \"model.lr\": 0.000147464,\n",
            "  \"model.grad_norm\": 0.6756055951118469,\n",
            "  \"model.elapsed_time\": 0.07134437561035156,\n",
            "  \"model.engine_step\": 741,\n",
            "  \"model.loss.nll\": 0.04046630859375,\n",
            "  \"elapsed_time\": 0.07134437561035156,\n",
            "  \"wall_time\": 1674064381.3578966,\n",
            "  \"global_step\": 741\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.373291015625,\n",
            "  \"model.lr\": 0.000147663,\n",
            "  \"model.grad_norm\": 6.180834770202637,\n",
            "  \"model.elapsed_time\": 0.07723331451416016,\n",
            "  \"model.engine_step\": 742,\n",
            "  \"model.loss.nll\": 0.373291015625,\n",
            "  \"elapsed_time\": 0.07723331451416016,\n",
            "  \"wall_time\": 1674064381.4373736,\n",
            "  \"global_step\": 742\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021514892578125,\n",
            "  \"model.lr\": 0.00014786200000000002,\n",
            "  \"model.grad_norm\": 0.7171406149864197,\n",
            "  \"model.elapsed_time\": 0.07860684394836426,\n",
            "  \"model.engine_step\": 743,\n",
            "  \"model.loss.nll\": 0.021514892578125,\n",
            "  \"elapsed_time\": 0.07860684394836426,\n",
            "  \"wall_time\": 1674064381.523854,\n",
            "  \"global_step\": 743\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.034820556640625,\n",
            "  \"model.lr\": 0.000148061,\n",
            "  \"model.grad_norm\": 0.5720672607421875,\n",
            "  \"model.elapsed_time\": 0.07094120979309082,\n",
            "  \"model.engine_step\": 744,\n",
            "  \"model.loss.nll\": 0.034820556640625,\n",
            "  \"elapsed_time\": 0.07094120979309082,\n",
            "  \"wall_time\": 1674064381.5968702,\n",
            "  \"global_step\": 744\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07110595703125,\n",
            "  \"model.lr\": 0.00014826,\n",
            "  \"model.grad_norm\": 1.9352613687515259,\n",
            "  \"model.elapsed_time\": 0.08205175399780273,\n",
            "  \"model.engine_step\": 745,\n",
            "  \"model.loss.nll\": 0.07110595703125,\n",
            "  \"elapsed_time\": 0.08205175399780273,\n",
            "  \"wall_time\": 1674064381.6872947,\n",
            "  \"global_step\": 745\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03570556640625,\n",
            "  \"model.lr\": 0.000148459,\n",
            "  \"model.grad_norm\": 0.6581553220748901,\n",
            "  \"model.elapsed_time\": 0.07981586456298828,\n",
            "  \"model.engine_step\": 746,\n",
            "  \"model.loss.nll\": 0.03570556640625,\n",
            "  \"elapsed_time\": 0.07981586456298828,\n",
            "  \"wall_time\": 1674064381.7693572,\n",
            "  \"global_step\": 746\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.08154296875,\n",
            "  \"model.lr\": 0.000148658,\n",
            "  \"model.grad_norm\": 2.061283826828003,\n",
            "  \"model.elapsed_time\": 0.0720512866973877,\n",
            "  \"model.engine_step\": 747,\n",
            "  \"model.loss.nll\": 0.08154296875,\n",
            "  \"elapsed_time\": 0.0720512866973877,\n",
            "  \"wall_time\": 1674064381.8480635,\n",
            "  \"global_step\": 747\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.12115478515625,\n",
            "  \"model.lr\": 0.00014885700000000001,\n",
            "  \"model.grad_norm\": 3.0535364151000977,\n",
            "  \"model.elapsed_time\": 0.0765070915222168,\n",
            "  \"model.engine_step\": 748,\n",
            "  \"model.loss.nll\": 0.12115478515625,\n",
            "  \"elapsed_time\": 0.0765070915222168,\n",
            "  \"wall_time\": 1674064381.9275575,\n",
            "  \"global_step\": 748\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:01\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.45263671875,\n",
            "  \"model.lr\": 0.000149056,\n",
            "  \"model.grad_norm\": 7.017239570617676,\n",
            "  \"model.elapsed_time\": 0.08639860153198242,\n",
            "  \"model.engine_step\": 749,\n",
            "  \"model.loss.nll\": 0.45263671875,\n",
            "  \"elapsed_time\": 0.08639860153198242,\n",
            "  \"wall_time\": 1674064382.0257385,\n",
            "  \"global_step\": 749\n",
            "}\n",
            "[2023-01-18 17:53:02,098] [INFO] [logging.py:68:log_dist] [Rank 0] step=750, skipped=4, lr=[0.000149255], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:02,101] [INFO] [timer.py:196:stop] epoch=0/micro_step=750/global_step=750, RunningAvgSamplesPerSec=12.135278282035953, CurrSamplesPerSec=13.648757089908006, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03338623046875,\n",
            "  \"model.lr\": 0.000149255,\n",
            "  \"model.grad_norm\": 0.6023712754249573,\n",
            "  \"model.elapsed_time\": 0.07394790649414062,\n",
            "  \"model.engine_step\": 750,\n",
            "  \"model.loss.nll\": 0.03338623046875,\n",
            "  \"elapsed_time\": 0.07394790649414062,\n",
            "  \"wall_time\": 1674064382.102692,\n",
            "  \"global_step\": 750\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0316162109375,\n",
            "  \"model.lr\": 0.000149454,\n",
            "  \"model.grad_norm\": 0.8212977647781372,\n",
            "  \"model.elapsed_time\": 0.07637286186218262,\n",
            "  \"model.engine_step\": 751,\n",
            "  \"model.loss.nll\": 0.0316162109375,\n",
            "  \"elapsed_time\": 0.07637286186218262,\n",
            "  \"wall_time\": 1674064382.1906173,\n",
            "  \"global_step\": 751\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0296783447265625,\n",
            "  \"model.lr\": 0.000149653,\n",
            "  \"model.grad_norm\": 0.5987740159034729,\n",
            "  \"model.elapsed_time\": 0.08091044425964355,\n",
            "  \"model.engine_step\": 752,\n",
            "  \"model.loss.nll\": 0.0296783447265625,\n",
            "  \"elapsed_time\": 0.08091044425964355,\n",
            "  \"wall_time\": 1674064382.273832,\n",
            "  \"global_step\": 752\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.075439453125,\n",
            "  \"model.lr\": 0.00014985200000000001,\n",
            "  \"model.grad_norm\": 1.8466088771820068,\n",
            "  \"model.elapsed_time\": 0.07278323173522949,\n",
            "  \"model.engine_step\": 753,\n",
            "  \"model.loss.nll\": 0.075439453125,\n",
            "  \"elapsed_time\": 0.07278323173522949,\n",
            "  \"wall_time\": 1674064382.3547132,\n",
            "  \"global_step\": 753\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.043701171875,\n",
            "  \"model.lr\": 0.000150051,\n",
            "  \"model.grad_norm\": 0.9365788698196411,\n",
            "  \"model.elapsed_time\": 0.07365894317626953,\n",
            "  \"model.engine_step\": 754,\n",
            "  \"model.loss.nll\": 0.043701171875,\n",
            "  \"elapsed_time\": 0.07365894317626953,\n",
            "  \"wall_time\": 1674064382.4304812,\n",
            "  \"global_step\": 754\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02740478515625,\n",
            "  \"model.lr\": 0.00015025000000000002,\n",
            "  \"model.grad_norm\": 0.6024587750434875,\n",
            "  \"model.elapsed_time\": 0.07272577285766602,\n",
            "  \"model.engine_step\": 755,\n",
            "  \"model.loss.nll\": 0.02740478515625,\n",
            "  \"elapsed_time\": 0.07272577285766602,\n",
            "  \"wall_time\": 1674064382.5112538,\n",
            "  \"global_step\": 755\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03558349609375,\n",
            "  \"model.lr\": 0.000150449,\n",
            "  \"model.grad_norm\": 0.8244337439537048,\n",
            "  \"model.elapsed_time\": 0.0812990665435791,\n",
            "  \"model.engine_step\": 756,\n",
            "  \"model.loss.nll\": 0.03558349609375,\n",
            "  \"elapsed_time\": 0.0812990665435791,\n",
            "  \"wall_time\": 1674064382.5945745,\n",
            "  \"global_step\": 756\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02618408203125,\n",
            "  \"model.lr\": 0.000150648,\n",
            "  \"model.grad_norm\": 0.5709249973297119,\n",
            "  \"model.elapsed_time\": 0.07847046852111816,\n",
            "  \"model.engine_step\": 757,\n",
            "  \"model.loss.nll\": 0.02618408203125,\n",
            "  \"elapsed_time\": 0.07847046852111816,\n",
            "  \"wall_time\": 1674064382.6822524,\n",
            "  \"global_step\": 757\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0235137939453125,\n",
            "  \"model.lr\": 0.00015084700000000001,\n",
            "  \"model.grad_norm\": 0.4251766502857208,\n",
            "  \"model.elapsed_time\": 0.07485389709472656,\n",
            "  \"model.engine_step\": 758,\n",
            "  \"model.loss.nll\": 0.0235137939453125,\n",
            "  \"elapsed_time\": 0.07485389709472656,\n",
            "  \"wall_time\": 1674064382.7591937,\n",
            "  \"global_step\": 758\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1561279296875,\n",
            "  \"model.lr\": 0.000151046,\n",
            "  \"model.grad_norm\": 4.484102249145508,\n",
            "  \"model.elapsed_time\": 0.07070183753967285,\n",
            "  \"model.engine_step\": 759,\n",
            "  \"model.loss.nll\": 0.1561279296875,\n",
            "  \"elapsed_time\": 0.07070183753967285,\n",
            "  \"wall_time\": 1674064382.840089,\n",
            "  \"global_step\": 759\n",
            "}\n",
            "[2023-01-18 17:53:02,914] [INFO] [logging.py:68:log_dist] [Rank 0] step=760, skipped=4, lr=[0.00015124500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:02,917] [INFO] [timer.py:196:stop] epoch=0/micro_step=760/global_step=760, RunningAvgSamplesPerSec=12.148885624287463, CurrSamplesPerSec=13.664498011070243, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0227508544921875,\n",
            "  \"model.lr\": 0.00015124500000000002,\n",
            "  \"model.grad_norm\": 0.43251460790634155,\n",
            "  \"model.elapsed_time\": 0.07371115684509277,\n",
            "  \"model.engine_step\": 760,\n",
            "  \"model.loss.nll\": 0.0227508544921875,\n",
            "  \"elapsed_time\": 0.07371115684509277,\n",
            "  \"wall_time\": 1674064382.9181993,\n",
            "  \"global_step\": 760\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:02\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021026611328125,\n",
            "  \"model.lr\": 0.000151444,\n",
            "  \"model.grad_norm\": 0.8948438167572021,\n",
            "  \"model.elapsed_time\": 0.07723379135131836,\n",
            "  \"model.engine_step\": 761,\n",
            "  \"model.loss.nll\": 0.021026611328125,\n",
            "  \"elapsed_time\": 0.07723379135131836,\n",
            "  \"wall_time\": 1674064383.002561,\n",
            "  \"global_step\": 761\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.279052734375,\n",
            "  \"model.lr\": 0.000151643,\n",
            "  \"model.grad_norm\": 3.7987444400787354,\n",
            "  \"model.elapsed_time\": 0.07831144332885742,\n",
            "  \"model.engine_step\": 762,\n",
            "  \"model.loss.nll\": 0.279052734375,\n",
            "  \"elapsed_time\": 0.07831144332885742,\n",
            "  \"wall_time\": 1674064383.083565,\n",
            "  \"global_step\": 762\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.034088134765625,\n",
            "  \"model.lr\": 0.00015184200000000001,\n",
            "  \"model.grad_norm\": 0.9789897799491882,\n",
            "  \"model.elapsed_time\": 0.08028936386108398,\n",
            "  \"model.engine_step\": 763,\n",
            "  \"model.loss.nll\": 0.034088134765625,\n",
            "  \"elapsed_time\": 0.08028936386108398,\n",
            "  \"wall_time\": 1674064383.171663,\n",
            "  \"global_step\": 763\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.09906005859375,\n",
            "  \"model.lr\": 0.000152041,\n",
            "  \"model.grad_norm\": 2.571197032928467,\n",
            "  \"model.elapsed_time\": 0.07726645469665527,\n",
            "  \"model.engine_step\": 764,\n",
            "  \"model.loss.nll\": 0.09906005859375,\n",
            "  \"elapsed_time\": 0.07726645469665527,\n",
            "  \"wall_time\": 1674064383.2514586,\n",
            "  \"global_step\": 764\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06536865234375,\n",
            "  \"model.lr\": 0.00015224000000000002,\n",
            "  \"model.grad_norm\": 1.4202816486358643,\n",
            "  \"model.elapsed_time\": 0.07193708419799805,\n",
            "  \"model.engine_step\": 765,\n",
            "  \"model.loss.nll\": 0.06536865234375,\n",
            "  \"elapsed_time\": 0.07193708419799805,\n",
            "  \"wall_time\": 1674064383.331619,\n",
            "  \"global_step\": 765\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04510498046875,\n",
            "  \"model.lr\": 0.000152439,\n",
            "  \"model.grad_norm\": 1.5455797910690308,\n",
            "  \"model.elapsed_time\": 0.07698893547058105,\n",
            "  \"model.engine_step\": 766,\n",
            "  \"model.loss.nll\": 0.04510498046875,\n",
            "  \"elapsed_time\": 0.07698893547058105,\n",
            "  \"wall_time\": 1674064383.4113402,\n",
            "  \"global_step\": 766\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0220947265625,\n",
            "  \"model.lr\": 0.000152638,\n",
            "  \"model.grad_norm\": 0.4119795262813568,\n",
            "  \"model.elapsed_time\": 0.07504105567932129,\n",
            "  \"model.engine_step\": 767,\n",
            "  \"model.loss.nll\": 0.0220947265625,\n",
            "  \"elapsed_time\": 0.07504105567932129,\n",
            "  \"wall_time\": 1674064383.4960666,\n",
            "  \"global_step\": 767\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0168914794921875,\n",
            "  \"model.lr\": 0.00015283700000000001,\n",
            "  \"model.grad_norm\": 0.5538188219070435,\n",
            "  \"model.elapsed_time\": 0.07824468612670898,\n",
            "  \"model.engine_step\": 768,\n",
            "  \"model.loss.nll\": 0.0168914794921875,\n",
            "  \"elapsed_time\": 0.07824468612670898,\n",
            "  \"wall_time\": 1674064383.5764096,\n",
            "  \"global_step\": 768\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2607421875,\n",
            "  \"model.lr\": 0.000153036,\n",
            "  \"model.grad_norm\": 4.470075607299805,\n",
            "  \"model.elapsed_time\": 0.07384204864501953,\n",
            "  \"model.engine_step\": 769,\n",
            "  \"model.loss.nll\": 0.2607421875,\n",
            "  \"elapsed_time\": 0.07384204864501953,\n",
            "  \"wall_time\": 1674064383.6568973,\n",
            "  \"global_step\": 769\n",
            "}\n",
            "[2023-01-18 17:53:03,734] [INFO] [logging.py:68:log_dist] [Rank 0] step=770, skipped=4, lr=[0.00015323500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:03,736] [INFO] [timer.py:196:stop] epoch=0/micro_step=770/global_step=770, RunningAvgSamplesPerSec=12.159934244765772, CurrSamplesPerSec=12.921454097350585, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.087646484375,\n",
            "  \"model.lr\": 0.00015323500000000002,\n",
            "  \"model.grad_norm\": 2.168785572052002,\n",
            "  \"model.elapsed_time\": 0.0778813362121582,\n",
            "  \"model.engine_step\": 770,\n",
            "  \"model.loss.nll\": 0.087646484375,\n",
            "  \"elapsed_time\": 0.0778813362121582,\n",
            "  \"wall_time\": 1674064383.7374845,\n",
            "  \"global_step\": 770\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.033599853515625,\n",
            "  \"model.lr\": 0.000153434,\n",
            "  \"model.grad_norm\": 0.6476193070411682,\n",
            "  \"model.elapsed_time\": 0.07293248176574707,\n",
            "  \"model.engine_step\": 771,\n",
            "  \"model.loss.nll\": 0.033599853515625,\n",
            "  \"elapsed_time\": 0.07293248176574707,\n",
            "  \"wall_time\": 1674064383.819472,\n",
            "  \"global_step\": 771\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03009033203125,\n",
            "  \"model.lr\": 0.000153633,\n",
            "  \"model.grad_norm\": 0.5546146035194397,\n",
            "  \"model.elapsed_time\": 0.08116340637207031,\n",
            "  \"model.engine_step\": 772,\n",
            "  \"model.loss.nll\": 0.03009033203125,\n",
            "  \"elapsed_time\": 0.08116340637207031,\n",
            "  \"wall_time\": 1674064383.9025805,\n",
            "  \"global_step\": 772\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:03\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022613525390625,\n",
            "  \"model.lr\": 0.00015383200000000001,\n",
            "  \"model.grad_norm\": 0.41112470626831055,\n",
            "  \"model.elapsed_time\": 0.07431197166442871,\n",
            "  \"model.engine_step\": 773,\n",
            "  \"model.loss.nll\": 0.022613525390625,\n",
            "  \"elapsed_time\": 0.07431197166442871,\n",
            "  \"wall_time\": 1674064383.9837642,\n",
            "  \"global_step\": 773\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1435546875,\n",
            "  \"model.lr\": 0.000154031,\n",
            "  \"model.grad_norm\": 3.2398085594177246,\n",
            "  \"model.elapsed_time\": 0.0836799144744873,\n",
            "  \"model.engine_step\": 774,\n",
            "  \"model.loss.nll\": 0.1435546875,\n",
            "  \"elapsed_time\": 0.0836799144744873,\n",
            "  \"wall_time\": 1674064384.0700445,\n",
            "  \"global_step\": 774\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04974365234375,\n",
            "  \"model.lr\": 0.00015423000000000002,\n",
            "  \"model.grad_norm\": 1.47084379196167,\n",
            "  \"model.elapsed_time\": 0.0738370418548584,\n",
            "  \"model.engine_step\": 775,\n",
            "  \"model.loss.nll\": 0.04974365234375,\n",
            "  \"elapsed_time\": 0.0738370418548584,\n",
            "  \"wall_time\": 1674064384.1523561,\n",
            "  \"global_step\": 775\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.043121337890625,\n",
            "  \"model.lr\": 0.000154429,\n",
            "  \"model.grad_norm\": 1.3803883790969849,\n",
            "  \"model.elapsed_time\": 0.07427692413330078,\n",
            "  \"model.engine_step\": 776,\n",
            "  \"model.loss.nll\": 0.043121337890625,\n",
            "  \"elapsed_time\": 0.07427692413330078,\n",
            "  \"wall_time\": 1674064384.2296085,\n",
            "  \"global_step\": 776\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.052276611328125,\n",
            "  \"model.lr\": 0.000154628,\n",
            "  \"model.grad_norm\": 1.3860657215118408,\n",
            "  \"model.elapsed_time\": 0.07286930084228516,\n",
            "  \"model.engine_step\": 777,\n",
            "  \"model.loss.nll\": 0.052276611328125,\n",
            "  \"elapsed_time\": 0.07286930084228516,\n",
            "  \"wall_time\": 1674064384.3105288,\n",
            "  \"global_step\": 777\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04998779296875,\n",
            "  \"model.lr\": 0.00015482700000000001,\n",
            "  \"model.grad_norm\": 1.7823896408081055,\n",
            "  \"model.elapsed_time\": 0.07680273056030273,\n",
            "  \"model.engine_step\": 778,\n",
            "  \"model.loss.nll\": 0.04998779296875,\n",
            "  \"elapsed_time\": 0.07680273056030273,\n",
            "  \"wall_time\": 1674064384.389418,\n",
            "  \"global_step\": 778\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.025421142578125,\n",
            "  \"model.lr\": 0.000155026,\n",
            "  \"model.grad_norm\": 0.46120721101760864,\n",
            "  \"model.elapsed_time\": 0.07040834426879883,\n",
            "  \"model.engine_step\": 779,\n",
            "  \"model.loss.nll\": 0.025421142578125,\n",
            "  \"elapsed_time\": 0.07040834426879883,\n",
            "  \"wall_time\": 1674064384.4674969,\n",
            "  \"global_step\": 779\n",
            "}\n",
            "[2023-01-18 17:53:04,546] [INFO] [logging.py:68:log_dist] [Rank 0] step=780, skipped=4, lr=[0.00015522500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:04,548] [INFO] [timer.py:196:stop] epoch=0/micro_step=780/global_step=780, RunningAvgSamplesPerSec=12.1719489239624, CurrSamplesPerSec=12.542700103468281, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0283050537109375,\n",
            "  \"model.lr\": 0.00015522500000000002,\n",
            "  \"model.grad_norm\": 0.5661920309066772,\n",
            "  \"model.elapsed_time\": 0.08016586303710938,\n",
            "  \"model.engine_step\": 780,\n",
            "  \"model.loss.nll\": 0.0283050537109375,\n",
            "  \"elapsed_time\": 0.08016586303710938,\n",
            "  \"wall_time\": 1674064384.5496664,\n",
            "  \"global_step\": 780\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02313232421875,\n",
            "  \"model.lr\": 0.000155424,\n",
            "  \"model.grad_norm\": 0.4264625608921051,\n",
            "  \"model.elapsed_time\": 0.0745699405670166,\n",
            "  \"model.engine_step\": 781,\n",
            "  \"model.loss.nll\": 0.02313232421875,\n",
            "  \"elapsed_time\": 0.0745699405670166,\n",
            "  \"wall_time\": 1674064384.6352236,\n",
            "  \"global_step\": 781\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.033905029296875,\n",
            "  \"model.lr\": 0.00015562300000000002,\n",
            "  \"model.grad_norm\": 0.668555736541748,\n",
            "  \"model.elapsed_time\": 0.07243084907531738,\n",
            "  \"model.engine_step\": 782,\n",
            "  \"model.loss.nll\": 0.033905029296875,\n",
            "  \"elapsed_time\": 0.07243084907531738,\n",
            "  \"wall_time\": 1674064384.7102973,\n",
            "  \"global_step\": 782\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0285491943359375,\n",
            "  \"model.lr\": 0.000155822,\n",
            "  \"model.grad_norm\": 0.45936471223831177,\n",
            "  \"model.elapsed_time\": 0.07885909080505371,\n",
            "  \"model.engine_step\": 783,\n",
            "  \"model.loss.nll\": 0.0285491943359375,\n",
            "  \"elapsed_time\": 0.07885909080505371,\n",
            "  \"wall_time\": 1674064384.7971082,\n",
            "  \"global_step\": 783\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.155029296875,\n",
            "  \"model.lr\": 0.000156021,\n",
            "  \"model.grad_norm\": 4.169711112976074,\n",
            "  \"model.elapsed_time\": 0.07245254516601562,\n",
            "  \"model.engine_step\": 784,\n",
            "  \"model.loss.nll\": 0.155029296875,\n",
            "  \"elapsed_time\": 0.07245254516601562,\n",
            "  \"wall_time\": 1674064384.8716269,\n",
            "  \"global_step\": 784\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:04\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021026611328125,\n",
            "  \"model.lr\": 0.00015622000000000002,\n",
            "  \"model.grad_norm\": 0.35829511284828186,\n",
            "  \"model.elapsed_time\": 0.0715334415435791,\n",
            "  \"model.engine_step\": 785,\n",
            "  \"model.loss.nll\": 0.021026611328125,\n",
            "  \"elapsed_time\": 0.0715334415435791,\n",
            "  \"wall_time\": 1674064384.950454,\n",
            "  \"global_step\": 785\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.017852783203125,\n",
            "  \"model.lr\": 0.000156419,\n",
            "  \"model.grad_norm\": 0.6441707015037537,\n",
            "  \"model.elapsed_time\": 0.07342767715454102,\n",
            "  \"model.engine_step\": 786,\n",
            "  \"model.loss.nll\": 0.017852783203125,\n",
            "  \"elapsed_time\": 0.07342767715454102,\n",
            "  \"wall_time\": 1674064385.0266984,\n",
            "  \"global_step\": 786\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05755615234375,\n",
            "  \"model.lr\": 0.00015661800000000002,\n",
            "  \"model.grad_norm\": 1.62522554397583,\n",
            "  \"model.elapsed_time\": 0.09592175483703613,\n",
            "  \"model.engine_step\": 787,\n",
            "  \"model.loss.nll\": 0.05755615234375,\n",
            "  \"elapsed_time\": 0.09592175483703613,\n",
            "  \"wall_time\": 1674064385.132069,\n",
            "  \"global_step\": 787\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.025115966796875,\n",
            "  \"model.lr\": 0.000156817,\n",
            "  \"model.grad_norm\": 0.5718014240264893,\n",
            "  \"model.elapsed_time\": 0.07819890975952148,\n",
            "  \"model.engine_step\": 788,\n",
            "  \"model.loss.nll\": 0.025115966796875,\n",
            "  \"elapsed_time\": 0.07819890975952148,\n",
            "  \"wall_time\": 1674064385.2130933,\n",
            "  \"global_step\": 788\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.025634765625,\n",
            "  \"model.lr\": 0.000157016,\n",
            "  \"model.grad_norm\": 0.4760405719280243,\n",
            "  \"model.elapsed_time\": 0.07109975814819336,\n",
            "  \"model.engine_step\": 789,\n",
            "  \"model.loss.nll\": 0.025634765625,\n",
            "  \"elapsed_time\": 0.07109975814819336,\n",
            "  \"wall_time\": 1674064385.291502,\n",
            "  \"global_step\": 789\n",
            "}\n",
            "[2023-01-18 17:53:05,361] [INFO] [logging.py:68:log_dist] [Rank 0] step=790, skipped=4, lr=[0.00015721500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:05,363] [INFO] [timer.py:196:stop] epoch=0/micro_step=790/global_step=790, RunningAvgSamplesPerSec=12.184013669994568, CurrSamplesPerSec=14.338912591620172, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02313232421875,\n",
            "  \"model.lr\": 0.00015721500000000002,\n",
            "  \"model.grad_norm\": 0.5307387709617615,\n",
            "  \"model.elapsed_time\": 0.07100200653076172,\n",
            "  \"model.engine_step\": 790,\n",
            "  \"model.loss.nll\": 0.02313232421875,\n",
            "  \"elapsed_time\": 0.07100200653076172,\n",
            "  \"wall_time\": 1674064385.3660343,\n",
            "  \"global_step\": 790\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022216796875,\n",
            "  \"model.lr\": 0.000157414,\n",
            "  \"model.grad_norm\": 0.4147816002368927,\n",
            "  \"model.elapsed_time\": 0.07491445541381836,\n",
            "  \"model.engine_step\": 791,\n",
            "  \"model.loss.nll\": 0.022216796875,\n",
            "  \"elapsed_time\": 0.07491445541381836,\n",
            "  \"wall_time\": 1674064385.4514923,\n",
            "  \"global_step\": 791\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0227508544921875,\n",
            "  \"model.lr\": 0.00015761300000000002,\n",
            "  \"model.grad_norm\": 0.4261733591556549,\n",
            "  \"model.elapsed_time\": 0.07228851318359375,\n",
            "  \"model.engine_step\": 792,\n",
            "  \"model.loss.nll\": 0.0227508544921875,\n",
            "  \"elapsed_time\": 0.07228851318359375,\n",
            "  \"wall_time\": 1674064385.5257964,\n",
            "  \"global_step\": 792\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022979736328125,\n",
            "  \"model.lr\": 0.000157812,\n",
            "  \"model.grad_norm\": 0.36543047428131104,\n",
            "  \"model.elapsed_time\": 0.08066272735595703,\n",
            "  \"model.engine_step\": 793,\n",
            "  \"model.loss.nll\": 0.022979736328125,\n",
            "  \"elapsed_time\": 0.08066272735595703,\n",
            "  \"wall_time\": 1674064385.6138632,\n",
            "  \"global_step\": 793\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0738525390625,\n",
            "  \"model.lr\": 0.000158011,\n",
            "  \"model.grad_norm\": 1.9301108121871948,\n",
            "  \"model.elapsed_time\": 0.0732719898223877,\n",
            "  \"model.engine_step\": 794,\n",
            "  \"model.loss.nll\": 0.0738525390625,\n",
            "  \"elapsed_time\": 0.0732719898223877,\n",
            "  \"wall_time\": 1674064385.6908662,\n",
            "  \"global_step\": 794\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0163726806640625,\n",
            "  \"model.lr\": 0.00015821000000000002,\n",
            "  \"model.grad_norm\": 0.28034257888793945,\n",
            "  \"model.elapsed_time\": 0.0751962661743164,\n",
            "  \"model.engine_step\": 795,\n",
            "  \"model.loss.nll\": 0.0163726806640625,\n",
            "  \"elapsed_time\": 0.0751962661743164,\n",
            "  \"wall_time\": 1674064385.7734618,\n",
            "  \"global_step\": 795\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.66552734375,\n",
            "  \"model.lr\": 0.000158409,\n",
            "  \"model.grad_norm\": 7.396425247192383,\n",
            "  \"model.elapsed_time\": 0.07609224319458008,\n",
            "  \"model.engine_step\": 796,\n",
            "  \"model.loss.nll\": 0.66552734375,\n",
            "  \"elapsed_time\": 0.07609224319458008,\n",
            "  \"wall_time\": 1674064385.8516607,\n",
            "  \"global_step\": 796\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:05\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0269622802734375,\n",
            "  \"model.lr\": 0.00015860800000000002,\n",
            "  \"model.grad_norm\": 1.1386427879333496,\n",
            "  \"model.elapsed_time\": 0.08017611503601074,\n",
            "  \"model.engine_step\": 797,\n",
            "  \"model.loss.nll\": 0.0269622802734375,\n",
            "  \"elapsed_time\": 0.08017611503601074,\n",
            "  \"wall_time\": 1674064385.9393642,\n",
            "  \"global_step\": 797\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.040191650390625,\n",
            "  \"model.lr\": 0.000158807,\n",
            "  \"model.grad_norm\": 1.0808846950531006,\n",
            "  \"model.elapsed_time\": 0.07207703590393066,\n",
            "  \"model.engine_step\": 798,\n",
            "  \"model.loss.nll\": 0.040191650390625,\n",
            "  \"elapsed_time\": 0.07207703590393066,\n",
            "  \"wall_time\": 1674064386.0141878,\n",
            "  \"global_step\": 798\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06304931640625,\n",
            "  \"model.lr\": 0.000159006,\n",
            "  \"model.grad_norm\": 1.684443473815918,\n",
            "  \"model.elapsed_time\": 0.07748723030090332,\n",
            "  \"model.engine_step\": 799,\n",
            "  \"model.loss.nll\": 0.06304931640625,\n",
            "  \"elapsed_time\": 0.07748723030090332,\n",
            "  \"wall_time\": 1674064386.0998209,\n",
            "  \"global_step\": 799\n",
            "}\n",
            "[2023-01-18 17:53:06,182] [INFO] [logging.py:68:log_dist] [Rank 0] step=800, skipped=4, lr=[0.00015920500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:06,184] [INFO] [timer.py:196:stop] epoch=0/micro_step=800/global_step=800, RunningAvgSamplesPerSec=12.194622120182391, CurrSamplesPerSec=12.126822216439702, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0257110595703125,\n",
            "  \"model.lr\": 0.00015920500000000002,\n",
            "  \"model.grad_norm\": 0.38531219959259033,\n",
            "  \"model.elapsed_time\": 0.08314681053161621,\n",
            "  \"model.engine_step\": 800,\n",
            "  \"model.loss.nll\": 0.0257110595703125,\n",
            "  \"elapsed_time\": 0.08314681053161621,\n",
            "  \"wall_time\": 1674064386.185672,\n",
            "  \"global_step\": 800\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021209716796875,\n",
            "  \"model.lr\": 0.000159404,\n",
            "  \"model.grad_norm\": 0.4807191491127014,\n",
            "  \"model.elapsed_time\": 0.07487201690673828,\n",
            "  \"model.engine_step\": 801,\n",
            "  \"model.loss.nll\": 0.021209716796875,\n",
            "  \"elapsed_time\": 0.07487201690673828,\n",
            "  \"wall_time\": 1674064386.2698066,\n",
            "  \"global_step\": 801\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.016754150390625,\n",
            "  \"model.lr\": 0.00015960300000000002,\n",
            "  \"model.grad_norm\": 0.32808592915534973,\n",
            "  \"model.elapsed_time\": 0.07542181015014648,\n",
            "  \"model.engine_step\": 802,\n",
            "  \"model.loss.nll\": 0.016754150390625,\n",
            "  \"elapsed_time\": 0.07542181015014648,\n",
            "  \"wall_time\": 1674064386.3473341,\n",
            "  \"global_step\": 802\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0168609619140625,\n",
            "  \"model.lr\": 0.000159802,\n",
            "  \"model.grad_norm\": 0.3036118447780609,\n",
            "  \"model.elapsed_time\": 0.08431196212768555,\n",
            "  \"model.engine_step\": 803,\n",
            "  \"model.loss.nll\": 0.0168609619140625,\n",
            "  \"elapsed_time\": 0.08431196212768555,\n",
            "  \"wall_time\": 1674064386.4407341,\n",
            "  \"global_step\": 803\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0262451171875,\n",
            "  \"model.lr\": 0.00016000100000000003,\n",
            "  \"model.grad_norm\": 1.2847728729248047,\n",
            "  \"model.elapsed_time\": 0.07205963134765625,\n",
            "  \"model.engine_step\": 804,\n",
            "  \"model.loss.nll\": 0.0262451171875,\n",
            "  \"elapsed_time\": 0.07205963134765625,\n",
            "  \"wall_time\": 1674064386.5150287,\n",
            "  \"global_step\": 804\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0174407958984375,\n",
            "  \"model.lr\": 0.00016020000000000002,\n",
            "  \"model.grad_norm\": 0.35056307911872864,\n",
            "  \"model.elapsed_time\": 0.07256364822387695,\n",
            "  \"model.engine_step\": 805,\n",
            "  \"model.loss.nll\": 0.0174407958984375,\n",
            "  \"elapsed_time\": 0.07256364822387695,\n",
            "  \"wall_time\": 1674064386.5994298,\n",
            "  \"global_step\": 805\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2783203125,\n",
            "  \"model.lr\": 0.000160399,\n",
            "  \"model.grad_norm\": 5.488484859466553,\n",
            "  \"model.elapsed_time\": 0.06898665428161621,\n",
            "  \"model.engine_step\": 806,\n",
            "  \"model.loss.nll\": 0.2783203125,\n",
            "  \"elapsed_time\": 0.06898665428161621,\n",
            "  \"wall_time\": 1674064386.6714413,\n",
            "  \"global_step\": 806\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02020263671875,\n",
            "  \"model.lr\": 0.00016059800000000002,\n",
            "  \"model.grad_norm\": 0.36283183097839355,\n",
            "  \"model.elapsed_time\": 0.07217860221862793,\n",
            "  \"model.engine_step\": 807,\n",
            "  \"model.loss.nll\": 0.02020263671875,\n",
            "  \"elapsed_time\": 0.07217860221862793,\n",
            "  \"wall_time\": 1674064386.755611,\n",
            "  \"global_step\": 807\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03253173828125,\n",
            "  \"model.lr\": 0.000160797,\n",
            "  \"model.grad_norm\": 0.6463555097579956,\n",
            "  \"model.elapsed_time\": 0.07797598838806152,\n",
            "  \"model.engine_step\": 808,\n",
            "  \"model.loss.nll\": 0.03253173828125,\n",
            "  \"elapsed_time\": 0.07797598838806152,\n",
            "  \"wall_time\": 1674064386.8361948,\n",
            "  \"global_step\": 808\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01287078857421875,\n",
            "  \"model.lr\": 0.00016099600000000003,\n",
            "  \"model.grad_norm\": 0.32657766342163086,\n",
            "  \"model.elapsed_time\": 0.07828044891357422,\n",
            "  \"model.engine_step\": 809,\n",
            "  \"model.loss.nll\": 0.01287078857421875,\n",
            "  \"elapsed_time\": 0.07828044891357422,\n",
            "  \"wall_time\": 1674064386.9222925,\n",
            "  \"global_step\": 809\n",
            "}\n",
            "[2023-01-18 17:53:06,995] [INFO] [logging.py:68:log_dist] [Rank 0] step=810, skipped=4, lr=[0.00016119500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:06,998] [INFO] [timer.py:196:stop] epoch=0/micro_step=810/global_step=810, RunningAvgSamplesPerSec=12.207607362613784, CurrSamplesPerSec=13.550119532209084, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:06\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.064208984375,\n",
            "  \"model.lr\": 0.00016119500000000002,\n",
            "  \"model.grad_norm\": 1.9047348499298096,\n",
            "  \"model.elapsed_time\": 0.07428288459777832,\n",
            "  \"model.engine_step\": 810,\n",
            "  \"model.loss.nll\": 0.064208984375,\n",
            "  \"elapsed_time\": 0.07428288459777832,\n",
            "  \"wall_time\": 1674064386.9992576,\n",
            "  \"global_step\": 810\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0380859375,\n",
            "  \"model.lr\": 0.000161394,\n",
            "  \"model.grad_norm\": 1.2434402704238892,\n",
            "  \"model.elapsed_time\": 0.0755758285522461,\n",
            "  \"model.engine_step\": 811,\n",
            "  \"model.loss.nll\": 0.0380859375,\n",
            "  \"elapsed_time\": 0.0755758285522461,\n",
            "  \"wall_time\": 1674064387.0824327,\n",
            "  \"global_step\": 811\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0147247314453125,\n",
            "  \"model.lr\": 0.00016159300000000002,\n",
            "  \"model.grad_norm\": 0.38815221190452576,\n",
            "  \"model.elapsed_time\": 0.08396434783935547,\n",
            "  \"model.engine_step\": 812,\n",
            "  \"model.loss.nll\": 0.0147247314453125,\n",
            "  \"elapsed_time\": 0.08396434783935547,\n",
            "  \"wall_time\": 1674064387.1685274,\n",
            "  \"global_step\": 812\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04229736328125,\n",
            "  \"model.lr\": 0.000161792,\n",
            "  \"model.grad_norm\": 1.4353423118591309,\n",
            "  \"model.elapsed_time\": 0.08089160919189453,\n",
            "  \"model.engine_step\": 813,\n",
            "  \"model.loss.nll\": 0.04229736328125,\n",
            "  \"elapsed_time\": 0.08089160919189453,\n",
            "  \"wall_time\": 1674064387.2618203,\n",
            "  \"global_step\": 813\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0202789306640625,\n",
            "  \"model.lr\": 0.00016199100000000003,\n",
            "  \"model.grad_norm\": 0.3423954248428345,\n",
            "  \"model.elapsed_time\": 0.06935310363769531,\n",
            "  \"model.engine_step\": 814,\n",
            "  \"model.loss.nll\": 0.0202789306640625,\n",
            "  \"elapsed_time\": 0.06935310363769531,\n",
            "  \"wall_time\": 1674064387.3339105,\n",
            "  \"global_step\": 814\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0301513671875,\n",
            "  \"model.lr\": 0.00016219000000000002,\n",
            "  \"model.grad_norm\": 0.5923545956611633,\n",
            "  \"model.elapsed_time\": 0.07042908668518066,\n",
            "  \"model.engine_step\": 815,\n",
            "  \"model.loss.nll\": 0.0301513671875,\n",
            "  \"elapsed_time\": 0.07042908668518066,\n",
            "  \"wall_time\": 1674064387.4138381,\n",
            "  \"global_step\": 815\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.281982421875,\n",
            "  \"model.lr\": 0.000162389,\n",
            "  \"model.grad_norm\": 6.656543254852295,\n",
            "  \"model.elapsed_time\": 0.07560372352600098,\n",
            "  \"model.engine_step\": 816,\n",
            "  \"model.loss.nll\": 0.281982421875,\n",
            "  \"elapsed_time\": 0.07560372352600098,\n",
            "  \"wall_time\": 1674064387.492558,\n",
            "  \"global_step\": 816\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.029510498046875,\n",
            "  \"model.lr\": 0.00016258800000000002,\n",
            "  \"model.grad_norm\": 0.5771293640136719,\n",
            "  \"model.elapsed_time\": 0.07451534271240234,\n",
            "  \"model.engine_step\": 817,\n",
            "  \"model.loss.nll\": 0.029510498046875,\n",
            "  \"elapsed_time\": 0.07451534271240234,\n",
            "  \"wall_time\": 1674064387.5746312,\n",
            "  \"global_step\": 817\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0977783203125,\n",
            "  \"model.lr\": 0.00016278699999999998,\n",
            "  \"model.grad_norm\": 2.374054193496704,\n",
            "  \"model.elapsed_time\": 0.0699315071105957,\n",
            "  \"model.engine_step\": 818,\n",
            "  \"model.loss.nll\": 0.0977783203125,\n",
            "  \"elapsed_time\": 0.0699315071105957,\n",
            "  \"wall_time\": 1674064387.6467853,\n",
            "  \"global_step\": 818\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.075927734375,\n",
            "  \"model.lr\": 0.000162986,\n",
            "  \"model.grad_norm\": 2.1418466567993164,\n",
            "  \"model.elapsed_time\": 0.07781195640563965,\n",
            "  \"model.engine_step\": 819,\n",
            "  \"model.loss.nll\": 0.075927734375,\n",
            "  \"elapsed_time\": 0.07781195640563965,\n",
            "  \"wall_time\": 1674064387.7314017,\n",
            "  \"global_step\": 819\n",
            "}\n",
            "[2023-01-18 17:53:07,801] [INFO] [logging.py:68:log_dist] [Rank 0] step=820, skipped=4, lr=[0.000163185], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:07,803] [INFO] [timer.py:196:stop] epoch=0/micro_step=820/global_step=820, RunningAvgSamplesPerSec=12.220753176724138, CurrSamplesPerSec=14.310634958545156, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0274505615234375,\n",
            "  \"model.lr\": 0.000163185,\n",
            "  \"model.grad_norm\": 0.5409768223762512,\n",
            "  \"model.elapsed_time\": 0.0703132152557373,\n",
            "  \"model.engine_step\": 820,\n",
            "  \"model.loss.nll\": 0.0274505615234375,\n",
            "  \"elapsed_time\": 0.0703132152557373,\n",
            "  \"wall_time\": 1674064387.8044841,\n",
            "  \"global_step\": 820\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02227783203125,\n",
            "  \"model.lr\": 0.000163384,\n",
            "  \"model.grad_norm\": 0.3916337788105011,\n",
            "  \"model.elapsed_time\": 0.06841421127319336,\n",
            "  \"model.engine_step\": 821,\n",
            "  \"model.loss.nll\": 0.02227783203125,\n",
            "  \"elapsed_time\": 0.06841421127319336,\n",
            "  \"wall_time\": 1674064387.879764,\n",
            "  \"global_step\": 821\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0262603759765625,\n",
            "  \"model.lr\": 0.000163583,\n",
            "  \"model.grad_norm\": 0.5995113849639893,\n",
            "  \"model.elapsed_time\": 0.06952905654907227,\n",
            "  \"model.engine_step\": 822,\n",
            "  \"model.loss.nll\": 0.0262603759765625,\n",
            "  \"elapsed_time\": 0.06952905654907227,\n",
            "  \"wall_time\": 1674064387.9518845,\n",
            "  \"global_step\": 822\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:07\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0206451416015625,\n",
            "  \"model.lr\": 0.00016378199999999998,\n",
            "  \"model.grad_norm\": 0.3863160014152527,\n",
            "  \"model.elapsed_time\": 0.0776515007019043,\n",
            "  \"model.engine_step\": 823,\n",
            "  \"model.loss.nll\": 0.0206451416015625,\n",
            "  \"elapsed_time\": 0.0776515007019043,\n",
            "  \"wall_time\": 1674064388.0477002,\n",
            "  \"global_step\": 823\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0225067138671875,\n",
            "  \"model.lr\": 0.000163981,\n",
            "  \"model.grad_norm\": 0.41389960050582886,\n",
            "  \"model.elapsed_time\": 0.06963014602661133,\n",
            "  \"model.engine_step\": 824,\n",
            "  \"model.loss.nll\": 0.0225067138671875,\n",
            "  \"elapsed_time\": 0.06963014602661133,\n",
            "  \"wall_time\": 1674064388.1198776,\n",
            "  \"global_step\": 824\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022247314453125,\n",
            "  \"model.lr\": 0.00016418,\n",
            "  \"model.grad_norm\": 0.4563461244106293,\n",
            "  \"model.elapsed_time\": 0.08716845512390137,\n",
            "  \"model.engine_step\": 825,\n",
            "  \"model.loss.nll\": 0.022247314453125,\n",
            "  \"elapsed_time\": 0.08716845512390137,\n",
            "  \"wall_time\": 1674064388.2145534,\n",
            "  \"global_step\": 825\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.64794921875,\n",
            "  \"model.lr\": 0.000164379,\n",
            "  \"model.grad_norm\": 8.421442985534668,\n",
            "  \"model.elapsed_time\": 0.07841730117797852,\n",
            "  \"model.engine_step\": 826,\n",
            "  \"model.loss.nll\": 0.64794921875,\n",
            "  \"elapsed_time\": 0.07841730117797852,\n",
            "  \"wall_time\": 1674064388.295666,\n",
            "  \"global_step\": 826\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.022216796875,\n",
            "  \"model.lr\": 0.000164578,\n",
            "  \"model.grad_norm\": 0.517718493938446,\n",
            "  \"model.elapsed_time\": 0.07422590255737305,\n",
            "  \"model.engine_step\": 827,\n",
            "  \"model.loss.nll\": 0.022216796875,\n",
            "  \"elapsed_time\": 0.07422590255737305,\n",
            "  \"wall_time\": 1674064388.3775563,\n",
            "  \"global_step\": 827\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1812744140625,\n",
            "  \"model.lr\": 0.000164777,\n",
            "  \"model.grad_norm\": 5.1504621505737305,\n",
            "  \"model.elapsed_time\": 0.07497406005859375,\n",
            "  \"model.engine_step\": 828,\n",
            "  \"model.loss.nll\": 0.1812744140625,\n",
            "  \"elapsed_time\": 0.07497406005859375,\n",
            "  \"wall_time\": 1674064388.4555058,\n",
            "  \"global_step\": 828\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.025390625,\n",
            "  \"model.lr\": 0.000164976,\n",
            "  \"model.grad_norm\": 0.5154916048049927,\n",
            "  \"model.elapsed_time\": 0.07850027084350586,\n",
            "  \"model.engine_step\": 829,\n",
            "  \"model.loss.nll\": 0.025390625,\n",
            "  \"elapsed_time\": 0.07850027084350586,\n",
            "  \"wall_time\": 1674064388.5424826,\n",
            "  \"global_step\": 829\n",
            "}\n",
            "[2023-01-18 17:53:08,618] [INFO] [logging.py:68:log_dist] [Rank 0] step=830, skipped=4, lr=[0.000165175], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:08,620] [INFO] [timer.py:196:stop] epoch=0/micro_step=830/global_step=830, RunningAvgSamplesPerSec=12.232351336503275, CurrSamplesPerSec=13.046736530391964, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06854248046875,\n",
            "  \"model.lr\": 0.000165175,\n",
            "  \"model.grad_norm\": 2.2163705825805664,\n",
            "  \"model.elapsed_time\": 0.07714486122131348,\n",
            "  \"model.engine_step\": 830,\n",
            "  \"model.loss.nll\": 0.06854248046875,\n",
            "  \"elapsed_time\": 0.07714486122131348,\n",
            "  \"wall_time\": 1674064388.6219027,\n",
            "  \"global_step\": 830\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0247955322265625,\n",
            "  \"model.lr\": 0.000165374,\n",
            "  \"model.grad_norm\": 0.5037056803703308,\n",
            "  \"model.elapsed_time\": 0.07439684867858887,\n",
            "  \"model.engine_step\": 831,\n",
            "  \"model.loss.nll\": 0.0247955322265625,\n",
            "  \"elapsed_time\": 0.07439684867858887,\n",
            "  \"wall_time\": 1674064388.7048483,\n",
            "  \"global_step\": 831\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.037322998046875,\n",
            "  \"model.lr\": 0.000165573,\n",
            "  \"model.grad_norm\": 0.8340162634849548,\n",
            "  \"model.elapsed_time\": 0.07467246055603027,\n",
            "  \"model.engine_step\": 832,\n",
            "  \"model.loss.nll\": 0.037322998046875,\n",
            "  \"elapsed_time\": 0.07467246055603027,\n",
            "  \"wall_time\": 1674064388.7830365,\n",
            "  \"global_step\": 832\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0184326171875,\n",
            "  \"model.lr\": 0.000165772,\n",
            "  \"model.grad_norm\": 0.5825240612030029,\n",
            "  \"model.elapsed_time\": 0.07233047485351562,\n",
            "  \"model.engine_step\": 833,\n",
            "  \"model.loss.nll\": 0.0184326171875,\n",
            "  \"elapsed_time\": 0.07233047485351562,\n",
            "  \"wall_time\": 1674064388.8656688,\n",
            "  \"global_step\": 833\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.8515625,\n",
            "  \"model.lr\": 0.000165971,\n",
            "  \"model.grad_norm\": 13.101984977722168,\n",
            "  \"model.elapsed_time\": 0.07042837142944336,\n",
            "  \"model.engine_step\": 834,\n",
            "  \"model.loss.nll\": 0.8515625,\n",
            "  \"elapsed_time\": 0.07042837142944336,\n",
            "  \"wall_time\": 1674064388.9383056,\n",
            "  \"global_step\": 834\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:08\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01837158203125,\n",
            "  \"model.lr\": 0.00016617,\n",
            "  \"model.grad_norm\": 0.3618563115596771,\n",
            "  \"model.elapsed_time\": 0.09235525131225586,\n",
            "  \"model.engine_step\": 835,\n",
            "  \"model.loss.nll\": 0.01837158203125,\n",
            "  \"elapsed_time\": 0.09235525131225586,\n",
            "  \"wall_time\": 1674064389.0392032,\n",
            "  \"global_step\": 835\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1551513671875,\n",
            "  \"model.lr\": 0.000166369,\n",
            "  \"model.grad_norm\": 3.498614549636841,\n",
            "  \"model.elapsed_time\": 0.07688736915588379,\n",
            "  \"model.engine_step\": 836,\n",
            "  \"model.loss.nll\": 0.1551513671875,\n",
            "  \"elapsed_time\": 0.07688736915588379,\n",
            "  \"wall_time\": 1674064389.1183074,\n",
            "  \"global_step\": 836\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.031768798828125,\n",
            "  \"model.lr\": 0.000166568,\n",
            "  \"model.grad_norm\": 0.7072185277938843,\n",
            "  \"model.elapsed_time\": 0.08610320091247559,\n",
            "  \"model.engine_step\": 837,\n",
            "  \"model.loss.nll\": 0.031768798828125,\n",
            "  \"elapsed_time\": 0.08610320091247559,\n",
            "  \"wall_time\": 1674064389.2141712,\n",
            "  \"global_step\": 837\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.22900390625,\n",
            "  \"model.lr\": 0.000166767,\n",
            "  \"model.grad_norm\": 4.7805094718933105,\n",
            "  \"model.elapsed_time\": 0.08444523811340332,\n",
            "  \"model.engine_step\": 838,\n",
            "  \"model.loss.nll\": 0.22900390625,\n",
            "  \"elapsed_time\": 0.08444523811340332,\n",
            "  \"wall_time\": 1674064389.3022075,\n",
            "  \"global_step\": 838\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0156402587890625,\n",
            "  \"model.lr\": 0.000166966,\n",
            "  \"model.grad_norm\": 0.32008275389671326,\n",
            "  \"model.elapsed_time\": 0.07234048843383789,\n",
            "  \"model.engine_step\": 839,\n",
            "  \"model.loss.nll\": 0.0156402587890625,\n",
            "  \"elapsed_time\": 0.07234048843383789,\n",
            "  \"wall_time\": 1674064389.3832378,\n",
            "  \"global_step\": 839\n",
            "}\n",
            "[2023-01-18 17:53:09,461] [INFO] [logging.py:68:log_dist] [Rank 0] step=840, skipped=4, lr=[0.000167165], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:09,463] [INFO] [timer.py:196:stop] epoch=0/micro_step=840/global_step=840, RunningAvgSamplesPerSec=12.238767781211997, CurrSamplesPerSec=12.723390959584776, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05975341796875,\n",
            "  \"model.lr\": 0.000167165,\n",
            "  \"model.grad_norm\": 1.7550715208053589,\n",
            "  \"model.elapsed_time\": 0.0790412425994873,\n",
            "  \"model.engine_step\": 840,\n",
            "  \"model.loss.nll\": 0.05975341796875,\n",
            "  \"elapsed_time\": 0.0790412425994873,\n",
            "  \"wall_time\": 1674064389.4643648,\n",
            "  \"global_step\": 840\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.297607421875,\n",
            "  \"model.lr\": 0.000167364,\n",
            "  \"model.grad_norm\": 6.116540431976318,\n",
            "  \"model.elapsed_time\": 0.07189130783081055,\n",
            "  \"model.engine_step\": 841,\n",
            "  \"model.loss.nll\": 0.297607421875,\n",
            "  \"elapsed_time\": 0.07189130783081055,\n",
            "  \"wall_time\": 1674064389.5437572,\n",
            "  \"global_step\": 841\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0204315185546875,\n",
            "  \"model.lr\": 0.000167563,\n",
            "  \"model.grad_norm\": 0.5532959699630737,\n",
            "  \"model.elapsed_time\": 0.0770728588104248,\n",
            "  \"model.engine_step\": 842,\n",
            "  \"model.loss.nll\": 0.0204315185546875,\n",
            "  \"elapsed_time\": 0.0770728588104248,\n",
            "  \"wall_time\": 1674064389.6238935,\n",
            "  \"global_step\": 842\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0214691162109375,\n",
            "  \"model.lr\": 0.000167762,\n",
            "  \"model.grad_norm\": 0.49759653210639954,\n",
            "  \"model.elapsed_time\": 0.07811808586120605,\n",
            "  \"model.engine_step\": 843,\n",
            "  \"model.loss.nll\": 0.0214691162109375,\n",
            "  \"elapsed_time\": 0.07811808586120605,\n",
            "  \"wall_time\": 1674064389.7109773,\n",
            "  \"global_step\": 843\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.50830078125,\n",
            "  \"model.lr\": 0.000167961,\n",
            "  \"model.grad_norm\": 7.770534038543701,\n",
            "  \"model.elapsed_time\": 0.07353377342224121,\n",
            "  \"model.engine_step\": 844,\n",
            "  \"model.loss.nll\": 0.50830078125,\n",
            "  \"elapsed_time\": 0.07353377342224121,\n",
            "  \"wall_time\": 1674064389.7879405,\n",
            "  \"global_step\": 844\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0435791015625,\n",
            "  \"model.lr\": 0.00016816,\n",
            "  \"model.grad_norm\": 0.8977478742599487,\n",
            "  \"model.elapsed_time\": 0.07082867622375488,\n",
            "  \"model.engine_step\": 845,\n",
            "  \"model.loss.nll\": 0.0435791015625,\n",
            "  \"elapsed_time\": 0.07082867622375488,\n",
            "  \"wall_time\": 1674064389.8668888,\n",
            "  \"global_step\": 845\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01548004150390625,\n",
            "  \"model.lr\": 0.000168359,\n",
            "  \"model.grad_norm\": 0.32710620760917664,\n",
            "  \"model.elapsed_time\": 0.07628917694091797,\n",
            "  \"model.engine_step\": 846,\n",
            "  \"model.loss.nll\": 0.01548004150390625,\n",
            "  \"elapsed_time\": 0.07628917694091797,\n",
            "  \"wall_time\": 1674064389.9451513,\n",
            "  \"global_step\": 846\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:09\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.026885986328125,\n",
            "  \"model.lr\": 0.000168558,\n",
            "  \"model.grad_norm\": 0.48389121890068054,\n",
            "  \"model.elapsed_time\": 0.08215594291687012,\n",
            "  \"model.engine_step\": 847,\n",
            "  \"model.loss.nll\": 0.026885986328125,\n",
            "  \"elapsed_time\": 0.08215594291687012,\n",
            "  \"wall_time\": 1674064390.0357652,\n",
            "  \"global_step\": 847\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0277557373046875,\n",
            "  \"model.lr\": 0.000168757,\n",
            "  \"model.grad_norm\": 0.5502468347549438,\n",
            "  \"model.elapsed_time\": 0.07179403305053711,\n",
            "  \"model.engine_step\": 848,\n",
            "  \"model.loss.nll\": 0.0277557373046875,\n",
            "  \"elapsed_time\": 0.07179403305053711,\n",
            "  \"wall_time\": 1674064390.1102083,\n",
            "  \"global_step\": 848\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0284881591796875,\n",
            "  \"model.lr\": 0.000168956,\n",
            "  \"model.grad_norm\": 0.5317550301551819,\n",
            "  \"model.elapsed_time\": 0.07546663284301758,\n",
            "  \"model.engine_step\": 849,\n",
            "  \"model.loss.nll\": 0.0284881591796875,\n",
            "  \"elapsed_time\": 0.07546663284301758,\n",
            "  \"wall_time\": 1674064390.1935334,\n",
            "  \"global_step\": 849\n",
            "}\n",
            "[2023-01-18 17:53:10,281] [INFO] [logging.py:68:log_dist] [Rank 0] step=850, skipped=4, lr=[0.000169155], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:10,283] [INFO] [timer.py:196:stop] epoch=0/micro_step=850/global_step=850, RunningAvgSamplesPerSec=12.248076637384315, CurrSamplesPerSec=11.347795267495279, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.020721435546875,\n",
            "  \"model.lr\": 0.000169155,\n",
            "  \"model.grad_norm\": 0.40272024273872375,\n",
            "  \"model.elapsed_time\": 0.08862948417663574,\n",
            "  \"model.engine_step\": 850,\n",
            "  \"model.loss.nll\": 0.020721435546875,\n",
            "  \"elapsed_time\": 0.08862948417663574,\n",
            "  \"wall_time\": 1674064390.2852135,\n",
            "  \"global_step\": 850\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0267181396484375,\n",
            "  \"model.lr\": 0.000169354,\n",
            "  \"model.grad_norm\": 0.5256052017211914,\n",
            "  \"model.elapsed_time\": 0.07573199272155762,\n",
            "  \"model.engine_step\": 851,\n",
            "  \"model.loss.nll\": 0.0267181396484375,\n",
            "  \"elapsed_time\": 0.07573199272155762,\n",
            "  \"wall_time\": 1674064390.36916,\n",
            "  \"global_step\": 851\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0145111083984375,\n",
            "  \"model.lr\": 0.000169553,\n",
            "  \"model.grad_norm\": 0.26516079902648926,\n",
            "  \"model.elapsed_time\": 0.07959699630737305,\n",
            "  \"model.engine_step\": 852,\n",
            "  \"model.loss.nll\": 0.0145111083984375,\n",
            "  \"elapsed_time\": 0.07959699630737305,\n",
            "  \"wall_time\": 1674064390.4509463,\n",
            "  \"global_step\": 852\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0268402099609375,\n",
            "  \"model.lr\": 0.000169752,\n",
            "  \"model.grad_norm\": 0.6859773993492126,\n",
            "  \"model.elapsed_time\": 0.07296061515808105,\n",
            "  \"model.engine_step\": 853,\n",
            "  \"model.loss.nll\": 0.0268402099609375,\n",
            "  \"elapsed_time\": 0.07296061515808105,\n",
            "  \"wall_time\": 1674064390.5313928,\n",
            "  \"global_step\": 853\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06768798828125,\n",
            "  \"model.lr\": 0.000169951,\n",
            "  \"model.grad_norm\": 1.9246219396591187,\n",
            "  \"model.elapsed_time\": 0.07150554656982422,\n",
            "  \"model.engine_step\": 854,\n",
            "  \"model.loss.nll\": 0.06768798828125,\n",
            "  \"elapsed_time\": 0.07150554656982422,\n",
            "  \"wall_time\": 1674064390.6051085,\n",
            "  \"global_step\": 854\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.6025390625,\n",
            "  \"model.lr\": 0.00017015000000000002,\n",
            "  \"model.grad_norm\": 10.084567070007324,\n",
            "  \"model.elapsed_time\": 0.08246946334838867,\n",
            "  \"model.engine_step\": 855,\n",
            "  \"model.loss.nll\": 0.6025390625,\n",
            "  \"elapsed_time\": 0.08246946334838867,\n",
            "  \"wall_time\": 1674064390.6943436,\n",
            "  \"global_step\": 855\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0265655517578125,\n",
            "  \"model.lr\": 0.000170349,\n",
            "  \"model.grad_norm\": 0.5936054587364197,\n",
            "  \"model.elapsed_time\": 0.07364797592163086,\n",
            "  \"model.engine_step\": 856,\n",
            "  \"model.loss.nll\": 0.0265655517578125,\n",
            "  \"elapsed_time\": 0.07364797592163086,\n",
            "  \"wall_time\": 1674064390.7706928,\n",
            "  \"global_step\": 856\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.08721923828125,\n",
            "  \"model.lr\": 0.000170548,\n",
            "  \"model.grad_norm\": 2.595851421356201,\n",
            "  \"model.elapsed_time\": 0.07104134559631348,\n",
            "  \"model.engine_step\": 857,\n",
            "  \"model.loss.nll\": 0.08721923828125,\n",
            "  \"elapsed_time\": 0.07104134559631348,\n",
            "  \"wall_time\": 1674064390.848384,\n",
            "  \"global_step\": 857\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01416015625,\n",
            "  \"model.lr\": 0.000170747,\n",
            "  \"model.grad_norm\": 0.3369526267051697,\n",
            "  \"model.elapsed_time\": 0.08351731300354004,\n",
            "  \"model.engine_step\": 858,\n",
            "  \"model.loss.nll\": 0.01416015625,\n",
            "  \"elapsed_time\": 0.08351731300354004,\n",
            "  \"wall_time\": 1674064390.9348123,\n",
            "  \"global_step\": 858\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:10\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.716796875,\n",
            "  \"model.lr\": 0.000170946,\n",
            "  \"model.grad_norm\": 9.17720890045166,\n",
            "  \"model.elapsed_time\": 0.07330751419067383,\n",
            "  \"model.engine_step\": 859,\n",
            "  \"model.loss.nll\": 0.716796875,\n",
            "  \"elapsed_time\": 0.07330751419067383,\n",
            "  \"wall_time\": 1674064391.0153039,\n",
            "  \"global_step\": 859\n",
            "}\n",
            "[2023-01-18 17:53:11,095] [INFO] [logging.py:68:log_dist] [Rank 0] step=860, skipped=4, lr=[0.00017114500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:11,097] [INFO] [timer.py:196:stop] epoch=0/micro_step=860/global_step=860, RunningAvgSamplesPerSec=12.257430087222373, CurrSamplesPerSec=12.493905417803568, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0162506103515625,\n",
            "  \"model.lr\": 0.00017114500000000002,\n",
            "  \"model.grad_norm\": 0.333516925573349,\n",
            "  \"model.elapsed_time\": 0.0805048942565918,\n",
            "  \"model.engine_step\": 860,\n",
            "  \"model.loss.nll\": 0.0162506103515625,\n",
            "  \"elapsed_time\": 0.0805048942565918,\n",
            "  \"wall_time\": 1674064391.0984337,\n",
            "  \"global_step\": 860\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01557159423828125,\n",
            "  \"model.lr\": 0.000171344,\n",
            "  \"model.grad_norm\": 0.3697510063648224,\n",
            "  \"model.elapsed_time\": 0.07990312576293945,\n",
            "  \"model.engine_step\": 861,\n",
            "  \"model.loss.nll\": 0.01557159423828125,\n",
            "  \"elapsed_time\": 0.07990312576293945,\n",
            "  \"wall_time\": 1674064391.1853473,\n",
            "  \"global_step\": 861\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0153961181640625,\n",
            "  \"model.lr\": 0.000171543,\n",
            "  \"model.grad_norm\": 0.3522690236568451,\n",
            "  \"model.elapsed_time\": 0.08697032928466797,\n",
            "  \"model.engine_step\": 862,\n",
            "  \"model.loss.nll\": 0.0153961181640625,\n",
            "  \"elapsed_time\": 0.08697032928466797,\n",
            "  \"wall_time\": 1674064391.2750957,\n",
            "  \"global_step\": 862\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0246429443359375,\n",
            "  \"model.lr\": 0.000171742,\n",
            "  \"model.grad_norm\": 0.48354169726371765,\n",
            "  \"model.elapsed_time\": 0.07550477981567383,\n",
            "  \"model.engine_step\": 863,\n",
            "  \"model.loss.nll\": 0.0246429443359375,\n",
            "  \"elapsed_time\": 0.07550477981567383,\n",
            "  \"wall_time\": 1674064391.358435,\n",
            "  \"global_step\": 863\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.043670654296875,\n",
            "  \"model.lr\": 0.000171941,\n",
            "  \"model.grad_norm\": 1.1619160175323486,\n",
            "  \"model.elapsed_time\": 0.07567644119262695,\n",
            "  \"model.engine_step\": 864,\n",
            "  \"model.loss.nll\": 0.043670654296875,\n",
            "  \"elapsed_time\": 0.07567644119262695,\n",
            "  \"wall_time\": 1674064391.4378216,\n",
            "  \"global_step\": 864\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0229644775390625,\n",
            "  \"model.lr\": 0.00017214000000000002,\n",
            "  \"model.grad_norm\": 0.6000534296035767,\n",
            "  \"model.elapsed_time\": 0.07552099227905273,\n",
            "  \"model.engine_step\": 865,\n",
            "  \"model.loss.nll\": 0.0229644775390625,\n",
            "  \"elapsed_time\": 0.07552099227905273,\n",
            "  \"wall_time\": 1674064391.5220187,\n",
            "  \"global_step\": 865\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.047576904296875,\n",
            "  \"model.lr\": 0.000172339,\n",
            "  \"model.grad_norm\": 1.1382980346679688,\n",
            "  \"model.elapsed_time\": 0.07743668556213379,\n",
            "  \"model.engine_step\": 866,\n",
            "  \"model.loss.nll\": 0.047576904296875,\n",
            "  \"elapsed_time\": 0.07743668556213379,\n",
            "  \"wall_time\": 1674064391.602339,\n",
            "  \"global_step\": 866\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.6142578125,\n",
            "  \"model.lr\": 0.000172538,\n",
            "  \"model.grad_norm\": 8.595935821533203,\n",
            "  \"model.elapsed_time\": 0.07848453521728516,\n",
            "  \"model.engine_step\": 867,\n",
            "  \"model.loss.nll\": 0.6142578125,\n",
            "  \"elapsed_time\": 0.07848453521728516,\n",
            "  \"wall_time\": 1674064391.6884825,\n",
            "  \"global_step\": 867\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04937744140625,\n",
            "  \"model.lr\": 0.000172737,\n",
            "  \"model.grad_norm\": 1.2788041830062866,\n",
            "  \"model.elapsed_time\": 0.0756223201751709,\n",
            "  \"model.engine_step\": 868,\n",
            "  \"model.loss.nll\": 0.04937744140625,\n",
            "  \"elapsed_time\": 0.0756223201751709,\n",
            "  \"wall_time\": 1674064391.7662022,\n",
            "  \"global_step\": 868\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1458740234375,\n",
            "  \"model.lr\": 0.000172936,\n",
            "  \"model.grad_norm\": 4.25753927230835,\n",
            "  \"model.elapsed_time\": 0.07398819923400879,\n",
            "  \"model.engine_step\": 869,\n",
            "  \"model.loss.nll\": 0.1458740234375,\n",
            "  \"elapsed_time\": 0.07398819923400879,\n",
            "  \"wall_time\": 1674064391.8477125,\n",
            "  \"global_step\": 869\n",
            "}\n",
            "[2023-01-18 17:53:11,917] [INFO] [logging.py:68:log_dist] [Rank 0] step=870, skipped=4, lr=[0.00017313500000000002], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:11,920] [INFO] [timer.py:196:stop] epoch=0/micro_step=870/global_step=870, RunningAvgSamplesPerSec=12.26559704773099, CurrSamplesPerSec=14.19545940677163, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.419189453125,\n",
            "  \"model.lr\": 0.00017313500000000002,\n",
            "  \"model.grad_norm\": 7.524938106536865,\n",
            "  \"model.elapsed_time\": 0.07088351249694824,\n",
            "  \"model.engine_step\": 870,\n",
            "  \"model.loss.nll\": 0.419189453125,\n",
            "  \"elapsed_time\": 0.07088351249694824,\n",
            "  \"wall_time\": 1674064391.921133,\n",
            "  \"global_step\": 870\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:11\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1097412109375,\n",
            "  \"model.lr\": 0.000173334,\n",
            "  \"model.grad_norm\": 3.0402991771698,\n",
            "  \"model.elapsed_time\": 0.07307243347167969,\n",
            "  \"model.engine_step\": 871,\n",
            "  \"model.loss.nll\": 0.1097412109375,\n",
            "  \"elapsed_time\": 0.07307243347167969,\n",
            "  \"wall_time\": 1674064392.0013726,\n",
            "  \"global_step\": 871\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03240966796875,\n",
            "  \"model.lr\": 0.000173533,\n",
            "  \"model.grad_norm\": 0.9016122221946716,\n",
            "  \"model.elapsed_time\": 0.07588982582092285,\n",
            "  \"model.engine_step\": 872,\n",
            "  \"model.loss.nll\": 0.03240966796875,\n",
            "  \"elapsed_time\": 0.07588982582092285,\n",
            "  \"wall_time\": 1674064392.079897,\n",
            "  \"global_step\": 872\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0384521484375,\n",
            "  \"model.lr\": 0.000173732,\n",
            "  \"model.grad_norm\": 0.8394725322723389,\n",
            "  \"model.elapsed_time\": 0.08160996437072754,\n",
            "  \"model.engine_step\": 873,\n",
            "  \"model.loss.nll\": 0.0384521484375,\n",
            "  \"elapsed_time\": 0.08160996437072754,\n",
            "  \"wall_time\": 1674064392.1687663,\n",
            "  \"global_step\": 873\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01465606689453125,\n",
            "  \"model.lr\": 0.000173931,\n",
            "  \"model.grad_norm\": 0.4244706630706787,\n",
            "  \"model.elapsed_time\": 0.08233523368835449,\n",
            "  \"model.engine_step\": 874,\n",
            "  \"model.loss.nll\": 0.01465606689453125,\n",
            "  \"elapsed_time\": 0.08233523368835449,\n",
            "  \"wall_time\": 1674064392.2543454,\n",
            "  \"global_step\": 874\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.420166015625,\n",
            "  \"model.lr\": 0.00017413000000000002,\n",
            "  \"model.grad_norm\": 5.970200061798096,\n",
            "  \"model.elapsed_time\": 0.06999540328979492,\n",
            "  \"model.engine_step\": 875,\n",
            "  \"model.loss.nll\": 0.420166015625,\n",
            "  \"elapsed_time\": 0.06999540328979492,\n",
            "  \"wall_time\": 1674064392.3340662,\n",
            "  \"global_step\": 875\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01666259765625,\n",
            "  \"model.lr\": 0.000174329,\n",
            "  \"model.grad_norm\": 0.37801915407180786,\n",
            "  \"model.elapsed_time\": 0.07100224494934082,\n",
            "  \"model.engine_step\": 876,\n",
            "  \"model.loss.nll\": 0.01666259765625,\n",
            "  \"elapsed_time\": 0.07100224494934082,\n",
            "  \"wall_time\": 1674064392.4072416,\n",
            "  \"global_step\": 876\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0239715576171875,\n",
            "  \"model.lr\": 0.00017452800000000002,\n",
            "  \"model.grad_norm\": 0.788063645362854,\n",
            "  \"model.elapsed_time\": 0.08695673942565918,\n",
            "  \"model.engine_step\": 877,\n",
            "  \"model.loss.nll\": 0.0239715576171875,\n",
            "  \"elapsed_time\": 0.08695673942565918,\n",
            "  \"wall_time\": 1674064392.503829,\n",
            "  \"global_step\": 877\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06951904296875,\n",
            "  \"model.lr\": 0.000174727,\n",
            "  \"model.grad_norm\": 2.7069857120513916,\n",
            "  \"model.elapsed_time\": 0.07361817359924316,\n",
            "  \"model.engine_step\": 878,\n",
            "  \"model.loss.nll\": 0.06951904296875,\n",
            "  \"elapsed_time\": 0.07361817359924316,\n",
            "  \"wall_time\": 1674064392.5799253,\n",
            "  \"global_step\": 878\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0751953125,\n",
            "  \"model.lr\": 0.000174926,\n",
            "  \"model.grad_norm\": 1.9355393648147583,\n",
            "  \"model.elapsed_time\": 0.08286809921264648,\n",
            "  \"model.engine_step\": 879,\n",
            "  \"model.loss.nll\": 0.0751953125,\n",
            "  \"elapsed_time\": 0.08286809921264648,\n",
            "  \"wall_time\": 1674064392.6704752,\n",
            "  \"global_step\": 879\n",
            "}\n",
            "[2023-01-18 17:53:12,742] [INFO] [logging.py:68:log_dist] [Rank 0] step=880, skipped=4, lr=[0.00017512500000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:12,744] [INFO] [timer.py:196:stop] epoch=0/micro_step=880/global_step=880, RunningAvgSamplesPerSec=12.273546564909894, CurrSamplesPerSec=13.814957543658558, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0204620361328125,\n",
            "  \"model.lr\": 0.00017512500000000001,\n",
            "  \"model.grad_norm\": 0.4049054682254791,\n",
            "  \"model.elapsed_time\": 0.07285904884338379,\n",
            "  \"model.engine_step\": 880,\n",
            "  \"model.loss.nll\": 0.0204620361328125,\n",
            "  \"elapsed_time\": 0.07285904884338379,\n",
            "  \"wall_time\": 1674064392.7460103,\n",
            "  \"global_step\": 880\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04541015625,\n",
            "  \"model.lr\": 0.000175324,\n",
            "  \"model.grad_norm\": 1.0529515743255615,\n",
            "  \"model.elapsed_time\": 0.07303023338317871,\n",
            "  \"model.engine_step\": 881,\n",
            "  \"model.loss.nll\": 0.04541015625,\n",
            "  \"elapsed_time\": 0.07303023338317871,\n",
            "  \"wall_time\": 1674064392.8270068,\n",
            "  \"global_step\": 881\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.020416259765625,\n",
            "  \"model.lr\": 0.00017552300000000002,\n",
            "  \"model.grad_norm\": 0.5755459666252136,\n",
            "  \"model.elapsed_time\": 0.07695293426513672,\n",
            "  \"model.engine_step\": 882,\n",
            "  \"model.loss.nll\": 0.020416259765625,\n",
            "  \"elapsed_time\": 0.07695293426513672,\n",
            "  \"wall_time\": 1674064392.9066088,\n",
            "  \"global_step\": 882\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:12\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.68798828125,\n",
            "  \"model.lr\": 0.000175722,\n",
            "  \"model.grad_norm\": 9.28740119934082,\n",
            "  \"model.elapsed_time\": 0.07879638671875,\n",
            "  \"model.engine_step\": 883,\n",
            "  \"model.loss.nll\": 0.68798828125,\n",
            "  \"elapsed_time\": 0.07879638671875,\n",
            "  \"wall_time\": 1674064392.9922245,\n",
            "  \"global_step\": 883\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.56396484375,\n",
            "  \"model.lr\": 0.000175921,\n",
            "  \"model.grad_norm\": 9.049835205078125,\n",
            "  \"model.elapsed_time\": 0.07499527931213379,\n",
            "  \"model.engine_step\": 884,\n",
            "  \"model.loss.nll\": 0.56396484375,\n",
            "  \"elapsed_time\": 0.07499527931213379,\n",
            "  \"wall_time\": 1674064393.0701988,\n",
            "  \"global_step\": 884\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01453399658203125,\n",
            "  \"model.lr\": 0.00017612000000000001,\n",
            "  \"model.grad_norm\": 0.23192323744297028,\n",
            "  \"model.elapsed_time\": 0.0783236026763916,\n",
            "  \"model.engine_step\": 885,\n",
            "  \"model.loss.nll\": 0.01453399658203125,\n",
            "  \"elapsed_time\": 0.0783236026763916,\n",
            "  \"wall_time\": 1674064393.1560566,\n",
            "  \"global_step\": 885\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.52685546875,\n",
            "  \"model.lr\": 0.000176319,\n",
            "  \"model.grad_norm\": 8.20711898803711,\n",
            "  \"model.elapsed_time\": 0.07161092758178711,\n",
            "  \"model.engine_step\": 886,\n",
            "  \"model.loss.nll\": 0.52685546875,\n",
            "  \"elapsed_time\": 0.07161092758178711,\n",
            "  \"wall_time\": 1674064393.2297413,\n",
            "  \"global_step\": 886\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01224517822265625,\n",
            "  \"model.lr\": 0.00017651800000000002,\n",
            "  \"model.grad_norm\": 0.20834945142269135,\n",
            "  \"model.elapsed_time\": 0.0970461368560791,\n",
            "  \"model.engine_step\": 887,\n",
            "  \"model.loss.nll\": 0.01224517822265625,\n",
            "  \"elapsed_time\": 0.0970461368560791,\n",
            "  \"wall_time\": 1674064393.33345,\n",
            "  \"global_step\": 887\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.488037109375,\n",
            "  \"model.lr\": 0.000176717,\n",
            "  \"model.grad_norm\": 6.8071980476379395,\n",
            "  \"model.elapsed_time\": 0.07098984718322754,\n",
            "  \"model.engine_step\": 888,\n",
            "  \"model.loss.nll\": 0.488037109375,\n",
            "  \"elapsed_time\": 0.07098984718322754,\n",
            "  \"wall_time\": 1674064393.4071813,\n",
            "  \"global_step\": 888\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07537841796875,\n",
            "  \"model.lr\": 0.000176916,\n",
            "  \"model.grad_norm\": 1.777167797088623,\n",
            "  \"model.elapsed_time\": 0.07866120338439941,\n",
            "  \"model.engine_step\": 889,\n",
            "  \"model.loss.nll\": 0.07537841796875,\n",
            "  \"elapsed_time\": 0.07866120338439941,\n",
            "  \"wall_time\": 1674064393.4929905,\n",
            "  \"global_step\": 889\n",
            "}\n",
            "[2023-01-18 17:53:13,567] [INFO] [logging.py:68:log_dist] [Rank 0] step=890, skipped=4, lr=[0.00017711500000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:13,569] [INFO] [timer.py:196:stop] epoch=0/micro_step=890/global_step=890, RunningAvgSamplesPerSec=12.280434728097932, CurrSamplesPerSec=13.410143523536389, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.07177734375,\n",
            "  \"model.lr\": 0.00017711500000000001,\n",
            "  \"model.grad_norm\": 2.2008190155029297,\n",
            "  \"model.elapsed_time\": 0.07502508163452148,\n",
            "  \"model.engine_step\": 890,\n",
            "  \"model.loss.nll\": 0.07177734375,\n",
            "  \"elapsed_time\": 0.07502508163452148,\n",
            "  \"wall_time\": 1674064393.5707574,\n",
            "  \"global_step\": 890\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01293182373046875,\n",
            "  \"model.lr\": 0.000177314,\n",
            "  \"model.grad_norm\": 0.4091307818889618,\n",
            "  \"model.elapsed_time\": 0.07815003395080566,\n",
            "  \"model.engine_step\": 891,\n",
            "  \"model.loss.nll\": 0.01293182373046875,\n",
            "  \"elapsed_time\": 0.07815003395080566,\n",
            "  \"wall_time\": 1674064393.6577144,\n",
            "  \"global_step\": 891\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0933837890625,\n",
            "  \"model.lr\": 0.00017751300000000002,\n",
            "  \"model.grad_norm\": 2.757537364959717,\n",
            "  \"model.elapsed_time\": 0.07486534118652344,\n",
            "  \"model.engine_step\": 892,\n",
            "  \"model.loss.nll\": 0.0933837890625,\n",
            "  \"elapsed_time\": 0.07486534118652344,\n",
            "  \"wall_time\": 1674064393.7348633,\n",
            "  \"global_step\": 892\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.035003662109375,\n",
            "  \"model.lr\": 0.000177712,\n",
            "  \"model.grad_norm\": 0.9118823409080505,\n",
            "  \"model.elapsed_time\": 0.08561539649963379,\n",
            "  \"model.engine_step\": 893,\n",
            "  \"model.loss.nll\": 0.035003662109375,\n",
            "  \"elapsed_time\": 0.08561539649963379,\n",
            "  \"wall_time\": 1674064393.828784,\n",
            "  \"global_step\": 893\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.037994384765625,\n",
            "  \"model.lr\": 0.000177911,\n",
            "  \"model.grad_norm\": 0.7795424461364746,\n",
            "  \"model.elapsed_time\": 0.07521843910217285,\n",
            "  \"model.engine_step\": 894,\n",
            "  \"model.loss.nll\": 0.037994384765625,\n",
            "  \"elapsed_time\": 0.07521843910217285,\n",
            "  \"wall_time\": 1674064393.9069135,\n",
            "  \"global_step\": 894\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:13\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0135955810546875,\n",
            "  \"model.lr\": 0.00017811000000000001,\n",
            "  \"model.grad_norm\": 0.3395198583602905,\n",
            "  \"model.elapsed_time\": 0.08001279830932617,\n",
            "  \"model.engine_step\": 895,\n",
            "  \"model.loss.nll\": 0.0135955810546875,\n",
            "  \"elapsed_time\": 0.08001279830932617,\n",
            "  \"wall_time\": 1674064393.9953992,\n",
            "  \"global_step\": 895\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0175323486328125,\n",
            "  \"model.lr\": 0.000178309,\n",
            "  \"model.grad_norm\": 0.4272880256175995,\n",
            "  \"model.elapsed_time\": 0.08066868782043457,\n",
            "  \"model.engine_step\": 896,\n",
            "  \"model.loss.nll\": 0.0175323486328125,\n",
            "  \"elapsed_time\": 0.08066868782043457,\n",
            "  \"wall_time\": 1674064394.0781817,\n",
            "  \"global_step\": 896\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021270751953125,\n",
            "  \"model.lr\": 0.00017850800000000002,\n",
            "  \"model.grad_norm\": 0.560633659362793,\n",
            "  \"model.elapsed_time\": 0.07495594024658203,\n",
            "  \"model.engine_step\": 897,\n",
            "  \"model.loss.nll\": 0.021270751953125,\n",
            "  \"elapsed_time\": 0.07495594024658203,\n",
            "  \"wall_time\": 1674064394.1606262,\n",
            "  \"global_step\": 897\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0467529296875,\n",
            "  \"model.lr\": 0.000178707,\n",
            "  \"model.grad_norm\": 1.1104085445404053,\n",
            "  \"model.elapsed_time\": 0.07551789283752441,\n",
            "  \"model.engine_step\": 898,\n",
            "  \"model.loss.nll\": 0.0467529296875,\n",
            "  \"elapsed_time\": 0.07551789283752441,\n",
            "  \"wall_time\": 1674064394.238889,\n",
            "  \"global_step\": 898\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0298309326171875,\n",
            "  \"model.lr\": 0.00017890600000000002,\n",
            "  \"model.grad_norm\": 0.5860147476196289,\n",
            "  \"model.elapsed_time\": 0.09187960624694824,\n",
            "  \"model.engine_step\": 899,\n",
            "  \"model.loss.nll\": 0.0298309326171875,\n",
            "  \"elapsed_time\": 0.09187960624694824,\n",
            "  \"wall_time\": 1674064394.3384542,\n",
            "  \"global_step\": 899\n",
            "}\n",
            "[2023-01-18 17:53:14,413] [INFO] [logging.py:68:log_dist] [Rank 0] step=900, skipped=4, lr=[0.00017910500000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:14,415] [INFO] [timer.py:196:stop] epoch=0/micro_step=900/global_step=900, RunningAvgSamplesPerSec=12.28426981420582, CurrSamplesPerSec=13.262872971503016, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.292236328125,\n",
            "  \"model.lr\": 0.00017910500000000001,\n",
            "  \"model.grad_norm\": 5.231348037719727,\n",
            "  \"model.elapsed_time\": 0.07586407661437988,\n",
            "  \"model.engine_step\": 900,\n",
            "  \"model.loss.nll\": 0.292236328125,\n",
            "  \"elapsed_time\": 0.07586407661437988,\n",
            "  \"wall_time\": 1674064394.417124,\n",
            "  \"global_step\": 900\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2171630859375,\n",
            "  \"model.lr\": 0.000179304,\n",
            "  \"model.grad_norm\": 4.114341735839844,\n",
            "  \"model.elapsed_time\": 0.07856321334838867,\n",
            "  \"model.engine_step\": 901,\n",
            "  \"model.loss.nll\": 0.2171630859375,\n",
            "  \"elapsed_time\": 0.07856321334838867,\n",
            "  \"wall_time\": 1674064394.5036664,\n",
            "  \"global_step\": 901\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1839599609375,\n",
            "  \"model.lr\": 0.00017950300000000002,\n",
            "  \"model.grad_norm\": 3.925464630126953,\n",
            "  \"model.elapsed_time\": 0.07392358779907227,\n",
            "  \"model.engine_step\": 902,\n",
            "  \"model.loss.nll\": 0.1839599609375,\n",
            "  \"elapsed_time\": 0.07392358779907227,\n",
            "  \"wall_time\": 1674064394.580963,\n",
            "  \"global_step\": 902\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.951171875,\n",
            "  \"model.lr\": 0.000179702,\n",
            "  \"model.grad_norm\": 12.067442893981934,\n",
            "  \"model.elapsed_time\": 0.07532405853271484,\n",
            "  \"model.engine_step\": 903,\n",
            "  \"model.loss.nll\": 0.951171875,\n",
            "  \"elapsed_time\": 0.07532405853271484,\n",
            "  \"wall_time\": 1674064394.6684566,\n",
            "  \"global_step\": 903\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.58935546875,\n",
            "  \"model.lr\": 0.00017990100000000002,\n",
            "  \"model.grad_norm\": 8.932031631469727,\n",
            "  \"model.elapsed_time\": 0.07700943946838379,\n",
            "  \"model.engine_step\": 904,\n",
            "  \"model.loss.nll\": 0.58935546875,\n",
            "  \"elapsed_time\": 0.07700943946838379,\n",
            "  \"wall_time\": 1674064394.747933,\n",
            "  \"global_step\": 904\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0126190185546875,\n",
            "  \"model.lr\": 0.00018010000000000001,\n",
            "  \"model.grad_norm\": 0.3113781213760376,\n",
            "  \"model.elapsed_time\": 0.0810093879699707,\n",
            "  \"model.engine_step\": 905,\n",
            "  \"model.loss.nll\": 0.0126190185546875,\n",
            "  \"elapsed_time\": 0.0810093879699707,\n",
            "  \"wall_time\": 1674064394.8365223,\n",
            "  \"global_step\": 905\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03143310546875,\n",
            "  \"model.lr\": 0.000180299,\n",
            "  \"model.grad_norm\": 0.8376282453536987,\n",
            "  \"model.elapsed_time\": 0.07315349578857422,\n",
            "  \"model.engine_step\": 906,\n",
            "  \"model.loss.nll\": 0.03143310546875,\n",
            "  \"elapsed_time\": 0.07315349578857422,\n",
            "  \"wall_time\": 1674064394.9119883,\n",
            "  \"global_step\": 906\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:14\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.028045654296875,\n",
            "  \"model.lr\": 0.00018049800000000002,\n",
            "  \"model.grad_norm\": 0.6402862071990967,\n",
            "  \"model.elapsed_time\": 0.07904052734375,\n",
            "  \"model.engine_step\": 907,\n",
            "  \"model.loss.nll\": 0.028045654296875,\n",
            "  \"elapsed_time\": 0.07904052734375,\n",
            "  \"wall_time\": 1674064394.9990594,\n",
            "  \"global_step\": 907\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.012054443359375,\n",
            "  \"model.lr\": 0.000180697,\n",
            "  \"model.grad_norm\": 0.2594743072986603,\n",
            "  \"model.elapsed_time\": 0.07578372955322266,\n",
            "  \"model.engine_step\": 908,\n",
            "  \"model.loss.nll\": 0.012054443359375,\n",
            "  \"elapsed_time\": 0.07578372955322266,\n",
            "  \"wall_time\": 1674064395.076976,\n",
            "  \"global_step\": 908\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01438140869140625,\n",
            "  \"model.lr\": 0.00018089600000000002,\n",
            "  \"model.grad_norm\": 0.39433255791664124,\n",
            "  \"model.elapsed_time\": 0.07164859771728516,\n",
            "  \"model.engine_step\": 909,\n",
            "  \"model.loss.nll\": 0.01438140869140625,\n",
            "  \"elapsed_time\": 0.07164859771728516,\n",
            "  \"wall_time\": 1674064395.156725,\n",
            "  \"global_step\": 909\n",
            "}\n",
            "[2023-01-18 17:53:15,243] [INFO] [logging.py:68:log_dist] [Rank 0] step=910, skipped=4, lr=[0.00018109500000000001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:15,245] [INFO] [timer.py:196:stop] epoch=0/micro_step=910/global_step=910, RunningAvgSamplesPerSec=12.291249806385355, CurrSamplesPerSec=11.440935066352068, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 1.2587890625,\n",
            "  \"model.lr\": 0.00018109500000000001,\n",
            "  \"model.grad_norm\": 12.937483787536621,\n",
            "  \"model.elapsed_time\": 0.08793520927429199,\n",
            "  \"model.engine_step\": 910,\n",
            "  \"model.loss.nll\": 1.2587890625,\n",
            "  \"elapsed_time\": 0.08793520927429199,\n",
            "  \"wall_time\": 1674064395.2470777,\n",
            "  \"global_step\": 910\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0286865234375,\n",
            "  \"model.lr\": 0.000181294,\n",
            "  \"model.grad_norm\": 0.6354685425758362,\n",
            "  \"model.elapsed_time\": 0.08364343643188477,\n",
            "  \"model.engine_step\": 911,\n",
            "  \"model.loss.nll\": 0.0286865234375,\n",
            "  \"elapsed_time\": 0.08364343643188477,\n",
            "  \"wall_time\": 1674064395.3386683,\n",
            "  \"global_step\": 911\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01511383056640625,\n",
            "  \"model.lr\": 0.00018149300000000002,\n",
            "  \"model.grad_norm\": 0.30001768469810486,\n",
            "  \"model.elapsed_time\": 0.079315185546875,\n",
            "  \"model.engine_step\": 912,\n",
            "  \"model.loss.nll\": 0.01511383056640625,\n",
            "  \"elapsed_time\": 0.079315185546875,\n",
            "  \"wall_time\": 1674064395.4201949,\n",
            "  \"global_step\": 912\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.009185791015625,\n",
            "  \"model.lr\": 0.000181692,\n",
            "  \"model.grad_norm\": 0.18358060717582703,\n",
            "  \"model.elapsed_time\": 0.07822751998901367,\n",
            "  \"model.engine_step\": 913,\n",
            "  \"model.loss.nll\": 0.009185791015625,\n",
            "  \"elapsed_time\": 0.07822751998901367,\n",
            "  \"wall_time\": 1674064395.506013,\n",
            "  \"global_step\": 913\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0292510986328125,\n",
            "  \"model.lr\": 0.00018189100000000002,\n",
            "  \"model.grad_norm\": 1.1952602863311768,\n",
            "  \"model.elapsed_time\": 0.07074570655822754,\n",
            "  \"model.engine_step\": 914,\n",
            "  \"model.loss.nll\": 0.0292510986328125,\n",
            "  \"elapsed_time\": 0.07074570655822754,\n",
            "  \"wall_time\": 1674064395.5787861,\n",
            "  \"global_step\": 914\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01255035400390625,\n",
            "  \"model.lr\": 0.00018209,\n",
            "  \"model.grad_norm\": 0.26823854446411133,\n",
            "  \"model.elapsed_time\": 0.08117556571960449,\n",
            "  \"model.engine_step\": 915,\n",
            "  \"model.loss.nll\": 0.01255035400390625,\n",
            "  \"elapsed_time\": 0.08117556571960449,\n",
            "  \"wall_time\": 1674064395.6744883,\n",
            "  \"global_step\": 915\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1258544921875,\n",
            "  \"model.lr\": 0.000182289,\n",
            "  \"model.grad_norm\": 3.554922580718994,\n",
            "  \"model.elapsed_time\": 0.07354092597961426,\n",
            "  \"model.engine_step\": 916,\n",
            "  \"model.loss.nll\": 0.1258544921875,\n",
            "  \"elapsed_time\": 0.07354092597961426,\n",
            "  \"wall_time\": 1674064395.7501488,\n",
            "  \"global_step\": 916\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.60546875,\n",
            "  \"model.lr\": 0.00018248800000000002,\n",
            "  \"model.grad_norm\": 6.888949394226074,\n",
            "  \"model.elapsed_time\": 0.0802617073059082,\n",
            "  \"model.engine_step\": 917,\n",
            "  \"model.loss.nll\": 0.60546875,\n",
            "  \"elapsed_time\": 0.0802617073059082,\n",
            "  \"wall_time\": 1674064395.838393,\n",
            "  \"global_step\": 917\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.10333251953125,\n",
            "  \"model.lr\": 0.000182687,\n",
            "  \"model.grad_norm\": 2.1729729175567627,\n",
            "  \"model.elapsed_time\": 0.0791940689086914,\n",
            "  \"model.engine_step\": 918,\n",
            "  \"model.loss.nll\": 0.10333251953125,\n",
            "  \"elapsed_time\": 0.0791940689086914,\n",
            "  \"wall_time\": 1674064395.9197187,\n",
            "  \"global_step\": 918\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:15\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01071929931640625,\n",
            "  \"model.lr\": 0.00018288600000000002,\n",
            "  \"model.grad_norm\": 0.2762240767478943,\n",
            "  \"model.elapsed_time\": 0.0769040584564209,\n",
            "  \"model.engine_step\": 919,\n",
            "  \"model.loss.nll\": 0.01071929931640625,\n",
            "  \"elapsed_time\": 0.0769040584564209,\n",
            "  \"wall_time\": 1674064396.0036125,\n",
            "  \"global_step\": 919\n",
            "}\n",
            "[2023-01-18 17:53:16,074] [INFO] [logging.py:68:log_dist] [Rank 0] step=920, skipped=4, lr=[0.000183085], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:16,076] [INFO] [timer.py:196:stop] epoch=0/micro_step=920/global_step=920, RunningAvgSamplesPerSec=12.297898477006385, CurrSamplesPerSec=14.077443823524476, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.015228271484375,\n",
            "  \"model.lr\": 0.000183085,\n",
            "  \"model.grad_norm\": 0.35778510570526123,\n",
            "  \"model.elapsed_time\": 0.07151579856872559,\n",
            "  \"model.engine_step\": 920,\n",
            "  \"model.loss.nll\": 0.015228271484375,\n",
            "  \"elapsed_time\": 0.07151579856872559,\n",
            "  \"wall_time\": 1674064396.0778787,\n",
            "  \"global_step\": 920\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0167388916015625,\n",
            "  \"model.lr\": 0.000183284,\n",
            "  \"model.grad_norm\": 0.41336485743522644,\n",
            "  \"model.elapsed_time\": 0.08018779754638672,\n",
            "  \"model.engine_step\": 921,\n",
            "  \"model.loss.nll\": 0.0167388916015625,\n",
            "  \"elapsed_time\": 0.08018779754638672,\n",
            "  \"wall_time\": 1674064396.1668055,\n",
            "  \"global_step\": 921\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.86572265625,\n",
            "  \"model.lr\": 0.00018348300000000002,\n",
            "  \"model.grad_norm\": 11.264225006103516,\n",
            "  \"model.elapsed_time\": 0.07376980781555176,\n",
            "  \"model.engine_step\": 922,\n",
            "  \"model.loss.nll\": 0.86572265625,\n",
            "  \"elapsed_time\": 0.07376980781555176,\n",
            "  \"wall_time\": 1674064396.2425704,\n",
            "  \"global_step\": 922\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.031494140625,\n",
            "  \"model.lr\": 0.000183682,\n",
            "  \"model.grad_norm\": 0.81999671459198,\n",
            "  \"model.elapsed_time\": 0.08672189712524414,\n",
            "  \"model.engine_step\": 923,\n",
            "  \"model.loss.nll\": 0.031494140625,\n",
            "  \"elapsed_time\": 0.08672189712524414,\n",
            "  \"wall_time\": 1674064396.3375137,\n",
            "  \"global_step\": 923\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0226593017578125,\n",
            "  \"model.lr\": 0.00018388100000000002,\n",
            "  \"model.grad_norm\": 0.7912154197692871,\n",
            "  \"model.elapsed_time\": 0.07787227630615234,\n",
            "  \"model.engine_step\": 924,\n",
            "  \"model.loss.nll\": 0.0226593017578125,\n",
            "  \"elapsed_time\": 0.07787227630615234,\n",
            "  \"wall_time\": 1674064396.4184277,\n",
            "  \"global_step\": 924\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.060638427734375,\n",
            "  \"model.lr\": 0.00018408,\n",
            "  \"model.grad_norm\": 1.6276577711105347,\n",
            "  \"model.elapsed_time\": 0.0718989372253418,\n",
            "  \"model.engine_step\": 925,\n",
            "  \"model.loss.nll\": 0.060638427734375,\n",
            "  \"elapsed_time\": 0.0718989372253418,\n",
            "  \"wall_time\": 1674064396.498872,\n",
            "  \"global_step\": 925\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.46484375,\n",
            "  \"model.lr\": 0.00018427900000000003,\n",
            "  \"model.grad_norm\": 6.291374683380127,\n",
            "  \"model.elapsed_time\": 0.07738733291625977,\n",
            "  \"model.engine_step\": 926,\n",
            "  \"model.loss.nll\": 0.46484375,\n",
            "  \"elapsed_time\": 0.07738733291625977,\n",
            "  \"wall_time\": 1674064396.578292,\n",
            "  \"global_step\": 926\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.27099609375,\n",
            "  \"model.lr\": 0.00018447800000000002,\n",
            "  \"model.grad_norm\": 4.05396842956543,\n",
            "  \"model.elapsed_time\": 0.07520008087158203,\n",
            "  \"model.engine_step\": 927,\n",
            "  \"model.loss.nll\": 0.27099609375,\n",
            "  \"elapsed_time\": 0.07520008087158203,\n",
            "  \"wall_time\": 1674064396.6606429,\n",
            "  \"global_step\": 927\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.09307861328125,\n",
            "  \"model.lr\": 0.000184677,\n",
            "  \"model.grad_norm\": 2.6585066318511963,\n",
            "  \"model.elapsed_time\": 0.07899332046508789,\n",
            "  \"model.engine_step\": 928,\n",
            "  \"model.loss.nll\": 0.09307861328125,\n",
            "  \"elapsed_time\": 0.07899332046508789,\n",
            "  \"wall_time\": 1674064396.7423859,\n",
            "  \"global_step\": 928\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.048980712890625,\n",
            "  \"model.lr\": 0.00018487600000000002,\n",
            "  \"model.grad_norm\": 1.3091862201690674,\n",
            "  \"model.elapsed_time\": 0.07100081443786621,\n",
            "  \"model.engine_step\": 929,\n",
            "  \"model.loss.nll\": 0.048980712890625,\n",
            "  \"elapsed_time\": 0.07100081443786621,\n",
            "  \"wall_time\": 1674064396.821072,\n",
            "  \"global_step\": 929\n",
            "}\n",
            "[2023-01-18 17:53:16,896] [INFO] [logging.py:68:log_dist] [Rank 0] step=930, skipped=4, lr=[0.000185075], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:16,898] [INFO] [timer.py:196:stop] epoch=0/micro_step=930/global_step=930, RunningAvgSamplesPerSec=12.305222886017999, CurrSamplesPerSec=13.141655778744896, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.049713134765625,\n",
            "  \"model.lr\": 0.000185075,\n",
            "  \"model.grad_norm\": 1.5176061391830444,\n",
            "  \"model.elapsed_time\": 0.07655835151672363,\n",
            "  \"model.engine_step\": 930,\n",
            "  \"model.loss.nll\": 0.049713134765625,\n",
            "  \"elapsed_time\": 0.07655835151672363,\n",
            "  \"wall_time\": 1674064396.8997612,\n",
            "  \"global_step\": 930\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:16\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01708984375,\n",
            "  \"model.lr\": 0.00018527400000000003,\n",
            "  \"model.grad_norm\": 0.35422593355178833,\n",
            "  \"model.elapsed_time\": 0.08128595352172852,\n",
            "  \"model.engine_step\": 931,\n",
            "  \"model.loss.nll\": 0.01708984375,\n",
            "  \"elapsed_time\": 0.08128595352172852,\n",
            "  \"wall_time\": 1674064396.9884944,\n",
            "  \"global_step\": 931\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.61865234375,\n",
            "  \"model.lr\": 0.00018547300000000002,\n",
            "  \"model.grad_norm\": 8.751906394958496,\n",
            "  \"model.elapsed_time\": 0.07130074501037598,\n",
            "  \"model.engine_step\": 932,\n",
            "  \"model.loss.nll\": 0.61865234375,\n",
            "  \"elapsed_time\": 0.07130074501037598,\n",
            "  \"wall_time\": 1674064397.0623856,\n",
            "  \"global_step\": 932\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2174072265625,\n",
            "  \"model.lr\": 0.000185672,\n",
            "  \"model.grad_norm\": 5.112551212310791,\n",
            "  \"model.elapsed_time\": 0.08222293853759766,\n",
            "  \"model.engine_step\": 933,\n",
            "  \"model.loss.nll\": 0.2174072265625,\n",
            "  \"elapsed_time\": 0.08222293853759766,\n",
            "  \"wall_time\": 1674064397.1515884,\n",
            "  \"global_step\": 933\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0199127197265625,\n",
            "  \"model.lr\": 0.00018587100000000002,\n",
            "  \"model.grad_norm\": 0.547141432762146,\n",
            "  \"model.elapsed_time\": 0.07066154479980469,\n",
            "  \"model.engine_step\": 934,\n",
            "  \"model.loss.nll\": 0.0199127197265625,\n",
            "  \"elapsed_time\": 0.07066154479980469,\n",
            "  \"wall_time\": 1674064397.224906,\n",
            "  \"global_step\": 934\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01995849609375,\n",
            "  \"model.lr\": 0.00018607,\n",
            "  \"model.grad_norm\": 0.5646399855613708,\n",
            "  \"model.elapsed_time\": 0.08225655555725098,\n",
            "  \"model.engine_step\": 935,\n",
            "  \"model.loss.nll\": 0.01995849609375,\n",
            "  \"elapsed_time\": 0.08225655555725098,\n",
            "  \"wall_time\": 1674064397.3145666,\n",
            "  \"global_step\": 935\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.11273193359375,\n",
            "  \"model.lr\": 0.00018626900000000003,\n",
            "  \"model.grad_norm\": 2.7921721935272217,\n",
            "  \"model.elapsed_time\": 0.08426237106323242,\n",
            "  \"model.engine_step\": 936,\n",
            "  \"model.loss.nll\": 0.11273193359375,\n",
            "  \"elapsed_time\": 0.08426237106323242,\n",
            "  \"wall_time\": 1674064397.4015155,\n",
            "  \"global_step\": 936\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.65625,\n",
            "  \"model.lr\": 0.00018646800000000002,\n",
            "  \"model.grad_norm\": 9.147502899169922,\n",
            "  \"model.elapsed_time\": 0.08330464363098145,\n",
            "  \"model.engine_step\": 937,\n",
            "  \"model.loss.nll\": 0.65625,\n",
            "  \"elapsed_time\": 0.08330464363098145,\n",
            "  \"wall_time\": 1674064397.4918182,\n",
            "  \"global_step\": 937\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.05694580078125,\n",
            "  \"model.lr\": 0.000186667,\n",
            "  \"model.grad_norm\": 1.854041576385498,\n",
            "  \"model.elapsed_time\": 0.07161808013916016,\n",
            "  \"model.engine_step\": 938,\n",
            "  \"model.loss.nll\": 0.05694580078125,\n",
            "  \"elapsed_time\": 0.07161808013916016,\n",
            "  \"wall_time\": 1674064397.566383,\n",
            "  \"global_step\": 938\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.048095703125,\n",
            "  \"model.lr\": 0.00018686600000000002,\n",
            "  \"model.grad_norm\": 1.4140881299972534,\n",
            "  \"model.elapsed_time\": 0.07929587364196777,\n",
            "  \"model.engine_step\": 939,\n",
            "  \"model.loss.nll\": 0.048095703125,\n",
            "  \"elapsed_time\": 0.07929587364196777,\n",
            "  \"wall_time\": 1674064397.6532109,\n",
            "  \"global_step\": 939\n",
            "}\n",
            "[2023-01-18 17:53:17,724] [INFO] [logging.py:68:log_dist] [Rank 0] step=940, skipped=4, lr=[0.000187065], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:17,726] [INFO] [timer.py:196:stop] epoch=0/micro_step=940/global_step=940, RunningAvgSamplesPerSec=12.3110261376918, CurrSamplesPerSec=14.017177651674654, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0128326416015625,\n",
            "  \"model.lr\": 0.000187065,\n",
            "  \"model.grad_norm\": 0.34780311584472656,\n",
            "  \"model.elapsed_time\": 0.07177329063415527,\n",
            "  \"model.engine_step\": 940,\n",
            "  \"model.loss.nll\": 0.0128326416015625,\n",
            "  \"elapsed_time\": 0.07177329063415527,\n",
            "  \"wall_time\": 1674064397.7274837,\n",
            "  \"global_step\": 940\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021942138671875,\n",
            "  \"model.lr\": 0.00018726400000000003,\n",
            "  \"model.grad_norm\": 0.5107898116111755,\n",
            "  \"model.elapsed_time\": 0.07142400741577148,\n",
            "  \"model.engine_step\": 941,\n",
            "  \"model.loss.nll\": 0.021942138671875,\n",
            "  \"elapsed_time\": 0.07142400741577148,\n",
            "  \"wall_time\": 1674064397.8065424,\n",
            "  \"global_step\": 941\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01551055908203125,\n",
            "  \"model.lr\": 0.00018746300000000002,\n",
            "  \"model.grad_norm\": 0.42693352699279785,\n",
            "  \"model.elapsed_time\": 0.07783246040344238,\n",
            "  \"model.engine_step\": 942,\n",
            "  \"model.loss.nll\": 0.01551055908203125,\n",
            "  \"elapsed_time\": 0.07783246040344238,\n",
            "  \"wall_time\": 1674064397.8868992,\n",
            "  \"global_step\": 942\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:17\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0267791748046875,\n",
            "  \"model.lr\": 0.000187662,\n",
            "  \"model.grad_norm\": 0.6601807475090027,\n",
            "  \"model.elapsed_time\": 0.08254003524780273,\n",
            "  \"model.engine_step\": 943,\n",
            "  \"model.loss.nll\": 0.0267791748046875,\n",
            "  \"elapsed_time\": 0.08254003524780273,\n",
            "  \"wall_time\": 1674064397.9776354,\n",
            "  \"global_step\": 943\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.023468017578125,\n",
            "  \"model.lr\": 0.000187861,\n",
            "  \"model.grad_norm\": 0.6605436205863953,\n",
            "  \"model.elapsed_time\": 0.0795600414276123,\n",
            "  \"model.engine_step\": 944,\n",
            "  \"model.loss.nll\": 0.023468017578125,\n",
            "  \"elapsed_time\": 0.0795600414276123,\n",
            "  \"wall_time\": 1674064398.0593688,\n",
            "  \"global_step\": 944\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.024261474609375,\n",
            "  \"model.lr\": 0.00018805999999999998,\n",
            "  \"model.grad_norm\": 0.5553434491157532,\n",
            "  \"model.elapsed_time\": 0.07619285583496094,\n",
            "  \"model.engine_step\": 945,\n",
            "  \"model.loss.nll\": 0.024261474609375,\n",
            "  \"elapsed_time\": 0.07619285583496094,\n",
            "  \"wall_time\": 1674064398.1466806,\n",
            "  \"global_step\": 945\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.092529296875,\n",
            "  \"model.lr\": 0.000188259,\n",
            "  \"model.grad_norm\": 2.787311315536499,\n",
            "  \"model.elapsed_time\": 0.07128000259399414,\n",
            "  \"model.engine_step\": 946,\n",
            "  \"model.loss.nll\": 0.092529296875,\n",
            "  \"elapsed_time\": 0.07128000259399414,\n",
            "  \"wall_time\": 1674064398.2201293,\n",
            "  \"global_step\": 946\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1015625,\n",
            "  \"model.lr\": 0.000188458,\n",
            "  \"model.grad_norm\": 3.0624189376831055,\n",
            "  \"model.elapsed_time\": 0.07605910301208496,\n",
            "  \"model.engine_step\": 947,\n",
            "  \"model.loss.nll\": 0.1015625,\n",
            "  \"elapsed_time\": 0.07605910301208496,\n",
            "  \"wall_time\": 1674064398.3031187,\n",
            "  \"global_step\": 947\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.044158935546875,\n",
            "  \"model.lr\": 0.000188657,\n",
            "  \"model.grad_norm\": 1.4945313930511475,\n",
            "  \"model.elapsed_time\": 0.0896139144897461,\n",
            "  \"model.engine_step\": 948,\n",
            "  \"model.loss.nll\": 0.044158935546875,\n",
            "  \"elapsed_time\": 0.0896139144897461,\n",
            "  \"wall_time\": 1674064398.3955398,\n",
            "  \"global_step\": 948\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.014190673828125,\n",
            "  \"model.lr\": 0.000188856,\n",
            "  \"model.grad_norm\": 0.348312109708786,\n",
            "  \"model.elapsed_time\": 0.07908177375793457,\n",
            "  \"model.engine_step\": 949,\n",
            "  \"model.loss.nll\": 0.014190673828125,\n",
            "  \"elapsed_time\": 0.07908177375793457,\n",
            "  \"wall_time\": 1674064398.4821546,\n",
            "  \"global_step\": 949\n",
            "}\n",
            "[2023-01-18 17:53:18,559] [INFO] [logging.py:68:log_dist] [Rank 0] step=950, skipped=4, lr=[0.000189055], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:18,562] [INFO] [timer.py:196:stop] epoch=0/micro_step=950/global_step=950, RunningAvgSamplesPerSec=12.315972112715428, CurrSamplesPerSec=12.728140078293327, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06939697265625,\n",
            "  \"model.lr\": 0.000189055,\n",
            "  \"model.grad_norm\": 2.0948965549468994,\n",
            "  \"model.elapsed_time\": 0.07903718948364258,\n",
            "  \"model.engine_step\": 950,\n",
            "  \"model.loss.nll\": 0.06939697265625,\n",
            "  \"elapsed_time\": 0.07903718948364258,\n",
            "  \"wall_time\": 1674064398.563252,\n",
            "  \"global_step\": 950\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.014892578125,\n",
            "  \"model.lr\": 0.000189254,\n",
            "  \"model.grad_norm\": 0.3672940135002136,\n",
            "  \"model.elapsed_time\": 0.07135915756225586,\n",
            "  \"model.engine_step\": 951,\n",
            "  \"model.loss.nll\": 0.014892578125,\n",
            "  \"elapsed_time\": 0.07135915756225586,\n",
            "  \"wall_time\": 1674064398.6428142,\n",
            "  \"global_step\": 951\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.018402099609375,\n",
            "  \"model.lr\": 0.000189453,\n",
            "  \"model.grad_norm\": 0.40288710594177246,\n",
            "  \"model.elapsed_time\": 0.073760986328125,\n",
            "  \"model.engine_step\": 952,\n",
            "  \"model.loss.nll\": 0.018402099609375,\n",
            "  \"elapsed_time\": 0.073760986328125,\n",
            "  \"wall_time\": 1674064398.7186906,\n",
            "  \"global_step\": 952\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0213165283203125,\n",
            "  \"model.lr\": 0.000189652,\n",
            "  \"model.grad_norm\": 0.6264263987541199,\n",
            "  \"model.elapsed_time\": 0.08704471588134766,\n",
            "  \"model.engine_step\": 953,\n",
            "  \"model.loss.nll\": 0.0213165283203125,\n",
            "  \"elapsed_time\": 0.08704471588134766,\n",
            "  \"wall_time\": 1674064398.8128886,\n",
            "  \"global_step\": 953\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.041229248046875,\n",
            "  \"model.lr\": 0.000189851,\n",
            "  \"model.grad_norm\": 1.974035620689392,\n",
            "  \"model.elapsed_time\": 0.07265424728393555,\n",
            "  \"model.engine_step\": 954,\n",
            "  \"model.loss.nll\": 0.041229248046875,\n",
            "  \"elapsed_time\": 0.07265424728393555,\n",
            "  \"wall_time\": 1674064398.8882668,\n",
            "  \"global_step\": 954\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:18\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0185089111328125,\n",
            "  \"model.lr\": 0.00019005,\n",
            "  \"model.grad_norm\": 0.4002359211444855,\n",
            "  \"model.elapsed_time\": 0.07319974899291992,\n",
            "  \"model.engine_step\": 955,\n",
            "  \"model.loss.nll\": 0.0185089111328125,\n",
            "  \"elapsed_time\": 0.07319974899291992,\n",
            "  \"wall_time\": 1674064398.9697535,\n",
            "  \"global_step\": 955\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.94091796875,\n",
            "  \"model.lr\": 0.000190249,\n",
            "  \"model.grad_norm\": 9.48753547668457,\n",
            "  \"model.elapsed_time\": 0.07850170135498047,\n",
            "  \"model.engine_step\": 956,\n",
            "  \"model.loss.nll\": 0.94091796875,\n",
            "  \"elapsed_time\": 0.07850170135498047,\n",
            "  \"wall_time\": 1674064399.0507407,\n",
            "  \"global_step\": 956\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.443359375,\n",
            "  \"model.lr\": 0.000190448,\n",
            "  \"model.grad_norm\": 5.7709431648254395,\n",
            "  \"model.elapsed_time\": 0.06956362724304199,\n",
            "  \"model.engine_step\": 957,\n",
            "  \"model.loss.nll\": 0.443359375,\n",
            "  \"elapsed_time\": 0.06956362724304199,\n",
            "  \"wall_time\": 1674064399.131515,\n",
            "  \"global_step\": 957\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06475830078125,\n",
            "  \"model.lr\": 0.000190647,\n",
            "  \"model.grad_norm\": 1.9415948390960693,\n",
            "  \"model.elapsed_time\": 0.06970024108886719,\n",
            "  \"model.engine_step\": 958,\n",
            "  \"model.loss.nll\": 0.06475830078125,\n",
            "  \"elapsed_time\": 0.06970024108886719,\n",
            "  \"wall_time\": 1674064399.2037673,\n",
            "  \"global_step\": 958\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.055511474609375,\n",
            "  \"model.lr\": 0.000190846,\n",
            "  \"model.grad_norm\": 1.6315052509307861,\n",
            "  \"model.elapsed_time\": 0.07969069480895996,\n",
            "  \"model.engine_step\": 959,\n",
            "  \"model.loss.nll\": 0.055511474609375,\n",
            "  \"elapsed_time\": 0.07969069480895996,\n",
            "  \"wall_time\": 1674064399.2903268,\n",
            "  \"global_step\": 959\n",
            "}\n",
            "[2023-01-18 17:53:19,365] [INFO] [logging.py:68:log_dist] [Rank 0] step=960, skipped=4, lr=[0.000191045], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:19,367] [INFO] [timer.py:196:stop] epoch=0/micro_step=960/global_step=960, RunningAvgSamplesPerSec=12.325784126132659, CurrSamplesPerSec=13.259351053337042, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0183563232421875,\n",
            "  \"model.lr\": 0.000191045,\n",
            "  \"model.grad_norm\": 0.4804770052433014,\n",
            "  \"model.elapsed_time\": 0.07586812973022461,\n",
            "  \"model.engine_step\": 960,\n",
            "  \"model.loss.nll\": 0.0183563232421875,\n",
            "  \"elapsed_time\": 0.07586812973022461,\n",
            "  \"wall_time\": 1674064399.3689756,\n",
            "  \"global_step\": 960\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01277923583984375,\n",
            "  \"model.lr\": 0.000191244,\n",
            "  \"model.grad_norm\": 0.30971211194992065,\n",
            "  \"model.elapsed_time\": 0.08650708198547363,\n",
            "  \"model.engine_step\": 961,\n",
            "  \"model.loss.nll\": 0.01277923583984375,\n",
            "  \"elapsed_time\": 0.08650708198547363,\n",
            "  \"wall_time\": 1674064399.4632034,\n",
            "  \"global_step\": 961\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.95556640625,\n",
            "  \"model.lr\": 0.000191443,\n",
            "  \"model.grad_norm\": 10.46560001373291,\n",
            "  \"model.elapsed_time\": 0.07952356338500977,\n",
            "  \"model.engine_step\": 962,\n",
            "  \"model.loss.nll\": 0.95556640625,\n",
            "  \"elapsed_time\": 0.07952356338500977,\n",
            "  \"wall_time\": 1674064399.5451,\n",
            "  \"global_step\": 962\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.071533203125,\n",
            "  \"model.lr\": 0.000191642,\n",
            "  \"model.grad_norm\": 2.1313323974609375,\n",
            "  \"model.elapsed_time\": 0.08290743827819824,\n",
            "  \"model.engine_step\": 963,\n",
            "  \"model.loss.nll\": 0.071533203125,\n",
            "  \"elapsed_time\": 0.08290743827819824,\n",
            "  \"wall_time\": 1674064399.6357963,\n",
            "  \"global_step\": 963\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0147857666015625,\n",
            "  \"model.lr\": 0.000191841,\n",
            "  \"model.grad_norm\": 0.5435407757759094,\n",
            "  \"model.elapsed_time\": 0.07586097717285156,\n",
            "  \"model.engine_step\": 964,\n",
            "  \"model.loss.nll\": 0.0147857666015625,\n",
            "  \"elapsed_time\": 0.07586097717285156,\n",
            "  \"wall_time\": 1674064399.7149425,\n",
            "  \"global_step\": 964\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01324462890625,\n",
            "  \"model.lr\": 0.00019204,\n",
            "  \"model.grad_norm\": 0.26520150899887085,\n",
            "  \"model.elapsed_time\": 0.0807640552520752,\n",
            "  \"model.engine_step\": 965,\n",
            "  \"model.loss.nll\": 0.01324462890625,\n",
            "  \"elapsed_time\": 0.0807640552520752,\n",
            "  \"wall_time\": 1674064399.8034434,\n",
            "  \"global_step\": 965\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02154541015625,\n",
            "  \"model.lr\": 0.000192239,\n",
            "  \"model.grad_norm\": 0.623584508895874,\n",
            "  \"model.elapsed_time\": 0.07267379760742188,\n",
            "  \"model.engine_step\": 966,\n",
            "  \"model.loss.nll\": 0.02154541015625,\n",
            "  \"elapsed_time\": 0.07267379760742188,\n",
            "  \"wall_time\": 1674064399.8788896,\n",
            "  \"global_step\": 966\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:19\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04742431640625,\n",
            "  \"model.lr\": 0.000192438,\n",
            "  \"model.grad_norm\": 1.2372119426727295,\n",
            "  \"model.elapsed_time\": 0.08281421661376953,\n",
            "  \"model.engine_step\": 967,\n",
            "  \"model.loss.nll\": 0.04742431640625,\n",
            "  \"elapsed_time\": 0.08281421661376953,\n",
            "  \"wall_time\": 1674064399.9694197,\n",
            "  \"global_step\": 967\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0203399658203125,\n",
            "  \"model.lr\": 0.000192637,\n",
            "  \"model.grad_norm\": 0.6022152304649353,\n",
            "  \"model.elapsed_time\": 0.07744193077087402,\n",
            "  \"model.engine_step\": 968,\n",
            "  \"model.loss.nll\": 0.0203399658203125,\n",
            "  \"elapsed_time\": 0.07744193077087402,\n",
            "  \"wall_time\": 1674064400.0496688,\n",
            "  \"global_step\": 968\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0692138671875,\n",
            "  \"model.lr\": 0.000192836,\n",
            "  \"model.grad_norm\": 1.491106629371643,\n",
            "  \"model.elapsed_time\": 0.08577203750610352,\n",
            "  \"model.engine_step\": 969,\n",
            "  \"model.loss.nll\": 0.0692138671875,\n",
            "  \"elapsed_time\": 0.08577203750610352,\n",
            "  \"wall_time\": 1674064400.1439514,\n",
            "  \"global_step\": 969\n",
            "}\n",
            "[2023-01-18 17:53:20,219] [INFO] [logging.py:68:log_dist] [Rank 0] step=970, skipped=4, lr=[0.000193035], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:20,221] [INFO] [timer.py:196:stop] epoch=0/micro_step=970/global_step=970, RunningAvgSamplesPerSec=12.327580878006186, CurrSamplesPerSec=13.06714104573168, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.06982421875,\n",
            "  \"model.lr\": 0.000193035,\n",
            "  \"model.grad_norm\": 1.6644556522369385,\n",
            "  \"model.elapsed_time\": 0.07699704170227051,\n",
            "  \"model.engine_step\": 970,\n",
            "  \"model.loss.nll\": 0.06982421875,\n",
            "  \"elapsed_time\": 0.07699704170227051,\n",
            "  \"wall_time\": 1674064400.2231512,\n",
            "  \"global_step\": 970\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0172119140625,\n",
            "  \"model.lr\": 0.000193234,\n",
            "  \"model.grad_norm\": 0.4933878481388092,\n",
            "  \"model.elapsed_time\": 0.07471060752868652,\n",
            "  \"model.engine_step\": 971,\n",
            "  \"model.loss.nll\": 0.0172119140625,\n",
            "  \"elapsed_time\": 0.07471060752868652,\n",
            "  \"wall_time\": 1674064400.3051944,\n",
            "  \"global_step\": 971\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.033538818359375,\n",
            "  \"model.lr\": 0.00019343300000000002,\n",
            "  \"model.grad_norm\": 0.7379509210586548,\n",
            "  \"model.elapsed_time\": 0.08121132850646973,\n",
            "  \"model.engine_step\": 972,\n",
            "  \"model.loss.nll\": 0.033538818359375,\n",
            "  \"elapsed_time\": 0.08121132850646973,\n",
            "  \"wall_time\": 1674064400.3884938,\n",
            "  \"global_step\": 972\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0166168212890625,\n",
            "  \"model.lr\": 0.000193632,\n",
            "  \"model.grad_norm\": 0.43559545278549194,\n",
            "  \"model.elapsed_time\": 0.0863947868347168,\n",
            "  \"model.engine_step\": 973,\n",
            "  \"model.loss.nll\": 0.0166168212890625,\n",
            "  \"elapsed_time\": 0.0863947868347168,\n",
            "  \"wall_time\": 1674064400.4830725,\n",
            "  \"global_step\": 973\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0301361083984375,\n",
            "  \"model.lr\": 0.000193831,\n",
            "  \"model.grad_norm\": 0.8700236678123474,\n",
            "  \"model.elapsed_time\": 0.08035659790039062,\n",
            "  \"model.engine_step\": 974,\n",
            "  \"model.loss.nll\": 0.0301361083984375,\n",
            "  \"elapsed_time\": 0.08035659790039062,\n",
            "  \"wall_time\": 1674064400.5655904,\n",
            "  \"global_step\": 974\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.01776123046875,\n",
            "  \"model.lr\": 0.00019403,\n",
            "  \"model.grad_norm\": 0.4094260036945343,\n",
            "  \"model.elapsed_time\": 0.08460164070129395,\n",
            "  \"model.engine_step\": 975,\n",
            "  \"model.loss.nll\": 0.01776123046875,\n",
            "  \"elapsed_time\": 0.08460164070129395,\n",
            "  \"wall_time\": 1674064400.6600327,\n",
            "  \"global_step\": 975\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.6474609375,\n",
            "  \"model.lr\": 0.000194229,\n",
            "  \"model.grad_norm\": 9.355820655822754,\n",
            "  \"model.elapsed_time\": 0.07473897933959961,\n",
            "  \"model.engine_step\": 976,\n",
            "  \"model.loss.nll\": 0.6474609375,\n",
            "  \"elapsed_time\": 0.07473897933959961,\n",
            "  \"wall_time\": 1674064400.7382166,\n",
            "  \"global_step\": 976\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.251953125,\n",
            "  \"model.lr\": 0.00019442800000000002,\n",
            "  \"model.grad_norm\": 4.397011756896973,\n",
            "  \"model.elapsed_time\": 0.09330320358276367,\n",
            "  \"model.engine_step\": 977,\n",
            "  \"model.loss.nll\": 0.251953125,\n",
            "  \"elapsed_time\": 0.09330320358276367,\n",
            "  \"wall_time\": 1674064400.8435223,\n",
            "  \"global_step\": 977\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02716064453125,\n",
            "  \"model.lr\": 0.000194627,\n",
            "  \"model.grad_norm\": 0.7199533581733704,\n",
            "  \"model.elapsed_time\": 0.07663893699645996,\n",
            "  \"model.engine_step\": 978,\n",
            "  \"model.loss.nll\": 0.02716064453125,\n",
            "  \"elapsed_time\": 0.07663893699645996,\n",
            "  \"wall_time\": 1674064400.9229598,\n",
            "  \"global_step\": 978\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:20\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.7587890625,\n",
            "  \"model.lr\": 0.000194826,\n",
            "  \"model.grad_norm\": 10.467231750488281,\n",
            "  \"model.elapsed_time\": 0.07372450828552246,\n",
            "  \"model.engine_step\": 979,\n",
            "  \"model.loss.nll\": 0.7587890625,\n",
            "  \"elapsed_time\": 0.07372450828552246,\n",
            "  \"wall_time\": 1674064401.003287,\n",
            "  \"global_step\": 979\n",
            "}\n",
            "[2023-01-18 17:53:21,077] [INFO] [logging.py:68:log_dist] [Rank 0] step=980, skipped=4, lr=[0.000195025], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:21,078] [INFO] [timer.py:196:stop] epoch=0/micro_step=980/global_step=980, RunningAvgSamplesPerSec=12.329672626987808, CurrSamplesPerSec=13.708264562749822, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.035980224609375,\n",
            "  \"model.lr\": 0.000195025,\n",
            "  \"model.grad_norm\": 1.213451623916626,\n",
            "  \"model.elapsed_time\": 0.07367420196533203,\n",
            "  \"model.engine_step\": 980,\n",
            "  \"model.loss.nll\": 0.035980224609375,\n",
            "  \"elapsed_time\": 0.07367420196533203,\n",
            "  \"wall_time\": 1674064401.0802534,\n",
            "  \"global_step\": 980\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.04443359375,\n",
            "  \"model.lr\": 0.000195224,\n",
            "  \"model.grad_norm\": 1.5079959630966187,\n",
            "  \"model.elapsed_time\": 0.07568120956420898,\n",
            "  \"model.engine_step\": 981,\n",
            "  \"model.loss.nll\": 0.04443359375,\n",
            "  \"elapsed_time\": 0.07568120956420898,\n",
            "  \"wall_time\": 1674064401.1652439,\n",
            "  \"global_step\": 981\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.046142578125,\n",
            "  \"model.lr\": 0.00019542300000000002,\n",
            "  \"model.grad_norm\": 1.5367953777313232,\n",
            "  \"model.elapsed_time\": 0.07204055786132812,\n",
            "  \"model.engine_step\": 982,\n",
            "  \"model.loss.nll\": 0.046142578125,\n",
            "  \"elapsed_time\": 0.07204055786132812,\n",
            "  \"wall_time\": 1674064401.2394903,\n",
            "  \"global_step\": 982\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.054290771484375,\n",
            "  \"model.lr\": 0.000195622,\n",
            "  \"model.grad_norm\": 1.7384968996047974,\n",
            "  \"model.elapsed_time\": 0.08005356788635254,\n",
            "  \"model.engine_step\": 983,\n",
            "  \"model.loss.nll\": 0.054290771484375,\n",
            "  \"elapsed_time\": 0.08005356788635254,\n",
            "  \"wall_time\": 1674064401.3279397,\n",
            "  \"global_step\": 983\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.03057861328125,\n",
            "  \"model.lr\": 0.000195821,\n",
            "  \"model.grad_norm\": 1.039798378944397,\n",
            "  \"model.elapsed_time\": 0.07921624183654785,\n",
            "  \"model.engine_step\": 984,\n",
            "  \"model.loss.nll\": 0.03057861328125,\n",
            "  \"elapsed_time\": 0.07921624183654785,\n",
            "  \"wall_time\": 1674064401.4093752,\n",
            "  \"global_step\": 984\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0257568359375,\n",
            "  \"model.lr\": 0.00019602,\n",
            "  \"model.grad_norm\": 0.9244012236595154,\n",
            "  \"model.elapsed_time\": 0.08572101593017578,\n",
            "  \"model.engine_step\": 985,\n",
            "  \"model.loss.nll\": 0.0257568359375,\n",
            "  \"elapsed_time\": 0.08572101593017578,\n",
            "  \"wall_time\": 1674064401.5033226,\n",
            "  \"global_step\": 985\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0247802734375,\n",
            "  \"model.lr\": 0.000196219,\n",
            "  \"model.grad_norm\": 0.7546364068984985,\n",
            "  \"model.elapsed_time\": 0.08432245254516602,\n",
            "  \"model.engine_step\": 986,\n",
            "  \"model.loss.nll\": 0.0247802734375,\n",
            "  \"elapsed_time\": 0.08432245254516602,\n",
            "  \"wall_time\": 1674064401.5910501,\n",
            "  \"global_step\": 986\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0242462158203125,\n",
            "  \"model.lr\": 0.00019641800000000002,\n",
            "  \"model.grad_norm\": 0.6621206998825073,\n",
            "  \"model.elapsed_time\": 0.0760650634765625,\n",
            "  \"model.engine_step\": 987,\n",
            "  \"model.loss.nll\": 0.0242462158203125,\n",
            "  \"elapsed_time\": 0.0760650634765625,\n",
            "  \"wall_time\": 1674064401.6767771,\n",
            "  \"global_step\": 987\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.021820068359375,\n",
            "  \"model.lr\": 0.000196617,\n",
            "  \"model.grad_norm\": 0.5575302243232727,\n",
            "  \"model.elapsed_time\": 0.07322454452514648,\n",
            "  \"model.engine_step\": 988,\n",
            "  \"model.loss.nll\": 0.021820068359375,\n",
            "  \"elapsed_time\": 0.07322454452514648,\n",
            "  \"wall_time\": 1674064401.752713,\n",
            "  \"global_step\": 988\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.02197265625,\n",
            "  \"model.lr\": 0.000196816,\n",
            "  \"model.grad_norm\": 0.5494257807731628,\n",
            "  \"model.elapsed_time\": 0.07581591606140137,\n",
            "  \"model.engine_step\": 989,\n",
            "  \"model.loss.nll\": 0.02197265625,\n",
            "  \"elapsed_time\": 0.07581591606140137,\n",
            "  \"wall_time\": 1674064401.8361163,\n",
            "  \"global_step\": 989\n",
            "}\n",
            "[2023-01-18 17:53:21,915] [INFO] [logging.py:68:log_dist] [Rank 0] step=990, skipped=4, lr=[0.000197015], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:21,917] [INFO] [timer.py:196:stop] epoch=0/micro_step=990/global_step=990, RunningAvgSamplesPerSec=12.33433837828204, CurrSamplesPerSec=12.585643084549346, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0294342041015625,\n",
            "  \"model.lr\": 0.000197015,\n",
            "  \"model.grad_norm\": 0.8489938378334045,\n",
            "  \"model.elapsed_time\": 0.0799713134765625,\n",
            "  \"model.engine_step\": 990,\n",
            "  \"model.loss.nll\": 0.0294342041015625,\n",
            "  \"elapsed_time\": 0.0799713134765625,\n",
            "  \"wall_time\": 1674064401.9189465,\n",
            "  \"global_step\": 990\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:21\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.037841796875,\n",
            "  \"model.lr\": 0.000197214,\n",
            "  \"model.grad_norm\": 1.1762683391571045,\n",
            "  \"model.elapsed_time\": 0.07880663871765137,\n",
            "  \"model.engine_step\": 991,\n",
            "  \"model.loss.nll\": 0.037841796875,\n",
            "  \"elapsed_time\": 0.07880663871765137,\n",
            "  \"wall_time\": 1674064402.00547,\n",
            "  \"global_step\": 991\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0193328857421875,\n",
            "  \"model.lr\": 0.00019741300000000002,\n",
            "  \"model.grad_norm\": 0.8260285258293152,\n",
            "  \"model.elapsed_time\": 0.07487893104553223,\n",
            "  \"model.engine_step\": 992,\n",
            "  \"model.loss.nll\": 0.0193328857421875,\n",
            "  \"elapsed_time\": 0.07487893104553223,\n",
            "  \"wall_time\": 1674064402.0830376,\n",
            "  \"global_step\": 992\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 1.21484375,\n",
            "  \"model.lr\": 0.000197612,\n",
            "  \"model.grad_norm\": 10.167463302612305,\n",
            "  \"model.elapsed_time\": 0.07096195220947266,\n",
            "  \"model.engine_step\": 993,\n",
            "  \"model.loss.nll\": 1.21484375,\n",
            "  \"elapsed_time\": 0.07096195220947266,\n",
            "  \"wall_time\": 1674064402.1612241,\n",
            "  \"global_step\": 993\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.041351318359375,\n",
            "  \"model.lr\": 0.00019781100000000002,\n",
            "  \"model.grad_norm\": 1.1347297430038452,\n",
            "  \"model.elapsed_time\": 0.09039545059204102,\n",
            "  \"model.engine_step\": 994,\n",
            "  \"model.loss.nll\": 0.041351318359375,\n",
            "  \"elapsed_time\": 0.09039545059204102,\n",
            "  \"wall_time\": 1674064402.2551374,\n",
            "  \"global_step\": 994\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.51123046875,\n",
            "  \"model.lr\": 0.00019801,\n",
            "  \"model.grad_norm\": 5.1501922607421875,\n",
            "  \"model.elapsed_time\": 0.07175612449645996,\n",
            "  \"model.engine_step\": 995,\n",
            "  \"model.loss.nll\": 0.51123046875,\n",
            "  \"elapsed_time\": 0.07175612449645996,\n",
            "  \"wall_time\": 1674064402.3351166,\n",
            "  \"global_step\": 995\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2496337890625,\n",
            "  \"model.lr\": 0.000198209,\n",
            "  \"model.grad_norm\": 3.2340080738067627,\n",
            "  \"model.elapsed_time\": 0.08340835571289062,\n",
            "  \"model.engine_step\": 996,\n",
            "  \"model.loss.nll\": 0.2496337890625,\n",
            "  \"elapsed_time\": 0.08340835571289062,\n",
            "  \"wall_time\": 1674064402.4212334,\n",
            "  \"global_step\": 996\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.1854248046875,\n",
            "  \"model.lr\": 0.00019840800000000002,\n",
            "  \"model.grad_norm\": 3.3253002166748047,\n",
            "  \"model.elapsed_time\": 0.08435344696044922,\n",
            "  \"model.engine_step\": 997,\n",
            "  \"model.loss.nll\": 0.1854248046875,\n",
            "  \"elapsed_time\": 0.08435344696044922,\n",
            "  \"wall_time\": 1674064402.5140646,\n",
            "  \"global_step\": 997\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.0188751220703125,\n",
            "  \"model.lr\": 0.000198607,\n",
            "  \"model.grad_norm\": 0.5599008202552795,\n",
            "  \"model.elapsed_time\": 0.07143592834472656,\n",
            "  \"model.engine_step\": 998,\n",
            "  \"model.loss.nll\": 0.0188751220703125,\n",
            "  \"elapsed_time\": 0.07143592834472656,\n",
            "  \"wall_time\": 1674064402.5875604,\n",
            "  \"global_step\": 998\n",
            "}\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2744140625,\n",
            "  \"model.lr\": 0.00019880600000000002,\n",
            "  \"model.grad_norm\": 4.618926525115967,\n",
            "  \"model.elapsed_time\": 0.07841944694519043,\n",
            "  \"model.engine_step\": 999,\n",
            "  \"model.loss.nll\": 0.2744140625,\n",
            "  \"elapsed_time\": 0.07841944694519043,\n",
            "  \"wall_time\": 1674064402.6841826,\n",
            "  \"global_step\": 999\n",
            "}\n",
            "[2023-01-18 17:53:22,757] [INFO] [logging.py:68:log_dist] [Rank 0] step=1000, skipped=4, lr=[0.000199005], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:22,759] [INFO] [timer.py:196:stop] epoch=0/micro_step=1000/global_step=1000, RunningAvgSamplesPerSec=12.339457325992507, CurrSamplesPerSec=13.593992370543948, MemAllocated=0.18GB, MaxMemAllocated=0.34GB\n",
            "\u001b[32m2023-01-18 17:53:22\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\n",
            "  \"model.loss\": 0.2218017578125,\n",
            "  \"model.lr\": 0.000199005,\n",
            "  \"model.grad_norm\": 6.248486042022705,\n",
            "  \"model.elapsed_time\": 0.0740199089050293,\n",
            "  \"model.engine_step\": 1000,\n",
            "  \"model.loss.nll\": 0.2218017578125,\n",
            "  \"elapsed_time\": 0.0740199089050293,\n",
            "  \"wall_time\": 1674064402.7605796,\n",
            "  \"global_step\": 1000\n",
            "}\n",
            "[2023-01-18 17:53:22,761] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint default is begin to save!\n",
            "/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1432: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.\n",
            "  warnings.warn(\n",
            "[2023-01-18 17:53:22,764] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ckpts/test/nar/model/default/mp_rank_00_model_states.pt\n",
            "[2023-01-18 17:53:22,764] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ckpts/test/nar/model/default/mp_rank_00_model_states.pt...\n",
            "[2023-01-18 17:53:23,243] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ckpts/test/nar/model/default/mp_rank_00_model_states.pt.\n",
            "[2023-01-18 17:53:23,243] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint default is ready now!\n",
            "100% 2/2 [00:00<00:00,  2.77it/s]\n",
            "\u001b[32m2023-01-18 17:53:23\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "Eval: {'global_step': 1000, 'name': 'subtrain'}.\n",
            "\u001b[32m2023-01-18 17:53:23\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\"global_step\": 1000, \"name\": \"subtrain\"}.\n",
            "0it [00:00, ?it/s]\n",
            "\u001b[32m2023-01-18 17:53:23\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "Eval: {'global_step': 1000, 'name': 'val'}.\n",
            "\u001b[32m2023-01-18 17:53:23\u001b[0m - \u001b[34m__main__\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "{\"global_step\": 1000, \"name\": \"val\"}.\n",
            "\u001b[32m2023-01-18 17:53:23\u001b[0m - \u001b[34mvall_e.utils.trainer\u001b[0m - \u001b[1;30mINFO\u001b[0m - GR=0;LR=0 - \n",
            "New epoch starts.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir -p zoo\n",
        "!python -m vall_e.export zoo/ar.pt yaml=config/test/ar.yml\n",
        "!python -m vall_e.export zoo/nar.pt yaml=config/test/nar.yml"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W_ju2JrZ_W3S",
        "outputId": "43a4f33d-e946-4fed-ca07-dae3b5b115a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2023-01-18 17:53:34,937] [INFO] [comm.py:657:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\n",
            "[2023-01-18 17:53:37,251] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False\n",
            "Installed CUDA version 11.2 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination\n",
            "Using /root/.cache/torch_extensions/py310_cu117 as PyTorch extensions root...\n",
            "Detected CUDA files, patching ldflags\n",
            "Emitting ninja build file /root/.cache/torch_extensions/py310_cu117/fused_adam/build.ninja...\n",
            "Building extension module fused_adam...\n",
            "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\n",
            "ninja: no work to do.\n",
            "Loading extension module fused_adam...\n",
            "Time to load fused_adam op: 0.09586524963378906 seconds\n",
            "[2023-01-18 17:53:37,637] [INFO] [logging.py:68:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adam as basic optimizer\n",
            "[2023-01-18 17:53:37,641] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam\n",
            "[2023-01-18 17:53:37,641] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 optimizer with dynamic loss scale\n",
            "[2023-01-18 17:53:37,648] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adam\n",
            "[2023-01-18 17:53:37,648] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupDecayLR\n",
            "[2023-01-18 17:53:37,648] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupDecayLR object at 0x7f1e060a3d60>\n",
            "[2023-01-18 17:53:37,648] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:37,649] [INFO] [config.py:1008:print] DeepSpeedEngine configuration:\n",
            "[2023-01-18 17:53:37,649] [INFO] [config.py:1012:print]   activation_checkpointing_config  {\n",
            "    \"partition_activations\": false, \n",
            "    \"contiguous_memory_optimization\": false, \n",
            "    \"cpu_checkpointing\": false, \n",
            "    \"number_checkpoints\": null, \n",
            "    \"synchronize_checkpoint_boundary\": false, \n",
            "    \"profile\": false\n",
            "}\n",
            "[2023-01-18 17:53:37,649] [INFO] [config.py:1012:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}\n",
            "[2023-01-18 17:53:37,649] [INFO] [config.py:1012:print]   amp_enabled .................. False\n",
            "[2023-01-18 17:53:37,649] [INFO] [config.py:1012:print]   amp_params ................... False\n",
            "[2023-01-18 17:53:37,649] [INFO] [config.py:1012:print]   autotuning_config ............ {\n",
            "    \"enabled\": false, \n",
            "    \"start_step\": null, \n",
            "    \"end_step\": null, \n",
            "    \"metric_path\": null, \n",
            "    \"arg_mappings\": null, \n",
            "    \"metric\": \"throughput\", \n",
            "    \"model_info\": null, \n",
            "    \"results_dir\": \"autotuning_results\", \n",
            "    \"exps_dir\": \"autotuning_exps\", \n",
            "    \"overwrite\": true, \n",
            "    \"fast\": true, \n",
            "    \"start_profile_step\": 3, \n",
            "    \"end_profile_step\": 5, \n",
            "    \"tuner_type\": \"gridsearch\", \n",
            "    \"tuner_early_stopping\": 5, \n",
            "    \"tuner_num_trials\": 50, \n",
            "    \"model_info_path\": null, \n",
            "    \"mp_size\": 1, \n",
            "    \"max_train_batch_size\": null, \n",
            "    \"min_train_batch_size\": 1, \n",
            "    \"max_train_micro_batch_size_per_gpu\": 1.024000e+03, \n",
            "    \"min_train_micro_batch_size_per_gpu\": 1, \n",
            "    \"num_tuning_micro_batch_sizes\": 3\n",
            "}\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   bfloat16_enabled ............. False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   checkpoint_parallel_write_pipeline  False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   checkpoint_tag_validation_enabled  True\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   checkpoint_tag_validation_fail  False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f1e060a3010>\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   communication_data_type ...... None\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   curriculum_enabled_legacy .... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   curriculum_params_legacy ..... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   data_efficiency_enabled ...... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   dataloader_drop_last ......... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   disable_allgather ............ False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   dump_state ................... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   dynamic_loss_scale_args ...... None\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_enabled ........... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_gas_boundary_resolution  1\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_layer_name ........ bert.encoder.layer\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_layer_num ......... 0\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_max_iter .......... 100\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_stability ......... 1e-06\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_tol ............... 0.01\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   eigenvalue_verbose ........... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   elasticity_enabled ........... False\n",
            "[2023-01-18 17:53:37,650] [INFO] [config.py:1012:print]   flops_profiler_config ........ {\n",
            "    \"enabled\": false, \n",
            "    \"profile_step\": 1, \n",
            "    \"module_depth\": -1, \n",
            "    \"top_modules\": 1, \n",
            "    \"detailed\": true, \n",
            "    \"output_file\": null\n",
            "}\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   fp16_auto_cast ............... False\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   fp16_enabled ................. True\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   fp16_master_weights_and_gradients  False\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   global_rank .................. 0\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   grad_accum_dtype ............. None\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   gradient_accumulation_steps .. 1\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   gradient_clipping ............ 100.0\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   gradient_predivide_factor .... 1.0\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   initial_dynamic_scale ........ 65536\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   load_universal_checkpoint .... False\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   loss_scale ................... 0\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   memory_breakdown ............. False\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   monitor_config ............... <deepspeed.monitor.config.DeepSpeedMonitorConfig object at 0x7f1e060a30d0>\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   nebula_config ................ {\n",
            "    \"enabled\": false, \n",
            "    \"persistent_storage_path\": null, \n",
            "    \"persistent_time_interval\": 100, \n",
            "    \"num_of_version_in_retention\": 2, \n",
            "    \"enable_nebula_load\": true, \n",
            "    \"load_path\": null\n",
            "}\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   optimizer_legacy_fusion ...... False\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   optimizer_name ............... adam\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   optimizer_params ............. None\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   pld_enabled .................. False\n",
            "[2023-01-18 17:53:37,651] [INFO] [config.py:1012:print]   pld_params ................... False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   prescale_gradients ........... False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   scheduler_name ............... WarmupDecayLR\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   scheduler_params ............. {'warmup_min_lr': 1e-06, 'warmup_max_lr': 0.0002, 'warmup_num_steps': 1000, 'total_num_steps': 1000, 'warmup_type': 'linear'}\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   sparse_attention ............. None\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   sparse_gradients_enabled ..... False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   steps_per_print .............. 10\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   train_batch_size ............. 1\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   train_micro_batch_size_per_gpu  1\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   use_node_local_storage ....... False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   wall_clock_breakdown ......... False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   world_size ................... 1\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   zero_allow_untested_optimizer  False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500,000,000 allgather_partitions=True allgather_bucket_size=500,000,000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   zero_enabled ................. False\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:1012:print]   zero_optimization_stage ...... 0\n",
            "[2023-01-18 17:53:37,652] [INFO] [config.py:997:print_user_config]   json = {\n",
            "    \"train_micro_batch_size_per_gpu\": 1, \n",
            "    \"gradient_accumulation_steps\": 1, \n",
            "    \"optimizer\": {\n",
            "        \"type\": \"Adam\", \n",
            "        \"lr\": 1e-06\n",
            "    }, \n",
            "    \"scheduler\": {\n",
            "        \"type\": \"WarmupDecayLR\", \n",
            "        \"params\": {\n",
            "            \"warmup_min_lr\": 1e-06, \n",
            "            \"warmup_max_lr\": 0.0002, \n",
            "            \"warmup_num_steps\": 1000, \n",
            "            \"total_num_steps\": 1000, \n",
            "            \"warmup_type\": \"linear\"\n",
            "        }\n",
            "    }, \n",
            "    \"gradient_clipping\": 100.0, \n",
            "    \"fp16\": {\n",
            "        \"enabled\": true\n",
            "    }\n",
            "}\n",
            "Using /root/.cache/torch_extensions/py310_cu117 as PyTorch extensions root...\n",
            "Emitting ninja build file /root/.cache/torch_extensions/py310_cu117/utils/build.ninja...\n",
            "Building extension module utils...\n",
            "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\n",
            "ninja: no work to do.\n",
            "Loading extension module utils...\n",
            "Time to load utils op: 0.08678960800170898 seconds\n",
            "[2023-01-18 17:53:37,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ckpts/test/ar/model/default/mp_rank_00_model_states.pt...\n",
            "[2023-01-18 17:53:37,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ckpts/test/ar/model/default/mp_rank_00_model_states.pt.\n",
            "[2023-01-18 17:53:37,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ckpts/test/ar/model/default/mp_rank_00_model_states.pt...\n",
            "[2023-01-18 17:53:37,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ckpts/test/ar/model/default/mp_rank_00_model_states.pt.\n",
            "2it [00:00, 4868.61it/s]\n",
            "/usr/local/lib/python3.10/site-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "zoo/ar.pt saved.\n",
            "[2023-01-18 17:53:42,486] [INFO] [comm.py:657:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl\n",
            "[2023-01-18 17:53:44,755] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False\n",
            "Installed CUDA version 11.2 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination\n",
            "Using /root/.cache/torch_extensions/py310_cu117 as PyTorch extensions root...\n",
            "Detected CUDA files, patching ldflags\n",
            "Emitting ninja build file /root/.cache/torch_extensions/py310_cu117/fused_adam/build.ninja...\n",
            "Building extension module fused_adam...\n",
            "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\n",
            "ninja: no work to do.\n",
            "Loading extension module fused_adam...\n",
            "Time to load fused_adam op: 0.08924126625061035 seconds\n",
            "[2023-01-18 17:53:45,161] [INFO] [logging.py:68:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adam as basic optimizer\n",
            "[2023-01-18 17:53:45,164] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam\n",
            "[2023-01-18 17:53:45,164] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 optimizer with dynamic loss scale\n",
            "[2023-01-18 17:53:45,171] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adam\n",
            "[2023-01-18 17:53:45,171] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupDecayLR\n",
            "[2023-01-18 17:53:45,171] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupDecayLR object at 0x7f88365c73a0>\n",
            "[2023-01-18 17:53:45,171] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.001], mom=[(0.9, 0.999)]\n",
            "[2023-01-18 17:53:45,172] [INFO] [config.py:1008:print] DeepSpeedEngine configuration:\n",
            "[2023-01-18 17:53:45,172] [INFO] [config.py:1012:print]   activation_checkpointing_config  {\n",
            "    \"partition_activations\": false, \n",
            "    \"contiguous_memory_optimization\": false, \n",
            "    \"cpu_checkpointing\": false, \n",
            "    \"number_checkpoints\": null, \n",
            "    \"synchronize_checkpoint_boundary\": false, \n",
            "    \"profile\": false\n",
            "}\n",
            "[2023-01-18 17:53:45,172] [INFO] [config.py:1012:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}\n",
            "[2023-01-18 17:53:45,172] [INFO] [config.py:1012:print]   amp_enabled .................. False\n",
            "[2023-01-18 17:53:45,172] [INFO] [config.py:1012:print]   amp_params ................... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   autotuning_config ............ {\n",
            "    \"enabled\": false, \n",
            "    \"start_step\": null, \n",
            "    \"end_step\": null, \n",
            "    \"metric_path\": null, \n",
            "    \"arg_mappings\": null, \n",
            "    \"metric\": \"throughput\", \n",
            "    \"model_info\": null, \n",
            "    \"results_dir\": \"autotuning_results\", \n",
            "    \"exps_dir\": \"autotuning_exps\", \n",
            "    \"overwrite\": true, \n",
            "    \"fast\": true, \n",
            "    \"start_profile_step\": 3, \n",
            "    \"end_profile_step\": 5, \n",
            "    \"tuner_type\": \"gridsearch\", \n",
            "    \"tuner_early_stopping\": 5, \n",
            "    \"tuner_num_trials\": 50, \n",
            "    \"model_info_path\": null, \n",
            "    \"mp_size\": 1, \n",
            "    \"max_train_batch_size\": null, \n",
            "    \"min_train_batch_size\": 1, \n",
            "    \"max_train_micro_batch_size_per_gpu\": 1.024000e+03, \n",
            "    \"min_train_micro_batch_size_per_gpu\": 1, \n",
            "    \"num_tuning_micro_batch_sizes\": 3\n",
            "}\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   bfloat16_enabled ............. False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   checkpoint_parallel_write_pipeline  False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   checkpoint_tag_validation_enabled  True\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   checkpoint_tag_validation_fail  False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f88365c7100>\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   communication_data_type ...... None\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}}\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   curriculum_enabled_legacy .... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   curriculum_params_legacy ..... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}}\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   data_efficiency_enabled ...... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   dataloader_drop_last ......... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   disable_allgather ............ False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   dump_state ................... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   dynamic_loss_scale_args ...... None\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_enabled ........... False\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_gas_boundary_resolution  1\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_layer_name ........ bert.encoder.layer\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_layer_num ......... 0\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_max_iter .......... 100\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_stability ......... 1e-06\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_tol ............... 0.01\n",
            "[2023-01-18 17:53:45,173] [INFO] [config.py:1012:print]   eigenvalue_verbose ........... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   elasticity_enabled ........... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   flops_profiler_config ........ {\n",
            "    \"enabled\": false, \n",
            "    \"profile_step\": 1, \n",
            "    \"module_depth\": -1, \n",
            "    \"top_modules\": 1, \n",
            "    \"detailed\": true, \n",
            "    \"output_file\": null\n",
            "}\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   fp16_auto_cast ............... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   fp16_enabled ................. True\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   fp16_master_weights_and_gradients  False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   global_rank .................. 0\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   grad_accum_dtype ............. None\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   gradient_accumulation_steps .. 1\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   gradient_clipping ............ 100.0\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   gradient_predivide_factor .... 1.0\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   initial_dynamic_scale ........ 65536\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   load_universal_checkpoint .... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   loss_scale ................... 0\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   memory_breakdown ............. False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   monitor_config ............... <deepspeed.monitor.config.DeepSpeedMonitorConfig object at 0x7f88365c71c0>\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   nebula_config ................ {\n",
            "    \"enabled\": false, \n",
            "    \"persistent_storage_path\": null, \n",
            "    \"persistent_time_interval\": 100, \n",
            "    \"num_of_version_in_retention\": 2, \n",
            "    \"enable_nebula_load\": true, \n",
            "    \"load_path\": null\n",
            "}\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   optimizer_legacy_fusion ...... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   optimizer_name ............... adam\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   optimizer_params ............. None\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   pld_enabled .................. False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   pld_params ................... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   prescale_gradients ........... False\n",
            "[2023-01-18 17:53:45,174] [INFO] [config.py:1012:print]   scheduler_name ............... WarmupDecayLR\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   scheduler_params ............. {'warmup_min_lr': 1e-06, 'warmup_max_lr': 0.0002, 'warmup_num_steps': 1000, 'total_num_steps': 1000, 'warmup_type': 'linear'}\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   sparse_attention ............. None\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   sparse_gradients_enabled ..... False\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   steps_per_print .............. 10\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   train_batch_size ............. 1\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   train_micro_batch_size_per_gpu  1\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   use_node_local_storage ....... False\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   wall_clock_breakdown ......... False\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   world_size ................... 1\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   zero_allow_untested_optimizer  False\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500,000,000 allgather_partitions=True allgather_bucket_size=500,000,000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   zero_enabled ................. False\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:1012:print]   zero_optimization_stage ...... 0\n",
            "[2023-01-18 17:53:45,175] [INFO] [config.py:997:print_user_config]   json = {\n",
            "    \"train_micro_batch_size_per_gpu\": 1, \n",
            "    \"gradient_accumulation_steps\": 1, \n",
            "    \"optimizer\": {\n",
            "        \"type\": \"Adam\", \n",
            "        \"lr\": 1e-06\n",
            "    }, \n",
            "    \"scheduler\": {\n",
            "        \"type\": \"WarmupDecayLR\", \n",
            "        \"params\": {\n",
            "            \"warmup_min_lr\": 1e-06, \n",
            "            \"warmup_max_lr\": 0.0002, \n",
            "            \"warmup_num_steps\": 1000, \n",
            "            \"total_num_steps\": 1000, \n",
            "            \"warmup_type\": \"linear\"\n",
            "        }\n",
            "    }, \n",
            "    \"gradient_clipping\": 100.0, \n",
            "    \"fp16\": {\n",
            "        \"enabled\": true\n",
            "    }\n",
            "}\n",
            "Using /root/.cache/torch_extensions/py310_cu117 as PyTorch extensions root...\n",
            "Emitting ninja build file /root/.cache/torch_extensions/py310_cu117/utils/build.ninja...\n",
            "Building extension module utils...\n",
            "Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)\n",
            "ninja: no work to do.\n",
            "Loading extension module utils...\n",
            "Time to load utils op: 0.09516406059265137 seconds\n",
            "[2023-01-18 17:53:45,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ckpts/test/nar/model/default/mp_rank_00_model_states.pt...\n",
            "[2023-01-18 17:53:45,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ckpts/test/nar/model/default/mp_rank_00_model_states.pt.\n",
            "[2023-01-18 17:53:45,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from ckpts/test/nar/model/default/mp_rank_00_model_states.pt...\n",
            "[2023-01-18 17:53:45,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from ckpts/test/nar/model/default/mp_rank_00_model_states.pt.\n",
            "2it [00:00, 4829.37it/s]\n",
            "/usr/local/lib/python3.10/site-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  warnings.warn(_create_warning_msg(\n",
            "zoo/nar.pt saved.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m vall_e 'hello world' data/test/test.wav toy.wav"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sk8IMWxR_cSM",
        "outputId": "ea01f072-0ae1-43ac-bce0-bb6b76633e85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 15% 151/1000 [00:02<00:13, 63.90it/s]\n",
            "toy.wav saved.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Audio\n",
        "Audio('toy.wav')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 75
        },
        "id": "XjCVyXv2_j5b",
        "outputId": "2d8fa9e4-a2f5-4694-cc40-090cda605777"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.lib.display.Audio object>"
            ],
            "text/html": [
              "\n",
              "                <audio  controls=\"controls\" >\n",
              "                    <source src=\"data:audio/x-wav;base64,UklGRqR5AQBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAAZGF0YYB5AQDWCNQI5QguCSoJoAhKCEMIZAdgBx4GIgX0BCsEjAY9CpULIwuAClsJ7wi5BzoH9wV6BRMFaASjA3gC/AHrAJEANQD+/1j/d/5r/a38bfsX+fX30PbU9S33dvhw+bL4EvfR9W70IfMS8bDvx+6K7mTulO7S74PxV/Lx8mDyt/Ev8kDzPvRY9NX0r/Q79Cj0RPPa8eXwPfCH8XLyJ/Ik8mPxk/Jd8wf0SPMQ8pvwEPBg8M3vh/HE8gr0d/S09Fr1w/Xq9Tj2Pvdd+R/8y/65AbED9QQlBaYD2AJmAvoCHQT1BFgFRQToBMMF3QdGCg0LkgsNDB4NIQ5WDnwNQQzTCn4KRApICggKmwnlCXsJOwnECMAHxgaeBdgFnAVxBdsETQSTBHYDLwTyA04DMQQDBQ8FmgMiAW3/eP+Y/gr+ef34/Lj8afv2+rn6gvpl+hb5+Pcm91X3V/mg+9b+GAH6AX8CrwIVAl4B/f9F/mb8SPuu+8b77/tE+//5FPia93L3mPd5+Pb3Hfg6+Hn52/rX+/77uvue/AL9lf6V/7QAxgG9AZwBqAB//1P+uf3S/LD8m/xC/TX+Jf97AJAAIwCH/63/2P4O/i/+Vf+GAJQBrAIzA9cDSgQJBSkG7AZ7B2MHZAdECSMLOA0CDvENpg0sDCcLfQrrCuELNwwNDYQNIg6fD7UPZhB1EIcRZBR+FdYVqxQzE/wRPBE4DywNmgsDCtUKTAvAC9gKsAd9BG0BEv94/Wf8PPsR/Hz+VgGXA6cF9gYBB4wG8AShAikA9/5i/jH+xv6S/1L/yv68/dL8WPse+ir54ffQ9l706PId8zDzBPID8Kvu4+0k7YfsUuxE7IjsF+1b7vru8u848bLxR/Os87H1Y/f592H4dfdY96H3SPgy+Q36Zvqd+qT6svor+yP7M/tY+4/7DPxL/GL8+/vz+/v7Vfxa/Kj8a/3O/Wf+zP6//p3+vv5q/mT+n/6P/qD+j/6R/ov+kf6b/qH+2v7D/t7+8P4J/03/Kf9s/5n/4P+4/7v/6P/o/yoAVgBuADwANwD8/+n/3v/m//7/AgDM/8T/x//5/4EAuwA6AU0B/gCBAE8AggCzAE8Axf+i/+3/cgB+AB4A6v+W/xv/uf6L/pT+Iv61/bD9pf26/eX9wv2u/eX9Lv4e/tr93P3R/Yf9kP1a/Tb9Ov09/UD9QP0m/Wf9J/0l/T/9Mv2E/Uj9Tf09/Wn9Vv1b/ST97/wK/S39T/1Z/V/9S/1v/Xv9qP3x/ff9Av4Y/iD+B/7r/cD9m/29/Y39n/2E/W79gf17/X/9jf2E/Ur9mv1l/SD9wvyX/LP8G/1P/Wf9r/1+/Xj9jP2N/a39xv2U/aL9xf35/Rr+Hv47/nX+vP7Z/jv/T/9x/6H/1/81ABoAGwA6AJcAqgCoAJ8AnQDHAM8AHQExATEBPQFDAXsBkwGiAeUBDgIkAjcCZwKFAnoCiQKGAnYCYwJQAkICNgIjAiwCKAIYAhICGAL6ARYCKwJmAo4CkQKfAnUCYwI2Ak4CTwJeAqQCtwLbAh0DNQNRA30DWwMoAwUDBwMrAzUDTQNTA0oDcwOSA2kDbQOFA3QDlgOIA4QDZQNiA2oDVQNeA08DTAM4Az4DTQNQAzkDDQP4AvcCGQMeAyQDHgMVAwYDuQJ9AnECZAI4Ai4CPQJaAloCEQLnAfQB6gHLAcoBrQGVAYMBWgFJAUQBKAEbARMB9wDiAMEAowCOAIEAaABPAEcAUABTAE4AOAAUAA0AFQAoAC0AIQATAAQA+f/h/9H/v/+3/6b/hf97/2T/Tv80/w7/7v7a/sf+wf7G/sL+xP7A/sj+wf6x/rz+sf65/rD+p/61/tb++P4Z/xj/7f7g/tD+5v4A/w7/Df/6/vH+9v4N/xn/HP8S/w7/FP8j/zf/Q/9R/1v/XP9o/2z/Z/9k/13/QP8X//b+z/60/p3+ov6s/sv+4P7X/tr+5/7p/vD+1f7E/tL+vf6q/oz+dP5y/nz+iP6l/qD+fP5k/mD+Zf5Z/kD+Ev4C/tn9u/2+/af9jv2L/XL9Z/1m/U39Xf1b/VX9Uv1N/Ur9Tf1b/Wz9gv2S/Z79qP28/cb95f3v/fP9B/4O/v798f3L/aX9l/2J/YX9ff1s/V39YP1s/Wj9V/1P/UH9J/0q/Sz9K/02/TX9P/1W/WH9d/2B/Wn9Yf1K/SD9Kv07/W79gf2f/bX9sv27/cT92P3Y/dn9/P0R/iT+TP6D/r/+4v7z/hD/Nv9E/13/gv/C/9b/8P/9/wEAFgAeAEMAUwBfAHoAhQCFAJMAiwCVAK4AygDzABsBGQEZASwBPgFHAToBNgE1ATkBNAFSAWgBcgFnAXYBiwGTAakBqQG+AdIBxQG0AZsBmgGuAZkBjwGkAbkB0gHtAfwBBwIVAhICDAIHAgECAwL/AeYB4QHuAQECCwIMAv0B/gEMAgoCHwIhAhYCCgIIAhcCJwIzAj8CRgJJAlECQwI0AiUCKwIoAhwCDQLsAdEBvAHAAcABwQGuAZ0BmgGGAYIBeQFyAWwBaQFhAVYBTwFFAUUBLwEXAQMB6ADXAMkAxADGAL0AsgCpAKMAqQCrAKUAowCgAJkAmwCcAJEAhQBxAGUASAA5AC0ALAAjABEABwDi/87/zP/H/8L/u/+x/5//nP+d/5z/iv9//4j/jv+R/4L/hv+H/5L/lf+C/2n/Vv9r/3D/dv9z/2v/bP9Q/zr/K/8c/x//Hv81/1//c/95/3n/h/+S/4b/ef+E/4P/hP99/2X/bf94/4b/h/+N/6b/mP+M/4H/eP+J/4j/dv9b/0r/Qf8//zn/N/89/yn/Lv8U/wv/AP/u/gP/9v70/vH+9P7p/tz+zf66/q/+n/6W/nz+cv5i/mL+Vv5F/kL+Kv4q/hf+EP4A/vX98P3r/fr9+/3+/QP+Ef4W/iH+J/4Z/iL+Kf4m/h/+Ev4H/gD+8v3s/ez99P0K/ib+Qv5Q/lr+YP5k/mT+X/5S/kr+UP5U/mb+bP5r/nT+a/5Y/k7+QP5C/j/+L/4n/iX+FP4h/jH+Tf5U/kv+U/5Y/l7+Yv5w/nT+dv6P/qn+vP7R/uX+8v4D/xX/LP9D/1D/Y/90/4j/j/+V/5v/qv/D/8//4//1/wcAGwAxAEYAWwBqAHUAeACKAJkAlgCqAKYAsgDHAM4A1ADHANEA4gDsAPEA7wDtAAMBGwEiARwBEgEcASMBKgElARgBDQEJARsBKQEuAUgBVAFhAW4BgwGaAa4BsAGwAaYBngGeAZwBqgGyAbABrQGxAbcBxQG8AbMBvQG6AcUBzgHIAccBywHEAcABxQHJAccBwAG1AaEBkAGRAY0BlwGXAZ0BkgGHAXgBbAFrAWkBcAFiAWYBVwFUAVcBWwFZAVMBUgE+ASwBHQELAQ8BDQEHAQQB8wDuAOkA1ADIAMcAwQCyALkAtwCqAJ4AgwByAHAAWQBeAGAAYgB8AH0AaQBbAFUAOwAxACQABwAHAAkABwAGAPb/6P/t/+v/6//p/93/1P/Y/9L/x/+//7X/sP+2/7z/wv/I/8b/yv/I/8D/tv+t/6b/rP+r/6D/lP+H/4r/hP+M/5T/lf+U/5H/mP+a/6H/qf+z/7b/uf+1/7j/vP+8/73/sv+q/43/g/+C/3j/dv9h/1b/Sf8+/zn/Ov8+/zf/L/8i/xz/Gv8h/y//M/83/y7/Jf8X/w3/B//z/uj+3f7V/tD+zP7M/s7+2v7Z/t/+4/7i/tP+w/65/rP+rv6d/pL+h/6F/n/+ff56/nL+bv5x/nL+d/57/oX+jv6R/pf+nf6d/p7+mv6b/pv+l/6W/pj+l/6d/pz+mP6W/oz+hf5//nz+ff54/oP+gP5+/nv+Z/5k/m/+Z/5f/lb+TP5A/kD+RP5D/lX+Y/5y/oP+ef6G/oj+iP6P/pP+l/6h/rP+x/7h/u/+9/4D/xD/I/81/z//Rf9P/1v/af+B/5D/qP+8/87/3P/r/wAAFgAtAEMAVgBrAH4AkACbAKcArgCxALAArwCyALIAswC3AMAAwADLANAA2wDoAOsA+AD/ABABGgEjASABJQEkAR0BIAEmASwBMgE0ATEBLwE1AT4BSgFTAVwBYQFiAWIBaQFqAWcBZgFoAW4BdQF+AYMBhAGQAZgBnQGmAakBqgGtAawBqQGqAaIBnAGcAZoBkwGHAYMBfwGDAX0BfAF0AW8BdAFqAWwBaAFiAWQBYQFbAVYBTAFBAUABNAEpASEBFQENAQwBAwH6APUA6ADlAOAA2ADQAMsAzADNAMMAuwCzALAApQCfAJgAiQB4AGsAZgBkAFgATQBYAFoAUgBHAEYARAA+AD0ALAAjABoADgAPABAAEwASABQAGQAXAAcA+v/y/93/0//B/7f/tP+p/63/o/+g/6X/o/+m/6v/sP+2/7j/tv+y/6j/ov+e/5r/mP+U/5b/lv+W/5z/n/+n/6b/pf+g/6D/ov+o/6j/qf+n/53/lP+H/43/iP+H/4P/gP+E/4H/hP+G/4b/g/+E/4T/hv+E/4P/ev92/3H/av9k/17/XP9Q/0r/Pv80/yf/Gv8O/wL/A/8B/wD/+f70/ur+4f7a/tP+1f7O/r/+sf6h/pf+kf6K/ov+iP6L/oj+f/6D/n7+fP55/nf+c/5x/nX+dP56/oD+hv6I/o/+nf6l/qv+rv65/rT+uP63/r3+v/66/sL+vf68/rf+uf68/r3+uP6v/rH+rP6r/rD+rf6u/q7+rf6v/rH+s/63/rn+t/64/rn+wf7N/tv+5/7y/vj+A/8I/wv/Df8Q/xH/Gv8h/yb/OP9Q/2r/gP+U/57/r//G/9j/5//y/wAADgAcACgAMgA5AEQAUQBfAGgAcAB6AIIAkgCcAKMAqwC0AMIA1QDhAOcA7gDyAPYA+QD2APoA/wAAAQABBQEBAQIBCwEQARcBJQEjASUBLQEzAUEBTgFYAV0BXwFjAWQBZwFmAWQBZQFkAV8BXwFoAXIBgwGMAYsBkQGRAZMBngGfAaMBqgGpAacBpAGlAaMBpwGnAa0BrQGrAacBrQGwAasBogGUAYsBggF5AXYBdQF3AW4BagFfAVwBVwFQAUUBOAEuASQBIQEYAQgB/QD0AOgA4ADXANAAywDKAM0AxwCvAKMAoACYAI4AgABtAGcAYgBYAEwARAA9ADkALgAgABwAEwAJAP7/9f/v/+7/5//l/+f/4//c/8z/xP+9/7b/q/+k/6f/qv+x/63/q/+q/6P/nv+X/5L/jf+M/4f/hf+B/3r/c/9r/2H/WP9Q/07/Tv9N/1L/UP9Q/1H/UP9M/03/S/9E/z3/L/8o/yL/I/8g/xv/Gv8V/xj/Fv8a/xb/Ef8N/wf/D/8M/xX/Hf8r/zv/Q/9T/2D/ZP9s/2r/ZP9t/2v/cP94/4T/jv+b/6f/rf+//8r/z//Z/93/4f/j/+X/6P/n/+3/7f/v//L/9P/5//v/BQAIABIAEwAUABMADwAQAAQA///8//z/9P/s/+T/2P/U/9P/0f/I/8X/wf+//8H/x//J/8v/1P/V/9r/4f/p//L/AAATABYAHQAhACkANAA6AEAAQQBGAEcATABUAFkAXgBkAGwAdwB/AIQAhwCOAJMAmQCoALoAyQDHAMsAzQDIAMgAwADDAMoAzgDRANEA1gDbANgA3wDjAOYA/wAHAQ0BEgEXARYBFAEYARcBEgEWARcBFAEVARcBFgEPAQ0BEQERARABGAEdASkBLwEwATMBMAE5ATgBOwE6ATkBNgE2ATgBOAE6ATwBOwE5AUEBSgFSAVgBVQFQAU8BTgFHAUABNwEzATMBMwE1AUABRAFNAVABUwFTAU8BTwFNAUsBSAFJAUcBRAE/AT4BOgExASgBFwEKAfcA5wDbANYA0gDNAMkAwgC6AKwAnwCTAIcAfAByAGgAYwBfAFsAUABJAD8AMgAgAAsA+f/v/+b/2v/O/73/s/+z/7D/rP+t/6b/nP+T/4f/fv94/3D/Y/9f/1r/Uv9c/1v/UP9U/1P/Uf9S/0r/Qv9B/zv/Of88/zn/OP8x/y3/Jv8r/xv/Dv8N/wf/Af/z/ub+4f7j/t3+3v7j/u/+8f7+/gj/EP8W/xT/G/8a/xL/+/71/vL+7f7h/tf+1/7R/sz+x/7O/s/+0/7V/tr+4v7l/uX+4f7i/ub+6v7f/tf+1/7c/t3+3/7b/tf+2v7Z/tf+0f7J/sz+1P7U/tD+wf6+/rH+s/62/rf+t/6x/r7+xv7Z/uH+7v71/vf+A//6/gj/Ef8Z/yn/Mv8+/1P/Zf9w/3r/gP+F/4r/hv+B/33/bf9j/2T/Zf9o/2z/cv9v/3f/cv9v/2z/b/98/4f/nP+b/6f/rP+n/7L/rP+2/7v/wf/I/8P/w/++/8X/yv/L/9T/6f/u/+j/6f/p//j/BwAbABgAHgAqADAAOQA/AEgATABRAFUAWABSAGQAawBmAHAAbABuAGoAaABiAFIAUwBMAEcATwBNAE8ASwBTAFEAXABlAGYAcwByAH8AhgCKAIsAjQCKAH4AdgBqAF4AZABqAHgAiQCTAKIApgCzALcAvAC9AL0AxADOAOQA8gD7APcAAAH/APcA/AD1AOwA6ADpAN8A0wDZANwA1gDMAMUAxADKANMA1ADVANgA3QDlAOMA5ADhAOAA4gDhANUAzAC7AKwAqwCiAI4AhACAAH0AhwCHAIYAhwCFAIMAfgB/AHwAeAB3AHsAfwCLAJEAkACQAJQAlQChAKkAqQCkAJ4AmACSAI4AgwB7AHQAcABwAGsAbwBxAGwAaABiAFwAYABeAFgAUgBLAEQAPAAyAC0AKAAiACMAHgAQAA0ADwAGAAgAAQD6/+v/2f/U/9f/1f/H/8L/uP+1/7L/qv+s/6r/nv+K/3T/aP9e/1//Xf9c/2H/YP9n/2j/Zv9h/2L/Zv9l/13/Sf9J/1r/aP9n/2L/aP94/4H/gP+F/47/lv+Z/5v/of+o/6j/qf+t/67/sf+6/77/zP/a/93/5v/v/wAACgAWABoAHwAsADEAOAA0AC4ALgAwAC8ALAAlACAAFgAZABsAGgAjACUALQA4AEIARQBJAE4AVQBbAF0AYwBlAGgAbgBzAHgAfACDAIsAigCUAJcAmwCgAJoAlgCSAJUAnACuALkAwQDJAMgAygDEAMMAvwC5ALkArgCuAKEAmgCOAIoAkACRAJcAngCqALAAsgCyALYAvQDAAMgAyADEAMMAuACwAKwApwCpAKwAsQC0ALoAvwDIANAA1ADVANQA0wDOAMoAyADFAMEAuwC4ALYAswC0ALEAqQCqAKcApwCmAKgApgChAJgAkQCTAJAAkgCRAI4AhwB9AHUAbABjAFgAUABEADUAKgAkACYAKgArACwAJwAgACQAKQAsACkAKQAvADMANgA7AEIARgBPAFIAVgBdAGYAZQBlAGcAXwBYAFQAUABKAEMAOwAzACkAIAAVAA0ABAD8//b/7v/r/+f/6f/m/+n/6P/j/9z/yv+8/6b/lv+K/4H/f/9y/2b/XP9M/z//Lv8l/yD/Gf8R/wr/Bv8A/wP/C/8V/xn/G/8h/yP/I/8h/x3/H/8h/yL/JP8f/yP/Kv81/zf/Of8+/0X/Sv9O/1H/U/9V/1L/Uf9R/1T/T/9Q/1P/Uv9W/1L/WP9Z/1v/XP9i/2H/Y/9m/2H/Wv9U/1H/T/9Q/0v/Sf89/z7/P/8w/yr/Hv8V/wz/Dv8J/wr/Dv8K/wL/Af/6/vb+8f7z/vb++f7//vz+A/8G/wv/B/8J/xb/Ev8T/xb/GP8Z/w7/Cv8A/wH/Cf8G/xf/G/8q/zn/SP9e/2b/Zv9p/2b/aP9d/1L/S/8+/0D/Pf9L/0X/Tf9I/0H/O/8u/yn/GP8T/wL//P76/gb/Ff8k/y//Mv88/0H/Qf9K/07/UP9I/z7/Nv83/zb/L/8v/y3/L/8s/yr/Lf8r/zX/Ov9C/0z/Vf9p/3b/hf+O/5D/nP+r/73/0f/g/+n/8//5//v////+//3////+//3/AAD9//r/+P/9/wUABAAHAAgABgABAP7//v/+//3/+f/1//P/8P/z//v/DgAUABcAGgAWABIADwAWABoAFwANAP3//f8GAA8AEAAfAC4APgBLAGEAfgCSAKcAqACmAKsAsQCwAMUAxgC8ALQApgClAKUAqACfAJYAlwCVAJEAlwCdAKMApgCfAJ8ApwCoAKkAswCzALsAxgDPANUA3ADaANQAywDCALUAnQCRAIwAjQCMAI0AlQCVAJkAnwCcAKIApwCmAK0AswC4AL8AzQDUAN0A4wDlAOEA3wDYANgA3QDaANgA0wDQAM0AywDFAMEAuwCvAKkAoQClALAAsACqAKMAngCjAK0AtgDAAMQAwADBALwAuQCzAKYAoACfAJ0AnACaAJwAlwCIAHkAbwBsAGsAagBwAH0AgQB+AIIAhQCKAIwAkQCUAJEAkgCMAIQAggCBAH8AfAB9AHgAcQBnAFcAUQBIAD8AOwAvAC4ALAAoAC4APABEAEkASwBJAEwASgBDADkAKwAdABAACwAJAA4ADgAFAAQABgANABAADQD+//b/8v/r/+j/6P/q/+z//v8GABEADgANAAgABgD//+3/6v/X/9r/2//T/8T/vf++/7v/sv+q/6v/qP+j/5j/lf+T/4z/g/+H/4r/i/+K/4r/jP+G/4X/f/99/3n/eP9x/3D/aP9n/2n/aP9n/2T/Zv9o/23/c/98/4P/hP+K/4r/jf+U/5X/k/+S/4//jP+P/4//kv+U/5H/kP+K/4j/h/9//3r/b/9m/2L/Wv9S/1T/Vf9Q/1L/U/9X/1r/Xv9d/1f/V/9O/1D/Sv9G/z//Mv8v/yr/Kf8l/yX/Iv8h/xr/Gv8X/xT/Ev8P/wz/DP8Q/w3/Ef8W/xr/Gv8V/xT/Ev8M/wj/Cv8K/wb/Df8P/xX/Jf8u/zr/Rv9P/1v/YP9j/2T/Yf9c/1r/Vv9O/0L/M/8x/yv/JP8f/xv/E/8T/xL/Df8O/w7/Dv8U/xb/Gf8c/yL/Jv8v/zH/Mv8z/y//Mf8w/zX/NP84/z3/Rf9Q/1n/Wv9a/2v/b/96/4D/g/+K/47/lf+l/7D/uP/I/9n/7P/5/w0AHwAsAEEASQBRAF8AZgBqAHEAagBoAGUAYgBhAGgAagBrAG4AbABxAHIAdQB5AIMAhwCKAIwAjQCRAJEAlgCaAJ8AqwCtALIAtQC0ALMAtAC2ALoAugDDAMcAyQDPANUA2ADcANwA3QDgAN4A5QDqAO4A8wDzAPQA+wD/AAABAQEAAQUBBQELAQ0BDgEUARsBGgEaARsBGgEaARoBGgEWAREBEwEPARIBFwEVARMBEQEHAQMB+AD3APwA9gD0APQA8gD6AP0A+QD5APMA6gDhAM4AwwC3ALIAqQCkAKMAqQCrAK0AsQCzALgAtQC0AK0AogCbAJEAhQB9AIAAggCDAIcAjACQAIgAgQB/AH0AdgB1AHsAfwCAAH0AfwCEAIkAhgCCAIAAeQB5AGwAZQBgAFoAYABlAG0AcQBxAHIAdAB2AHYAcwBzAHQAeQB3AHcAeABzAHYAdgB6AHoAggCJAJEAngCmALEAvQDGANMA4QDqAPEA9wD6APcA9gDuAOMA3QDVANQA0wDPAMYAtwCsAKEAnACeAJ4AogCpAKsAqwCvAK0AtgC3ALUAswCmAKEAkgCAAG4AWgBEACcAEQAHAP7/7v/l/97/2//d/+H/4f/n/+n/5//k/+H/2v/S/9D/y//H/8f/xv/B/8L/wv/F/8v/zP/Q/9b/1v/Y/93/3//j/+7/9v/2//j/9//5//n/8//w/+3/6f/g/+H/4f/h/+L/3v/a/9T/zf+9/6//o/+S/4L/dP9t/2n/aP9m/2X/Zv9p/2v/af9r/2X/Zf9i/2f/bP9z/3n/ev+A/4D/ff94/3j/fP+E/4n/jv+T/5z/pv+y/8T/z//Z/+j/+P/+/wgAEQAWABgAHAAmADEAPABCAE8AWwBhAGoAdACGAIwAjACUAJgAnACkAKYAoACrAKcAmACSAIoAiwCLAIMAfQCCAIcAhwCKAIgAhQCEAH8AfAB7AHgAdwB0AHEAbgBtAGkAZABgAFoAUgBCADgAKgAnACUAIQAdABsAIQAnAC0ALwArACkAJgAiACQAJQAZAA0AAQD6//P/6//h/9n/0v/G/7r/sP+v/7H/qP+j/5z/mP+R/4r/gf92/2z/Wf9I/zv/Mf8p/yT/HP8T/xH/Df8N/wr/Bf8F//7+/f78/vz++P7d/tP+x/7B/r3+sf6s/qb+ov6e/pv+mP6S/pH+iv6C/nv+eP56/nr+e/55/nr+af5d/ln+UP5H/j3+Of45/kD+Rv5N/k/+Vf5V/ln+W/5c/l7+X/5h/mP+Yf5h/mD+Zf5k/mH+Yv5l/mn+Zv5k/mP+Yv5m/mf+a/5u/nn+e/57/oD+iP6N/pb+of6x/sL+0P7c/ur+8f71/vr+/v4C/wr/Cv8O/xb/G/8h/yf/L/85/0f/Uv9g/3P/hP+M/5b/nP+p/7H/uP+//8H/yP/N/9v/5v/t//n/AwAPABgAIwArAC4ANAAzADIAMQAvACsAJgAnACkALAAvADYANwA3ADcANwA5ADoAQQBGAE0ATgBQAFMAWgBcAFsAYgBiAGMAXQBTAE4ATQBQAFEAXQBtAIMAkgCgAKkArgCwALUAugC7AMAAwwDLANQA2ADUANgA3ADcAN8A4wDpAPIA+gD9AAQBCwEOARcBIAEqATMBPQE9AT4BPgE/AUUBRwFMAVUBYAFtAXoBhgGMAY8BkQGYAZ0BqwG1AbsBvwHCAcQBxwHKAcgBwwHDAcABvwG/Ab0BtwGxAakBpQGhAZ0BnQGcAZUBkwGQAYsBjQGNAYgBhgGAAXsBeAFyAWsBYgFTAUoBQQEzASkBGwEKAfoA8ADnAOUA5ADjAOcA6gDvAPMA9wD4APkA9QDtAOYA2wDSAMgAwgC5AK4AoQCWAIcAdQBhAE8AQAAyACwAJgAjACEAIAAfABYAEAAHAPz/7v/h/8r/uf+r/5v/kP+F/33/ev96/3j/fP97/3P/a/9e/1f/Sf9G/0T/Rf9F/0P/Sf9J/0f/QP84/zP/Lf8k/yL/If8f/x//Hf8h/yP/JP8h/yH/Gf8L/wP/9f7o/uH+2P7U/sz+xf7F/sX+xP7B/sT+xP7B/sf+yP7L/s7+yf7D/rv+tP6s/p/+lP6L/of+gv6C/n/+fv6J/pH+mf6i/qz+tv7C/s/+2/7r/vf+Bv8T/yP/Kv85/z//Sv9W/13/bP9w/3j/ev9+/4P/hv+F/3//hP+O/5v/pP+q/6z/qv+r/7H/tv+5/77/vv+7/7v/uf+q/5z/i/93/2//YP9R/0v/Rf84/zT/Kv8q/zb/P/9P/1n/aP9q/27/c/93/3b/b/9r/2r/bf9t/2//a/9h/1n/Wf9X/1n/X/9d/1z/Wv9c/1b/Uf9O/0f/RP9C/0T/R/9D/0L/P/84/zH/Lv8o/x7/Hf8X/xb/Fv8T/xr/Gv8Y/xr/F/8U/xT/Ef8I//f+6P7d/tP+y/7E/sT+wv7D/sj+zP7J/sf+wv7C/sL+yP7M/tj+3P7j/uf+7/73/v3+Dv8S/yD/Iv8l/zD/N/87/zr/Ov9B/1T/X/9r/3b/f/+I/5T/n/+u/8b/3f/1/xAAKwBCAGIAdgCFAJEAkgCWAJwAoACkAKcApAChAKUAqACrALYAvQDBAMgAzADNANMA0wDXAN4A4wDxAPwACQENARQBEwEXARoBIgEtATABOgE+AT4BPQE/AUABQgFLAU4BVwFgAWgBbQFpAWcBXwFWAVQBWwFmAWgBbgF3AYQBjwGbAaQBrwG0AbYBvwHBAcQBxQHAAbsBuQG2AbEBsQGsAasBoQGVAZABiwGKAZABlAGVAZwBpgGqAaoBpgGhAZ4BlwGYAZYBjAGMAYkBgQGBAYEBfwGEAYkBiAGQAZoBnAGgAaEBnwGiAZ0BkwGSAZABgAF2AWkBWQFPAUABNQEtAScBIAEcARQBEwERAQcBBAH6AAEB9ADwAPEA6ADfANEAzQDHAL8AsACpAKAAlwCNAHYAZgBZAE0AQQA7ADUAKwAiAB8AGQAUABAADAAKAAwAFwAbAB0AHQAjACsAKAAsACkAKQAqACwALgAsACcAIwAkACkALQAvADAANQA6ADoAQQBCAEIARABJAFMAWgBeAFkAVgBbAFoAVgBRAEsAUgBVAE0ASQBGAD0APQA5ADQANQA2ADsAPwBDAEEAOwA2AC4ALAApACUAIwAeAB4AGwAeACAAHAAaABgAFgASAA0AAgD5//b/7v/r/+v/8f/2//3/AgAFAAoADgAVABsAIgAqADEANgA8AEMASQBNAFAAVABYAFkAVwBZAFkAWABaAF4AZgBrAHMAegCAAIMAfwB8AH4AgQCGAIgAhQCIAIoAkACUAJoAmgCaAJUAlgCVAJEAlgCSAKAAqgCvALkAwADJAM8A0wDTANYA2ADgAOQA6wDxAPkA/QAGARMBGAEaASMBKQEmASYBKAEqASsBKgEvAToBPwE/AUUBSwFMAUwBSwFLAUsBRQE+ATkBMQEoARoBCwEBAfgA5QDWAMgAuQCrAJ4AkgCLAIMAegByAGsAYgBVAEkAPgAxACMAFAAFAPX/6f/d/9b/zf/D/7//tv+s/6L/nP+X/5X/lv+V/5H/j/+P/4z/jP+I/4P/gf96/3f/eP9z/2v/Zv9j/2D/X/9a/1b/VP9N/0b/O/84/zL/Kv8h/xP/Cf/5/uz+3f7N/sT+tP6k/pb+iP5+/nH+YP5Q/kX+Nv4s/iH+Ff4J/vn97f3l/eP93/3c/eH93v3m/ez95/3n/e/98P32/fn9+f34/QD++v0F/gP+CP4M/gz+Cv4C/gD+6f3g/dr91P3L/c391/3l/fr9//0L/gb+C/4N/hz+H/4v/j/+Rv5X/mX+a/5x/nn+fv6E/oj+jv6W/p/+nP6k/qr+sP64/rn+v/6+/tH+0f7X/uT+7P72/gT/E/8k/zn/Q/9a/3D/ev+F/43/mP+h/6v/rv+2/83/z//X/+X/8P/6/wgACQAMABYAEQAPAAcAAQANAAMAAQD1//T//f/7/xkAJgAoADMASABaAGMAaABiAFwAWgBcAF0AYABYAE4AOwAqACMAGQAKAAAA9v/e/8v/vf+u/6j/rP+v/7D/rP+l/6T/qP+h/6D/of+f/5T/iP9//3P/aP9b/1T/V/9n/3P/dv+E/4r/jP+M/43/k/+X/5j/l/+d/6T/qP+k/67/r/+q/67/nv+V/5f/j/+D/4v/hv+J/5b/n/+l/6X/pP+e/6X/n/+h/6P/ov+t/8H/2//x/wgAEgAZACIAJwAqACEAHgAfABoAIQAlACMAKQApACMAKQAoACcAMQAuACcAMQAhACsAOQA/AF8AYgB4AHsAfgCFAIgAjwCEAIAAfQCBAIEAggB2AG0AcQBrAGQAXwBVAFMAWABaAFUATgBVAF8AcwCHAJoAqQC7AMEAywDWAN4A5wDxAP0ABwEWASABHQEgARYBAgH6AOsA3wDeAOAA4gDmAPMABAEYAR8BLwFBAVABVgFZAV0BVQFRAUYBPAE+ATgBNwE0AToBQAFLAUoBUQFYAVgBWwFKAVgBTwFAAUoBTAFEAUcBVAFPAVoBUgFGAUQBQwE5ATUBKgEWARYBFgEXAR8BIgEVARUBDwH/APoA6QDfAN8A0QDWANsAzQDSAMwAxQDCALMAtgCrAKUAnACWAI0AfwBzAF0AVwBGADMAJwAaAA0AAwD9//X/8v/v//T/+v8DAAoADQALAAMA9//z/+b/2//Y/9n/0f/V/8v/yP/J/8H/z//O/9f/2v/g/+T/4P/e/9L/2v/k//b/AAADAAMA+v/5//v/+f/6//z/+v/3//n//f/x/+v/4f/O/9T/y//I/9f/2//d/+n/6P/y/wMADgAeACMALQAxADwASgBRAFIAUABKAEsAUABPAFwAWwBRAE8AVQBXAF4AagBtAHIAdwCCAIUAjwCYAJ8ApACxAMAAywDSANkA2gDSAMgAwwC/ALMAtQC5AL8AyQDOANYA3ADeAOIA4ADeANwA0wDDAK0AmQCIAHUAZABWAEwAQwA8ADcAMQAhABEAAgD4//X/9P/y//H/7P/i/9f/z/+//7n/rv+d/5b/hP92/2v/X/9M/0P/Nf8p/zD/Jv8m/yj/J/8m/yr/MP84/0T/Tf9e/2j/a/94/4T/if+P/5H/kf+R/5D/h/+B/3P/Yv9S/z3/L/8f/x3/Gf8e/yT/Jf8l/yT/Jv8h/x//E/8O/wb//v7x/uf+3/7f/t/+3v7i/tv+1f7L/sT+r/6n/qX+pP6n/qX+rv63/r7+wv7L/tT+yv7K/sX+yf7L/sj+zv7W/tn+3P7q/vT+BP8J/w//GP8X/yD/Iv8i/yL/Jv8k/yP/Jf8k/x7/Gf8R/xX/F/8d/yP/Kv8z/zb/Pv86/zj/OP8z/zD/Kf8y/zD/N/9F/0b/TP9Y/2P/cP+C/4n/kP+c/6P/rv+5/73/v//C/8f/yf/V/9T/1v/W/8z/yf/H/8L/wP/A/8b/yf/L/87/2v/n/+3/9/8DABgAHwAuADkAPgBFAEoAXABmAG0AcAB7AIIAhQCLAIcAjQCPAJAAmACiAK4ArwC5AMIA0QDhAPMA/AAIARMBIwEzAT4BSwFUAWABZwFsAW0BcgFzAXUBcwFxAXQBdQF5AYABiQGMAZIBlAGWAZMBkQGKAYQBfAF0AXABaQFeAVMBRwFBATcBLwEjARwBGQEUAQsBAwH8AO0A5ADgANgA1gDSANIA0ADLAMMAuQCuAKEAlgCMAIYAhACBAIIAggCIAI0AjwCWAJoAmQCWAJEAhgB8AHYAbwBoAGYAZgBmAGcAagBtAG0AagBoAGYAZABhAF0AWgBXAFMAUABMAEoASgBIAEMAPQA6ADMALQAnACQAIQAfAB8AGgAVAA0A/v/y/+v/5P/g/9v/1P/T/9X/2f/e/+H/4P/f/9n/1v/R/87/z//P/+D/5v/s//X//f8CAAAA///7//j/9f/x/+v/6P/k/+D/2//Z/9r/1v/T/9X/1f/M/8n/y//N/8z/zf/R/9f/3P/g/+f/7f/w//T/9v/2//j/+v/4//T/8P/r/+X/2//Z/9X/zP/I/8T/wv/A/8H/xP/K/9b/4f/q//T//v8FAAsACwAKAAkABQABAP7//P/8//7//////wQABAADAAMABgAIAA0AEwAZAB0AIQAoAC8ANgA7ADwAOwA2ADEALgAlABgAEQAIAP7/9//s/+T/2//Q/8X/uf+z/6r/of+W/4j/fv9w/2X/Wv9O/0X/N/8r/x3/E/8J/wL/+P7q/uP+1/7S/sz+x/7C/rn+s/6z/rj+uf7A/sj+yf7S/tb+1/7a/t/+4v7k/un+6P7l/ur+5P7r/uT+4f7e/tv+0/7N/sj+s/6o/p3+kf6F/oD+i/6X/qb+sf66/rn+vv7A/sT+xf7I/s7+z/7T/tb+2P7i/un+8/79/gX/FP8h/zT/OP9F/0n/Uf9W/1H/Uf9L/1T/Uf9V/2P/bf95/4n/nP+p/7r/vf/P/+P/5//0//b/AAAKABcAHgArAD4AOQA+AEYARQBMAFMATABJAEoARABDAD0ANQA8ADAAMAAlACcAMAAxAFUAXQBiAHUAiwCdAKUApgCeAJYAlQCYAJgAlwCJAH0AaQBcAFUATwBGAEMAPQAnABYACAAAAPz/AQACAAEA/f/3//f/+P/v/+f/3//W/8P/sP+g/4v/ef9m/1j/W/9r/3n/gv+R/5b/k/+Q/5H/lf+b/5j/l/+b/6L/pv+k/6//s/+y/7P/pP+Z/5n/kv+B/4j/f/+B/5H/nf+o/6n/rf+m/7T/sP+w/7T/rf+3/8r/4v/7/xAAFQAbACcAMAA4ADUAMQAyACoALQArACcAJAAhABcAFQAXABIAHAAdABIAGgAIAA8AGAAeAD4AOwBSAFMAVABdAF4AZgBdAF8AaABxAG8AcwBnAGYAeAB0AG4AaABdAFwAZABpAFoAVABSAFgAagB5AJMAoACxALYAxgDTAOMA6QDuAPQA8QD6APsA+AD4AOcA0gDMALwAtQC0ALAAsgC4AMYA3gDpAO8A/AD/AAQBBgEKAQoBAwEAAfIA6wDzAPIA/AD+AAgBEwEiASEBNAFDAUQBRgE3AT4BLQEmAS4BNQEzATcBRQFAAUYBPAEwASoBJwEWAQ8BBQH2APEA9AD1APkA+wD2APoA+wDyAO8A5QDcAN8A1gDYAN0A2ADdAN0A2ADWANQA1ADSAM0AyQDKAMcAwgC7AKsAqAClAJsAmACTAI0AhwCFAIQAhQCGAIYAhACIAJAAkQCVAJIAiwCMAIYAfgB1AG4AYwBhAFYATQBOAEsAVQBVAFcAUwBNAEsARgBBADEALwAvADcAPAA7AD0AOAA2ADoAPQBAAEMAPwA4ADUANAAtACsAJgAcACEAIAAhACsAMQA5AEUASABOAFwAZgBwAHUAegB5AH0AggCEAIMAfAB2AHEAbwBuAHIAcQBmAF8AXwBfAGMAbgBwAHIAdAB3AHYAdwB4AHgAdwB8AIQAiACIAIgAhAB7AHIAbABnAF0AWgBcAGAAZQBmAGoAbABtAG8AbQBpAGQAWQBKADQAHgAJAPT/4P/N/7//s/+q/6P/nP+O/4H/d/9u/2v/Zv9g/1z/V/9R/0//Sv9A/zn/L/8h/xf/B//5/u7+3f7P/sT+uv6v/qz+qf6p/q3+sv63/sD+y/7R/tz+5P7t/vX+//4P/xz/Jf8u/zT/N/86/z3/PP86/zT/K/8g/xD/Bf/5/vb+9f76/gL/Bv8K/w//FP8W/xX/D/8O/wz/C/8G/wT/Af8C/wT/Bv8J/wn/Cf8G/wX/+f7y/u7+6v7m/uD+3/7f/t/+3f7d/t3+0v7O/sn+xf7E/sH+wv7C/r7+vf6//r3+vf64/rT+tP6u/q/+rv6q/qj+pf6f/pv+kf6I/n/+dv5t/mz+av5w/nT+e/6H/o/+m/6h/qn+sf6z/rP+sv65/rn+vv7G/sz+0P7a/uT+8f7//gr/Ev8g/yX/LP80/zn/Pf9G/1//bP9+/4n/lf+k/6b/rf+4/8L/zv/V/+P/6v/1/wEAFQAqADYARQBWAHUAgQCZAKYAqACxALkAzgDdAOQA4gDtAPMA+AD6APEA8ADtAOcA7AD0APgA8QDzAPcA/wAHAREBFAEZASABKwEzATYBNwE6AUEBQQFDAUABQQE/AT0BNQEuAS4BLAErATMBOgE7AT4BQgFEAUMBPwE1AS0BJQEZARQBCwH8AOYA0gDCALUAowCPAHsAcgBlAFIAQwA1ACMAGwAYABIAEAARABYAGAAZABsAGAAYABQAEgAOAAgABwAFAAcADAAWABgAFQAYABcAFQATAAkA+f/x/+r/5P/h/9//4f/f/+H/5f/s/+//8//2//b/+P/5//n/+f/8//r/+f/2//j//f////f/7//q/+H/2f/T/9H/1P/Z/+D/4//m/+v/7P/0/wAADQAhADEAPQBNAGAAdgCPAKYAuADFAMkA2ADbAOIA7gDxAAcBEgEZASYBLgE3AToBPAE+AUMBRgFKAUwBTwFWAV0BXwFnAXEBdQF4AYEBhQF/AXsBeAFyAWoBYgFiAWcBZgFgAWEBYQFfAVsBWgFZAVYBTgFGAUABOAEwAScBIAEjASQBHAEZARsBGgEdASEBJgEuAToBRwFTAWQBcwF+AY8BnAGkAa8BswG4Ab8BwwHJAdIB2AHYAd8B3wHeAdgB0wHMAcUBvwG4Aa0BoAGWAYwBhQF9AXABYgFTAUIBOgEoARQBCwECAfMA6wDcAM0AwQCtAJUAgABvAGAAUABAAC0AIAANAP//8f/o/+3/6v/n/9z/2P/V/9f/2f/W/93/3P/p/+n/6v/r/+P/2P/R/8r/wv+w/5v/hv93/2z/VP9D/zL/I/8Y/xD/Bv/8/u/+4f7X/sr+vv61/qz+n/6W/or+eP5o/lv+TP46/i3+I/4d/hj+Gf4d/h3+IP4f/h/+Hv4g/iX+K/4u/jH+NP49/kH+TP5U/lr+Zf5x/oD+iP6R/pj+ov6r/rH+uf6//tH+2P7i/u/+/P4G/xD/Hv8p/zD/Nf8+/0X/Q/9D/0P/RP9E/0f/Q/9D/0b/QP9A/z7/OP81/zX/L/8q/yn/Jv8f/xn/E/8U/w3/CP8C//z+/f76/gT/BP8C/wf/Cv8O/xH/Ev8S/xL/F/8a/xn/Hf8a/xz/Hf8g/yT/Jv8p/yz/M/82/zf/PP9B/0f/Uf9b/2L/af9v/3b/gP+A/4P/if+J/4j/gv99/3n/d/93/3j/gf+Q/5//rP+5/8L/yv/R/9j/3v/h/+b/6v/u//X/9f/w//b/+v/3//n/+P/4//z//P/3//r/9//4////BgALAA4AFAAVABoAHAAgACYAKAAuADcAPQBFAE4AVABaAF4AYwBmAGgAbgB0AHcAeAB3AHYAdAB1AHEAbQBsAGYAYwBfAFoAVgBPAEwARgBDAEIAPQBAADsAOAA2ADIAMwAwADIALwArACcAJQAgABwAGgANAAAA9//r/97/1P/F/7T/rP+i/57/nf+Y/5r/mP+a/5v/of+k/6b/pP+g/5z/l/+W/5P/kf+K/4T/ff95/3P/cf9t/2v/bv9v/3P/eP99/4L/h/+K/4b/iP+N/4v/jP+N/4j/iv+R/5P/mP+X/57/o/+r/6//uf/A/8P/xv/F/83/zv/Z/+L/6f/u//b///8AAAMABAAEAAcACAAGAAsADwAPAA8AKQA9AEUATwBUAGUAbgBpAHQAdQB1AH0AfgCKAJsAogC0AMIAyQDTAN8A6ADxAPAA8wD6AP0A/gD5AO4A8wDvAN4A1QDLAMcAxgDCAMUAyADNANMA2ADjAPUA+AAHARABEQEcAR4BHAEgASIBIgEvAS8BMQE+AT8BTwFYAVwBXAFZAVcBWQFcAVYBXQFuAYQBlwGkAbIBsgG2AbYBtgGxAbYBqwGhAaABmQGLAYMBdwFkAWEBVAFFAUQBOQExATMBKwEjAS4BMQE3ATwBPwE8AT8BQQE+ATwBMgEhAREBBQH4APEA5ADMALYArQCkAKQApACdAI8AigCGAH0AdABvAGMAVgBQAFAATQBBADYAKgAZAAkA/v/u/9j/zf/E/8P/wv++/8L/w//D/8f/xv/F/8P/vf+z/53/iP90/2T/Vf9E/zn/Lv8m/yP/I/8c/xP/Df8K/wz/FP8a/yP/Jv8p/zD/Pf9C/0r/U/9T/1r/V/9V/1z/V/9N/0j/QP88/0r/R/9E/0z/Tf9Q/13/ZP9s/33/hf+V/6T/sP/C/9j/4//v//r//f8AAAYABwAGAAUAAwD9//n/9P/p/+X/4//p//T//f8AAAEA///8/////f8AAAUACwANABIAEAAQABMAGQAkACgAMAAyADIAKQAkACIAIwAmAB8AIgAgABoAEAACAPr/5f/U/8X/vP+5/6//qv+o/6v/pv+k/6T/qP+j/5X/i/+C/37/dP9n/13/U/9E/y//IP8V/wj/9P7i/tj+0P7P/tH+1f7c/uP+7v7v/vD+8/7w/ur+4P7b/s/+wv67/rP+pP6Y/pP+jP6K/on+g/6F/oH+ff6A/oX+i/6W/qX+r/6//sr+0P7Z/tX+0f7R/tD+zP7G/sP+uv6y/qz+rf6y/rL+t/69/tD+1v7n/uv+6/7v/vL+AP8M/xP/E/8Y/xv/Gv8Y/w//C/8F/wf/CP8P/xX/Ff8e/yT/KP8z/0P/Uf9f/3D/iv+f/7T/xf/d//f/BQAXAB0AJAApACsALQApACgAJgAnAC8ANQA5ADwAQwBFAEsAUwBTAE8AUABQAFQAVQBPAEQAOQA5ADgAOAA3ADYAQQBIAEwAWABhAGcAcAB3AIAAhwCRAKAApQCvALQAtgC9AL0AwwDIAMcAzADUAOQA9AAOARwBKAE6AUUBTAFVAVQBTAFMAUYBPgE6ATABKgEcARQBEgEQAQsBBgEBAQAB/QD/AAEBAwEJAQ4BEAERARIBFgEXAQ8BBAH7AO0A4gDYANIAzwDLAMwAwwDBAMMAvAC+AMQAxADKAM0AzQDQANUA2wDlAOwA6wDmANUAzQC8AK4ApQCWAJ0AlgCNAIMAdgBpAFgARwA6AC8AJgAgABcAEgANAAoACAADAAIA/P/5//j/9v/r/97/1//Q/8X/wP/B/8f/zP/R/97/7//+/xAAIgAxAEIATQBYAGEAZABoAGsAawBvAHAAawBpAGkAaABqAGwAcAB1AIIAjgCbAKoAuADEANUA3ADhAOQA5gDoAOsA6QDrAO8A8ADwAPIA8QDvAOoA5QDgAN4A3gDdANkA2QDcAOEA4wDnAOcA5gDnAOoA7gDtAOUA5gDrAPAA+wAAAQYBDwETARIBEAETARUBFgEQAQgBAwH4APEA5QDgAOIA4QDlAN8A5ADnAOwA8gDyAPcA/AAFAQYBBgEBAfcA5ADXAMcAtQCvAKEAkACIAIEAcABoAF0AUQBSAFEATQBOAEoAQgBFADQAKwAiABIA///x/97/zP+9/6v/nf+H/3z/dP9t/2z/av9t/2z/Z/9g/13/WP9W/1j/V/9Z/1X/Uf9Q/07/Sf9A/zL/Kf8i/xv/EP8D//b+6/7c/s7+xf66/r3+tv6y/rP+tv6x/rH+u/6+/rz+tf65/rj+qP6e/pj+kf6L/o3+h/6F/oH+c/5w/nj+cf5x/nX+b/5v/m/+a/5p/mf+Z/5u/mr+Zv5d/lj+Xf5b/nH+dv54/oD+iP6M/pL+kv6R/pP+mv6Z/pL+kf6J/ob+hP6D/oX+hf6A/oH+hf6D/oH+hv6L/pH+nP6m/q/+tf69/sf+1/7Y/t3+5/7t/u7+7v7s/u7+8/76/gX/Fv8w/0f/YP92/4j/mv+p/7b/wv/K/9b/4f/s//v/AwAFAA8AFgAXACIAJwAsADwAQQBCAE0ATABSAFwAZgBtAHIAdwB1AH0AewB9AIEAfgCEAIsAkgCaAKQAqACuALAAtwC9AL8AyQDQANQA1gDWANEAzgDNAMUAvAC0AKkAoACYAI0AhAB2AGsAYQBZAFMATABNAEcAQwBBAD4APgA6ADgAMQArACMAHQAVAAgAAADw/9r/zv+3/6L/lv+A/2r/Wv9L/0D/PP8z/zP/MP8w/y7/M/81/zv/O/86/zf/MP8u/yn/I/8Z/w3//v7w/uL+2v7Q/sr+yP7I/sz+0P7X/t3+4/7n/uX+5f7l/uH+2/7e/tP+0f7S/s/+0/7R/tb+1/7d/uH+6v73/vr+/P74/gH/+/4C/wv/Ff8d/yD/K/8w/zb/OP82/zz/P/8+/0n/U/9b/2L/fP+O/5v/rP+2/8n/1f/a/+n/+P8DABQAIQAyAEsAXgB4AI4AngCwAMUA1gDtAPsACgEaASwBOwFCAUMBTgFVAVcBXAFeAWIBZwFrAXABdAF8AYQBiQGSAZ4BoAGtAbcBuwHIAc0B0QHXAdwB3gHpAeoB7QH5Af0BDQIZAiQCKQIrAi0CMQI0AjECNQJCAlECXAJkAm0CbwJyAnICcQJtAnACagJmAmYCYgJYAlUCTQJBAj8CNAIoAiUCHgIbAhwCFwIPAhcCEwIRAgsCCQL9AfcB7AHjAdoBywG4AagBmAGIAXsBagFRATwBMAEjARsBFAELAf4A+gDyAOUA2wDUAMgAvQC1ALIAqQCcAJEAhAB2AGUAWQBNAD8AOAA2ADUAMwAuAC4ALQArACoAJQAiAB4AFwAPAAAA8P/h/9L/xf+3/6z/of+X/5D/i/+A/3X/a/9l/2T/Zv9n/2j/Zv9i/2L/ZP9h/2H/Yf9e/2H/YP9f/2P/Yf9e/2H/X/9k/27/bf9z/33/g/+L/5f/oP+q/7v/xv/U/9//5//3/wkAEwAgACoAMAA4AEMASQBNAFEAVABUAFMAUQBKAEgARQBHAEsATABIAEYAQgA/AD8APQA+AD8AQgA/AD8AOgA4ADgAOwBBAD8AQwBCAEEAOAAzAC8ALQAtACUAJQAjABsAEAACAPn/6P/Z/8z/w/++/7L/rP+o/6b/nv+c/5f/lP+P/4T/fP91/27/Zv9d/1P/TP9C/zX/LP8j/xj/B//8/vT+7P7n/uP+4f7f/tz+3f7Z/tX+1f7Q/sz+xv7E/rz+sv6u/qr+nv6X/pX+kf6S/pX+k/6Y/pr+mP6c/p/+nv6k/qn+pv6n/qn+p/6q/qf+o/6l/qf+pP6n/qz+r/6z/rv+x/7W/uD+6/7z/gb/Cv8X/x7/IP8j/yP/Lv81/zr/NP80/zX/MP8s/yT/IP8a/xn/GP8d/x//HP8e/yD/HP8f/yH/H/8h/yD/Kv8t/y7/LP8u/zL/Kv8p/x3/GP8P/wr/Bf/+/vv+9/76/gj/FP8e/yX/LP8x/zj/Qf9D/0L/RP9F/0j/S/9I/0L/Pf8+/zz/Of84/zL/N/82/zH/L/8y/y7/Nf86/z3/Qv9I/1b/Xf9m/2z/bP9v/23/cv94/3n/fv+B/4X/iv+X/6L/pf+q/6n/qv+r/6v/o/+c/5n/lP+W/5n/n/+i/6r/sP+6/8L/yv/W/+D/6v/z//j///8JABIAGgAcAB4AIAAjAB4AFgAPAAkAAAD4//n/+//7/wIABQAJAA8ADQANABIAFwAgACQAJQAtADcAQwBRAF4AZQBpAF4AXQBSAEYARAA2AEIAQQA6ADoANgA4ADYANQAyAC8AMAAvACsALQAvADMAOABAAEwAUgBaAGQAagBnAGAAWwBVAEwARAA+ADsAMwAoACUAJgAkACYAJwAoACwALAArACoAJgAgABsAEwATABUADAAHAAIA+//1/+//6f/l/9//2v/V/9T/0v/N/87/yv/F/8f/xP/G/8n/yv/P/9j/3f/i/+r/8P/1//b/+f/5//3/BQALAA0AEAAWACEAKwA3AEEARwBSAF0AbgB3AH0AjACZAKkAuQDDAM4A3ADjAOkA7ADzAPYA+wD5APgA+QD1APUA8gDxAPEA9QD5APgAAAEDAQsBEgEYAR8BLAE4AUIBTAFSAVQBVAFVAVMBTgFfAWQBYQFqAXEBaAFrAW8BcwF8AYcBkAGbAZ8BqQG4Ab0BxwHUAd0B5gHzAfgB9AH2AfMB7QHnAecB5AHnAegB7wH2AfoB/gEBAgECAQL8AQMCAQIAAv4B+wH2AfMB7AHkAdUBzwHFAbsBrgGfAYsBegFnAVIBQwExASwBGQEOAQQB9wDlANwA0gDHALsAqwCiAJMAfgBoAFMAQQAyACUACwD8//T/4v/V/8v/uf+q/6D/kP+F/3z/cP9n/13/Vv9W/0//Sv9D/zz/PP86/0b/Sv9M/1P/WP9c/13/Xv9b/1r/Wv9Z/1D/TP9E/0L/Pf86/zj/Mf8p/yX/Iv8c/xL/DP8H/wL/Af/7/vf+8P7s/uj+5/7a/tH+yf67/q3+nf6R/oj+ff51/nP+e/6H/pP+ov6w/rn+wv7K/tT+2/7e/uf+7v7y/vr++/78/gL/Bv8I/wr/Bf8D/wT/Av/7/v/+/P4A/wb/Dv8V/xz/Jf8q/zL/OP8//0f/Tf9U/13/ZP9s/3X/e/9+/4D/gv+B/4H/hP+I/4n/h/+D/4D/ev99/3r/e/98/3j/e/98/37/gP98/37/e/99/37/gf+L/4z/k/+Z/53/pv+t/7b/v//B/8H/x//I/8n/z//I/8L/w//D/8L/xP/A/7b/tP+x/7T/vf/D/87/1f/g/+b/8f/7/wMABwAHAAUA/f/6//X/8v/s/+D/1f/N/8H/vP+1/63/qP+l/6X/pv+p/6r/rP+t/6v/rP+y/7X/tf+5/7T/tf/B/8b/yv/P/9T/2//k/+r/+/8JAAwAEQAMABMAFAAeACoANQA9AEMATgBPAFUAVgBSAFMAUgBKAEwATQBIAEUASgBNAE8AVQBRAFIAVABJAEYAQwA5AD4APwBCAEsAUABdAGwAbwB0AIMAiwCWAJsAoQCrALUAuQC2ALMAugC9AL4AvgC+AMMAxADIAMwAywDLAM4A0wDdAO0A7wAAAQkBEQEhASgBLQE2ATgBOQFGAUQBSQFSAVMBZQFrAXABcgFuAWwBagFjAVYBUwFVAVwBXgFcAVoBVAFTAVgBXAFfAWUBYgFgAWIBYQFWAVABRgE5AUEBPwE8AUUBSAFKAVcBWAFcAW4BeAGDAY8BmgGhAawBtgG7AcABvwG7Ab8BugG0AbIBqwGaAY4BigGBAXoBgAF6AXMBdgF4AXEBcQFuAWcBYQFhAWYBZAFcAVgBUQFFATwBNwEyASgBKgEtATQBOgE5AT0BPwE9AT0BOwE5ATwBOQE2ASsBHgETAQkB/wD3APMA6QDkAOQA4QDaAM8AxQDAAL8AwwDHAMwA0ADQANUA3ADbAN0A4gDeAOEA2wDYANoA1ADRAM8AxQDBAMgAvQC5ALYArgCnAKUAngCcAJsAlACWAJMAjACPAJcAkwCWAJQAjQCMAIoAhQB8AHIAaABaAFEASAA4ADAAJgAjAB8AGgAQAAgA/P/u/+T/2//U/9H/0//N/8z/xf/B/7//vP+9/7f/tf+w/6z/ov+c/5n/mP+b/5X/mP+X/5f/k/+K/4f/e/9y/2n/Y/9j/17/Xv9i/2f/af9z/3n/f/+C/33/f/9+/33/e/92/2//af9g/1T/Tf9F/z3/MP8l/x//Gv8Y/xn/Gv8Z/xj/GP8S/w//Df8G/wD/+P70/u3+4/7c/tb+yf68/rb+rv6o/qf+nv6b/pr+lP6T/pb+lf6b/qT+qP6w/rX+tP63/rL+qv6m/qH+m/6V/pP+jv6H/oL+gP6C/oL+gv6C/on+g/6F/oL+e/51/mv+b/5x/nH+a/5p/mX+Xf5W/kr+Qf42/jL+Lv4w/i/+Kv4q/i7+Lf4y/jX+Ov5B/kb+U/5g/mX+bP52/oT+iP6N/ov+jP6K/oz+jP6K/of+gv6E/oz+kP6S/pH+k/6Q/oz+jP6H/oL+gf59/n7+e/51/m3+Zv5j/mH+Xv5g/l7+Y/5m/mT+Zv5s/m7+eP6A/oT+if6T/p7+qP6x/rf+vf6+/rz+wf7E/sj+zv7S/tj+3P7j/u3+7/70/vX+9f7z/vH+7P7l/uL+4P7e/uD+4/7l/ur+7f7w/vf+/P4F/w//GP8e/yP/Kv8w/zb/Pv9C/0b/TP9R/1T/VP9V/1f/V/9a/2H/a/93/4T/j/+Z/6f/r/+1/8D/zP/c/+z/9/8FABIAJQA0AEYAUgBbAF0AXgBfAFsAYQBhAG4AeQB9AIIAhgCJAIoAhwCCAH4AfAB/AIEAigCSAJsApACsALkAwADHANEA2QDXANUA1gDVANMA0ADSANoA3wDdAOMA6ADrAO8A9AD7AAMBBAEIAQ0BEAETARUBEgEVARcBEgERARABCwEIAQUBAQEAAfsA+gD6AP0A/QD7APgA9QDtAOUA2gDNAL8AsQChAJQAhwB1AGgAWABHADUAIgATAAIA8//j/9H/vv+r/5j/iP92/2T/V/9J/z7/Ov8z/y7/Kf8m/yb/Jf8i/x3/G/8V/w//DP8M/xD/E/8X/xn/IP8i/yf/LP8w/zv/Qf9L/1L/Xf9q/3r/iP+V/6X/t//L/9r/6f/0//v/AwAOABgAHQASAAsABgAIAA4AAwD+////+v8AAAgABQANABEAEgAeAB8AIgAjACsALwAxADQAJwAnACEAHwAVAA0ADQAMABQAFQAcABkAHQAhACQAJAAnAC0AMAA1ADYALwAvAC0AMQAwACoALQAqACoAIAAZAAkAAwD1/+X/2P/I/8v/u/+5/7b/sP+s/6v/sP+0/7f/sP+1/7j/r/+u/6j/p/+k/6H/l/+W/5v/kf+Q/4//hf+C/4L/ff96/3z/eP9z/23/aP9u/2n/Zv9g/13/Yv9f/2//cf90/3j/f/+J/4v/kP+N/4j/jf+K/4n/jv+N/4//jf+O/5L/lv+W/5n/mf+X/5L/k/+V/5f/oP+k/6f/q/+v/7P/vP+9/73/vv+7/7b/sf+v/6r/qv+p/6r/tf/G/9j/5//4////BgAOABYAHAAdAB4AHwAkACsALQAqAC8ALQAmACcAIAAcAB8AGQAPAA8ABAACAAcADQAVABUAFgARABUAEAAQABEACAAKAA4AGAAiACwAMQA1ADoAPwBGAEsAUgBaAF4AYQBhAFwAXABdAFcAVQBQAEQAQAA5AC8ALwAeABgAFAAPABgAEgAaABUADwANAAsADQAGAAgACAAOAA0ADAAJAAQACQD9//D/5f/V/8j/v/+y/5z/kf+J/4f/jf+R/53/pP+s/67/tv+9/8P/xf/H/8v/y//S/9f/3P/f/9r/1v/X/9T/1v/a/93/4v/q//T/AAAOABkAKgA2AEQAUwBhAG8AdgCBAIQAhwCSAJUAnACcAKMAqQCzALQAvgDGAMgAyQC9AMcAxADGANIA1ADVANwA5QDoAPQA9QD1APcA+gD3APsA+wD1APMACgEXARsBKAEsATIBNgEvATUBNAE4AUYBRAFNAV4BZgF1AYUBjQGbAakBtgHBAcMByAHRAdgB2wHbAdUB3wHkAeYB6QHqAesB7QHtAe0B6AHpAekB6QHrAfEB6wHtAeUB3AHZAcgBvAGyAaYBmwGaAYsBiAGRAYYBlwGWAZkBlQGHAX8BdQFoAU0BQAE3ATsBNgEpASIBDwH7AO8A4ADYANYAzQDBAL0AtgCrAKMAmACJAIwAgwB7AH0AeABzAHQAagBkAGgAZgBoAGIAXgBVAEsARAA5ACsAFwAFAPP/6v/c/9X/zf+3/6v/rP+o/6//u/+9/77/x//S/9L/2//h/9//4f/p//D/+P/z//T/9f/u/+v/6//q/+P/6v/z/wAADwAcAC4APgBSAGgAfQCQAKEArgC1ALkAugC6ALkAuAC7AMMAyADNANgA4ADmAOYA5QDoAO0A9QD8AAcBEAEWASMBMAEyAToBQQFAAUoBRAFEAUYBPQE0ATABHwEcAR0BCgEIAQUB/AD2APEA6QDqAOcA3wDmAOcA6AD2AAUBDAEdASgBNQFAAUsBTwFTAU0BRwE9ASwBIgEPAQgB/QD8APwA+QDxAOoA3wDRAMwAwgC4AK0ApgCZAI8AggB7AHcAcQBwAGgAZgBeAFIAPQAvAB8AEgAHAPT/6//g/9T/xf+y/6f/kf+C/3D/Z/9j/1//Y/9p/3P/e/+Q/53/sf++/8D/zP/P/9r/4P/h/93/2v/R/8f/yf/H/8T/vP+2/6//r/+0/7r/xP/F/8v/0//U/9z/4v/j/+n/6P/n/+f/4//k/+H/1f/J/7//tP+p/6P/lP+O/47/hP+C/4H/f/+F/5X/nv+t/73/wv/M/83/yf/O/8//yv/D/8r/xv+9/7r/vf/G/83/0P/Q/+P/4f/l/+T/0P/A/7H/rf+w/7f/r/+o/6X/mf+K/3L/X/9M/z3/Lf8n/yT/FP8K///+8v7q/uv+6f7q/uv+8v7x/ur+5v7o/vT+8/7+/vj++/72/vf+8/7u/u3+4v7h/un+7f7t/uj+7v71/vz+Cf8M/xL/GP8f/yf/K/8l/xn/Df8B//b+7f7l/tv+3v7c/t7+5f7v/vL+//4L/xP/G/8j/zX/Ov9C/0r/R/9H/0P/R/9H/0f/Rv9G/0v/Tf9b/13/Vf9W/1H/S/9L/0L/Kv8h/xP/C/8K/wr/Dv8L/w3/GP8k/y3/O/9K/1j/Zv90/3//jP+Y/6H/qf+t/7D/t/+7/7H/pv+e/5T/h/9//3v/e/9+/4f/jf+R/5H/i/+O/5H/l/+f/6T/qP+x/73/z//i//L/+//+/+//5P/P/8X/xP++/9P/2P/g/+n/8f/6//n/9//2//z/CAAPABgAKQA3AEMATQBgAHQAfACJAJkApACdAJUAkQCJAH4AcwBxAHkAeQByAHwAhgCNAJwApwC4AMcAzwDaAOYA7ADtAPEA7gD2APUA5gDfANcAzQDGAMAAugC4ALcAtgC8AMcA0ADWAN4A3QDbANoAzgDKAMYAwwDGAM4A1ADZAN4A3QDYAMsAuQCmAJIAgABqAE4ANQAjABQADAAEAPn/8P/n/+L/5v/g/9T/y//F/8X/zP/J/8r/z//N/8b/v//A/8P/yP/C/7f/sv+g/5L/gP95/3v/d/9z/2j/af9q/3H/df96/4f/lP+n/7D/uf++/73/u/++/8H/wP/G/8n/zf/Z/+j/6P/r/+3/8P/7/wUACQAQABEAEAAXABIAEAAOAAoAAgD6//D/4P/X/8n/uv+j/5H/hv97/3X/bv9s/2b/X/9Y/1H/R/8+/zb/L/8r/yL/GP8T/xD/Df8G//r++P76/vz+9P7r/t7+0v7G/rn+sv6q/rD+r/6x/rj+w/7K/tX+5f7z/vv+Av8P/xn/H/8m/yz/Mv8//03/W/9q/3f/ev9//4X/h/+N/5P/kf+V/5n/nf+f/6j/rf+4/73/v//C/8b/1P/c//b/CQAWACwAQwBYAGUAcwB3AH8AigCSAJsAogCoAKoArgC4AMIAywDOAN4A6ADzAPkA/gACARABGgEkAS8BOgFEAU4BWAFbAVwBXwFhAVsBUwFIAT4BOAEyATIBOQFHAVkBbQF9AYoBjgGUAZoBmQGWAZYBmQGfAaoBrAGoAakBowGcAZkBkwGKAYoBhAF7AXYBZgFdAVwBWgFZAVcBWAFZAVsBWQFbAV4BXQFfAWkBdAGAAYYBiwGKAYYBhAGEAYMBhwGHAYEBegFtAVsBRwE1ARoBAwHtANsAyAC8AKkAlwCDAHAAYABTAEkAOQAyACUAGAAPAAYA///4//T/7v/r/+T/4f/a/9T/z//C/7X/rf+g/5H/i/99/3D/bf9q/27/d/9+/4r/lf+h/6v/uP/B/8z/0v/V/9T/zv/N/8z/zv/L/8P/uP+u/6D/lv+I/3v/dP9t/2//bv9w/23/b/9s/2X/X/9Z/0//Qv84/x//Cv/4/uX+2/7J/sH+tv6t/qP+oP6f/pL+hv53/nH+Yf5h/mL+aP5u/nD+fP5//n/+fv54/nf+cP5q/mr+bP5s/nD+ev6F/pP+of6u/r3+x/7L/tL+2P7a/ub+6P71/v/+Cf8Y/yP/K/8t/zT/Mf8y/zH/Jv8g/xf/Cv/2/uD+y/68/qn+lP6C/nP+Y/5b/lD+Rv49/jf+Nf43/jX+L/4t/iX+If4g/h/+G/4Z/hv+Fv4Y/hL+E/4V/hn+Gv4e/iL+Hv4l/iv+M/4+/kX+WP5r/oT+l/6n/rP+u/7C/sv+z/7R/tX+1f7W/tv+3v7Z/t/+5/7s/vv+BP8O/xz/Kf8z/0X/VP9i/3j/jf+i/7P/yf/X/+n//v8OAB4AJgAuADkAQQBMAFkAXgBiAGkAeQCHAJMApwC1AL8AzQDdAOUA7wD5AP4AAwEPARgBIAEnATIBPgFEAUgBTwFUAVYBXAFkAWwBdAF5AYQBiAGJAYsBiwGJAY0BiQGFAX4BdgFvAWsBZAFfAV8BXgFdAV8BYAFfAV4BWgFaAWIBZwFuAXgBgQGKAZcBoQGqAbQBugHAAcgByAHMAc8BzQHKAckBwwHDAcUBwAG/AcEBvQG7AbkBtgG0AbIBrAGrAaYBoAGeAZ0BlQGMAYEBbgFeAUsBNgEeAQgB8QDXAL8ApgCIAGwAUQA+ACsAGQANAAUA+v/w/+b/3//Z/9X/2P/V/9b/1f/Z/93/2//d/9r/2v/W/9T/zf/K/8r/y//N/8v/zv/R/9L/z//M/8r/xP/E/8X/x//O/9L/2f/f/+b/7P/u/+r/5P/c/8//xP+2/6z/nv+Q/4L/df9n/1f/Sv83/yb/Dv/8/ur+2v7N/sP+t/6q/qL+m/6S/oz+h/6A/nv+cv5r/mT+W/5V/lP+S/5H/kr+Tf5S/lr+Xv5n/m7+dv5+/oj+j/6Y/p3+ov6p/q/+q/6q/qT+nv6Z/pb+kP6I/oj+gf5//n3+fv6F/o7+mP6c/q7+r/62/rj+tP61/rX+v/7J/tb+2/7g/ub+5v7o/uX+4/7h/uH+5v7t/u/+7v7v/vT+8f77/gL/D/8d/yj/PP9H/1T/X/9v/4P/jv+e/6P/qf+x/7j/vv++/8H/vv+8/77/vf+7/7P/rv+m/5//mv+U/4j/gv96/3j/e/9z/27/X/9b/1P/Sf9H/zv/PP83/zH/Mf82/zj/Q/9O/1j/Zf9p/3v/e/98/3v/b/9r/13/Xf9d/1r/Wv9d/2f/c/+K/5//rf+6/8X/yv/O/8//wP+2/6v/nP+W/4//i/+H/4P/g/+H/4f/iv+O/5L/lf+a/53/pf+x/7v/x//V/+D/7//+/wMABQAIAAcABgAEAAcACwAJABIAGwAkAC0AMAA8AEgAVwBrAHgAgACSAKQAtADKAN0A6QDxAOcA7QDlAOUA6wDqAPoA/QD6APkA7wDmANkAyQC7AK8ApACdAJYAlQCVAJkAnQCgAKoArwCyALwAwwC/ALsAuwC2ALEAqACoAK0ArQCsALAAtgC7AMMAygDUAN0A5ADsAPAA9ADyAPAA6wDqAOgA3QDVANIAzADIAMUAwwDBAMIAwwDFAMgAxgC/ALgAqgCXAIUAcABgAFAAQgA4ADMALwAqACoAJwAiABkADwAGAPz/9f/t/+P/3P/Z/9j/2f/Z/9f/1f/V/9X/2P/U/8//zv/Q/9b/3f/g/+T/6f/m/+H/2v/Y/9X/1P/J/7v/tf+o/6H/nP+e/6T/qv+y/7j/w//K/9T/2//c/9//3//i/+P/4f/f/9f/z//M/8f/w/+7/7X/sf+t/63/ov+b/5f/kf+U/5j/mP+e/5//o/+u/63/s/+4/77/wf/E/8n/yf/O/9L/1P/U/9j/4f/s//b/BQAUAB0AKQAxADwARgBMAFgAYQBnAG0AcgB4AH4AhACHAIcAjwCVAJsAngCcAJYAkACFAHgAbQBeAFsAUQBMAE0ASQBIAEwAUwBcAGMAZQBvAG8AbQBtAGwAcABwAHUAdAB4AIIAgwCJAJMAkQCVAJwAlwCaAJ0AlgCXAJQAjQCOAIIAgwB4AG4AbgBjAHQAbwBmAGIAWwBRAEQANQAkABMABAD1/9//zv+8/67/n/+Q/4L/dv9o/13/Uv9F/zT/Jv8a/xL/Cf///vX+7f7l/t/+2/7N/sH+tv6m/pX+gf5x/mL+Uv5H/j/+QP5K/lP+YP5o/m/+df52/nn+eP5y/nD+bf5j/lv+Tv5C/jv+Mv4p/ib+Hf4W/hL+D/4K/g7+C/4O/hH+Fv4T/hH+Ev4S/hP+E/4Q/g/+DP4N/hH+F/4h/iz+M/45/kH+S/5U/l/+a/51/n7+gf6E/of+iP6Q/pL+k/6S/o/+jv6P/o3+jv6G/oT+fP53/nH+bv5y/m7+b/5w/m/+b/5y/nf+eP54/nf+ef50/nD+a/5h/ln+VP5K/kD+OP4r/iL+H/4X/hn+Hf4h/ir+NP5D/lL+ZP54/on+lv6d/qH+ov6l/qT+o/6g/pz+l/6T/pH+mf6e/qf+s/6//sv+1f7g/ub+6/7w/vD+8v71/vP+8P72/vH++P4A/wT/D/8S/xv/Hv8k/yn/Mf85/zf/Nf8v/zT/L/82/z//R/9P/1L/Wv9d/17/XP9Y/1n/Wf9X/13/Yf9m/2r/cP92/37/hf+I/4//kv+Q/5T/lP+R/5r/mv+n/7r/xv/f/+//+P8HABYAHgAwADgAPgBPAFgAYwBmAFwAXQBbAE8ASgBBADYAMwAyADEANgA8AEgAUgBfAHUAfQCKAI4AjACTAJMAjwCNAIsAhwCNAIgAhACEAIAAiwCJAIgAfwB1AHIAbQBoAFUAUwBVAF4AYgBgAGMAYgBnAHEAeAB/AIEAfQB7AHcAdQBpAF8AUwA+ADcAKgAXAA8ABgD8//z/9f/s/+//8//4//z/AQACAAgAFAAfACkALwA1AD0ASABTAGIAagBsAHIAfACIAJEAmwCiAKYAsQC+AMIAywDWAN4A5gD3AAgBFAEbASQBKgEqASoBLQEuAS4BOAFIAVsBbAF7AY4BnQGoAbIBuAG4Ab4BtgGuAZ8BjAF7AXABYgFaAVcBUQFMAU4BSwFHAUEBNwExATEBMgEyATUBOAE4AT0BPQE1ATIBLQEjASEBFwERAQ8BBwEDAQMB/gABAQwBDAEUAR4BIgEnASwBMAE3AT4BRQFTAV8BagF8AY4BlgGkAawBrQGyAbMBrwGqAZ8BlAGFAXkBbgFnAWMBZAFlAWsBcAFwAXABbgFrAW8BbwFxAXEBdwF4AXkBdQF3AXcBfAF9AXUBdAFxAWoBYgFdAVgBWAFcAVsBZAFrAXMBeAF5AX0BewF7AX0BfwGHAYkBjgGSAZkBmQGcAZsBngGZAZMBkgGKAYEBeAFsAWEBWAFNAUIBOgExASMBFAEKAQMB+wD4APcA+wD6AP0ABQEDAQQBBwEEAQUBAgH/APwA8wDtAOQA2wDXANAAzQDNANAAzADPANQA1ADUANUA1ADYAM0AwwDHAMMAuQCwAJ8AkACJAHkAawBfAFUASQA8ADAAKAAlACgALgAzAEEARABLAEYAOQAtABwADQD6/+b/z/+7/6b/kP99/2b/VP9G/zz/NP8r/yL/D/8B//L+3v7O/sL+tP6r/qT+o/6j/p/+m/6b/pv+lf6S/ov+hv5//nf+bP5k/l3+Vv5Y/mH+Z/5v/nj+g/6M/pL+m/6d/qD+o/6k/qf+pv6f/pj+kf6J/oL+ev5y/mn+ZP5h/l/+YP5k/mf+cf55/n/+iP6Q/pv+nf6c/pr+jf5//nD+Yf5O/j7+Mv4u/iv+Kf4x/jj+Pv5H/lL+VP5S/kz+Q/4//jf+Mf4v/jD+Mf46/kb+W/5s/oT+mf6x/sP+1v7o/vn+Cv8a/yn/Of9I/1b/Z/91/3v/fv+D/4f/if+N/5b/nv+n/7H/uP++/8T/zP/U/97/6//7/wUADQAWACIALQA5AEMARgBGADoANwAxADEANgA4AE0AWgBnAHYAfwCMAJIAlACYAJwAoQCpALEAvwDQAOEA8AAHAR0BLAE8AU0BVwFXAVUBUgFKAUEBNgEwATMBMwEyATUBOQE7AUABSAFNAVIBUgFSAVABSAE+AS8BHQEOAfsA4gDPAMAAsgCmAJ0AlwCVAJYAngCnALEAuwDCAMsAzQDOAM4AzADMAM0AzwDXAOMA8AD5AAMBCwEQARIBFAETARIBEQESARMBFgEbASIBLAE6AUQBTwFXAV0BYwFkAV4BXwFgAWMBaQFrAW8BdQF3AXYBdAF2AXUBcQFnAVsBUAE/ATABIAEXAREBCwEIAQMBBAEIARIBGgEfASkBMgE9AUMBRgFGAUABNQEtASQBGgE9AVABUQFtAXYBdgGBAXcBbgFnAVoBTQE7ATEBEgEKAeMA0gDAAKUAnQCUAJMAjwCOAIUAhwB/AHMAhQCCAIMAiQCOAIMAbwBsAFUAOwAwACMADgD+//T/2f/I/8P/vf+x/7b/yP/W/+n/9v8EAAgAFgAfABkAHgAOABQABQD6//z/8P/d/9r/8v/v//r/BgALABIAGAAaAAYABAD8//D/3f/d/9j/sP+s/6n/mf+c/67/pf+p/67/nP+e/6P/of+p/5j/mf+H/37/gf9m/4T/df9v/3T/c/9m/1T/Wf9E/z3/P/9B/0T/T/9V/3L/i/+i/8n/3f/x/wwAGgAbABMACAD4/+3/6f/g/9v/1v/L/8j/xP+//8b/y//V/9H/1v/Y/9b/2//T/9P/3v/q/+X//f8IAPb/7//w/9//xf/L/7f/r/+4/7r/p/+d/6z/nv+X/7H/p/+e/7H/p/+K/5T/e/9z/3P/dP9l/1r/U/9M/2P/UP9U/1X/Uf9X/13/aP9x/3j/ff9+/4D/hv+J/4n/g/+E/2//ZP9V/0L/Nv8z/yv/MP86/zf/S/9T/07/d/9g/3X/ff9s/33/Tf9g/0f/PP81/yT/G/8E/xD/Bf8a/wz/H/8a/xP/M/80/zz/Vf9o/2z/fP+N/5D/nv+c/5z/ov+T/5L/hv93/2L/W/9R/1X/Uv9a/2T/av99/3//hP+L/4X/ef99/2f/Xf9H/z3/M/8p/yX/Iv8m/xn/KP8w/yj/Mf80/zj/MP9A/0L/Rv9U/1X/Y/9X/2X/X/9j/1b/YP9f/0j/Rv81/0P/MP9A/1z/c/+S/6//4//8/x0AOgBHAGAAcQB1AIAAfgB5AG8AUQBBAEQAQAA1ADAALQAlACcAHwARABoACQAGAAgA//8DAPv/8P/k/+H/1v/O/8j/u/+3/7L/p/+Y/4L/ev9y/2H/Wf9P/0f/Q/9C/z3/P/84/zf/Nf82/z3/OP83/zT/K/8m/yb/H/8e/yD/G/8j/xj/GP8f/x//Kv8m/zD/Jv8p/yj/IP8m/xb/IP8j/zH/Ov8+/0n/RP9N/1H/VP9f/1z/Uf9H/z3/Qf80/zf/Pf8y/0b/T/9U/3D/gP+L/6j/tP/F/9r/5f/2/+z/9f/2//b/AgAKABQAGwArAEEAWABoAIYAmACbAKkAtwC/AMIAxgDDALkAvADLAMMAxQDRANYA2gDpAPoAAwECAQ4BGAEWARcBIAEgARcBIAEpATIBPwFIAVIBXgFlAW0BdQF8AYQBhQGJAYIBfAF+AX4BfQF/AY0BkgGXAaUBsAG3Ab0BxAHIAdUB2QHaAd0B3wHaAdUBzgHFAb8BuAGwAbIBqAGrAacBnwGXAZUBgwF9AX4BZwFdAVUBQgEyASEBEAEQAQQB8wD+APgA8AD8AAMB9wD1AOsA3QDXAMgAuQCqAJgAjwB/AHIAawBeAFoAUgBVAFYAVwBWAFQAUwBNAFEAVQBWAFkAZQBpAGsAawByAHIAaABiAEwAPQArABcA/P/r/+H/3//j/+D/7P/w//b/+f/1//v/6//i/9L/uv+t/53/kv+F/3//dv91/3n/f/9+/3z/hP96/37/ff95/3X/dP9t/2b/a/9u/3D/b/90/3b/ff+F/5H/mf+a/6P/q/+m/67/tf+4/8H/w//M/9b/2v/d/9j/0f/M/8b/vv+6/7j/rf+w/67/qf+w/6//sP+3/8T/x//P/9D/yP/G/7n/sv+w/67/rP+k/6n/rP+y/7v/yP/X/+T/+f8JACIAKAA2ADsANQA7ADoAPwBBAEkARwBJAFEAUABUAFAAVQBaAGMAcACAAI4AjgCWAJYAkwCUAJgAlQCWAJYAmwChAKcArwC3AL4AvADCAL0AuwC0AK0AogCSAIwAgAB4AHkAdgB3AHoAgQCJAI4AlACUAJUAlQCXAJcAkgCFAHUAZABUAEUANQAqACEAIAAcAB0AIwAtAC4AOABDAEoAVQBYAGMAZgBiAF4AVABNAD8ANwAuACIAGwAXABcAGQAnAC4AMwBFAFEAWgBlAGsAYwBhAF8AUgBKAEAANwAuACUAIQAfAB4AHgAiACgAKgAvADcAPgBGAE0AUgBTAFQAVwBXAE8ASABBADkAMAAnACUAIwAfACAAGQASAA4ABAD///f/7v/n/93/0f/K/8f/wP/A/7z/sf+n/4v/ev9h/1P/R/88/z7/L/8j/xX/A//1/t/+zf66/qj+nP6K/n/+fv5+/oD+e/56/n7+ev53/nf+d/5o/l7+Vf5N/j/+Mv4q/i7+LP4v/jj+Qv5J/lj+af55/on+lv6h/qf+pv6g/pv+lv6Q/on+gf5+/nX+a/5h/lz+V/5Z/lz+X/5f/mD+YP5f/l7+W/5b/lr+Uv5L/kf+R/5M/lr+a/56/oz+nP6q/rb+wv7K/tH+3P7j/uf+6/70/v7+CP8W/yT/L/88/0v/WP9n/27/ff+N/5v/rv+//9f/7v8GABYAJwA9AE8AZwBuAHYAfQB/AIYAhQCMAJMAmgClAKQAtQDBANMA6QD3AA8BJgE7AUsBWAFhAWEBXwFdAVgBTwFOAUcBPAE8ATwBMgEzATEBMQE3AUIBRAFMAUwBSQFUAUcBQgE5ASoBGwEOAQQB8wDrAOEA3gDYANwA6wD2AAkBHQE2AUEBRAFFAUIBOgErASQBFgEMAQEB+QD4AP0ABQEHAQcBDQEUAQ4BAQHwANcAvwCgAIIAbABRAEYANQA0ADQAOgBIAFQAbAB4AIEAewB5AG0AXgBTAD0ANQAyADMAMQA+AFAAUgBgAG8AcAB1AHgAbQBfAFkAQwA9ADQAKQAsACEALAApAC4APwBAAF8AawBxAHwAigCLAIEAeQBqAGIAYwBnAGUAZgBeAF0AWQBNAEwARAA8ADUAMgAsACUAJAAmADMAQABQAF0AbAB2AH4AgAB3AHEAYwBZAEQANAAjABIACgAAAAAABQARACAAMgA/AEEAQAA9ADYAKQAfABYAEQASABEACAD7//D/4v/S/8T/s/+h/5b/h/90/2b/U/9L/0r/Sv9E/z//Qv9D/0f/Q/9F/0T/Pf86/zj/NP8z/zX/Nv8z/zP/N/86/zv/NP8t/yT/Fv8I//f+6f7f/tL+zf7M/s3+0f7e/uP+8v70/v3+/P76/vX+4v7f/sf+tv6j/or+d/5c/lf+Rf5F/jz+P/48/j3+Qf5E/kr+T/5S/kv+Q/48/i3+LP4l/iT+Kf4o/in+Kv4r/jH+Qv5S/mn+cv58/oD+fv6E/oP+hP6K/ov+jf6Q/pX+pP6w/sL+2f7v/gT/Gf8v/zz/Tf9b/2T/a/9t/3D/bP9y/27/c/9+/4f/lP+U/5z/mv+c/5n/mf+b/5f/mv+X/57/nv+n/7b/xf/V/97/6//s/+z/6//i/97/2//U/9f/2//h/+b/9v8NACEANQBFAFYAXwBdAGEAXQBNAEgANQAyADEALAA6AD8ASwBWAGoAewCUAKMAqwC/AMUAywDAAKoApACWAIAAbgBfAFQATwBMAEgATQBSAFgAVwBYAFoAUABIADoAKAAfABIAAQD4//P/5P/q/+H/3f/i/9j/2v/T/8r/tP+f/4b/cP9U/zD/I/8U/xb/Fv8N/wz/BP/9/vv++f7v/vD+3v7L/rz+p/6U/oX+dP5j/mP+Wf5Q/lL+Vv5Y/mj+d/5//pb+qf6w/rv+wf69/r/+wf7B/sD+t/60/q3+rv6x/rz+xf7C/sz+3f7t/gD/Ff8i/yL/Kf8q/yb/If8j/yT/Jv8x/zv/R/9L/1L/W/9b/2D/av9z/3j/g/+R/5v/qf+5/87/5f/4/wgAFgAgACsANwA5ADEAKQAfABcAEQAPABgAGgAnAEAAVwBqAHwAhQCSAKIArQC4AMQAygDKAMwAyAC8ALgAugC9AMgAwQDHAM8A0wDaAN8A2ADkAPIA7AD2APQA8QDvAOYA3QDeAOEA5QD4AAcBGAE5AVkBbQGJAZUBmwGiAZ4BlgGQAXwBbQFWAUABMgElASoBLgFBAVoBcQGIAZsBrQGyAbsBswGwAaUBoQGVAYMBbQFfAVQBUAFUAU0BUgFaAVoBUgFQAVABUAFSAUwBTQFMAUoBRAE7ATUBJwEbARQBEQESAQwBBwEFAQUBAgEKARUBJwE1ATkBSAFTAV4BZgFoAWsBdAF4AXMBeQF9AXcBcQFtAWUBZAFkAWoBcwF9AYwBngGpAbYBxAHSAd0B5wHxAfMB6wHnAdMBwQGxAZoBiwF/AXUBcQF2AXkBfQGEAYQBhwGJAZMBkQGPAYwBfgFxAWABVAFQAUMBOwE1ATYBNgE4ATkBPQFIAVABXAFjAXsBfAGBAYIBdgFwAWkBZgFcAVsBTAFEAT8BNQEvASEBFgEVARQBDQEUARYBDQEIAfwA7wDhANwAywC+ALIArwCpAKMAqgCvAMAAwQDOAMYAxAC8ALEAoQCHAHkAYQBOAEUANgAwACkALAA0ADMAOQA2ADcANgAzADIAKQAZAAQA8v/f/9X/x/+//7n/vP/A/8L/zP/S/9D/1v/R/8v/xP+2/7n/rP+f/5T/gf94/2b/Xv9R/0L/Of8x/yz/I/8v/yn/KP8z/zH/Lf8u/yf/D/8I//v+5v7Z/sn+wP6x/qf+pf6h/pr+l/6U/pD+jv6P/pH+lv6b/qH+qf6t/rL+wf7J/sn+x/7D/sH+tv6w/q3+qv6m/qT+n/6c/pL+if6I/ob+hP6A/n3+d/54/n/+gP6I/oj+gv58/mz+Z/5b/ln+Xv5d/nP+cv52/nj+dv50/nD+Z/5d/ln+W/5R/kj+T/5b/mX+a/6A/pX+oP6r/rb+wP67/rr+u/64/rD+q/6o/rD+sv66/sf+3f7r/gP/Gf8s/0H/Sv9R/1T/Uf9P/0r/R/9I/0X/QP9D/0L/Q/9I/0n/S/9Q/1z/Yf9n/3P/ev9+/4r/iv+K/47/if+M/5H/lf+l/7r/yv/Z/+r/9//+//7/AAD+//r//f/9//v/AQALABcAIQA2AEMATgBcAGcAcwB4AG8AbwB2AHsAiQCMAJsApwCyALgAtwDHANAA3QDaANYA0gDEALsAqACoAKcApQCmAJgAqACoALIAvwDBANAA1gDfAN0A3QDYANIAwwC9ALUAqgCZAIkAhACAAIEAdQBxAHAAbwB1AHwAhACLAI4AjQCRAIgAfQBuAFoAQwAvAB4AEAAFAPz/+f/y//H//f8GABUAKgA/AEoATwBTAFEASgBAADoAMgAsACUAJAAoADQAPwA/AEEARQBRAFYAVQBMAD0ALAARAPv/5P/L/7//sv+w/7f/wP/H/9L/5v/w//n/+P/+//7/9P/u/9z/1P/P/8//1P/i//z/CgAdADQARABbAHQAewB9AIUAfACAAH8AeACAAHMAfQB7AIIAngCpANcA9wALAS0BSgFeAWQBawFjAV0BVgFTAUYBOgEqAR8BGAEPAREBCwEFAQsBDQELAQoBBgEDAQoBDQEQARQBFwEaAR8BIgEbARYBGAETAQoB+QDjANEAzADGAMoA1wDzABgBPgFhAYEBlgGpAbkBuwG4Ab4BwQHGAdMB0wHJAckBwQG3AbYBsQGrAbYBugG2AbsBrAGqAbABsAGwAa0BsAGoAaMBlAGGAXkBYQFQAUMBOAE0ATMBLwEuASwBNAE6AUEBRwFJAUYBOgEpAQ8B9wDjAMoAtQCoAJ4AmACfAJ4ApACiAKEAnACVAJEAgAB7AGsAXQBUAEkAQwA3ADUANAA2ADgAPgBAAEQARwA+ADIAJAASAP3/6f/O/67/mv+D/3b/cf9r/3H/d/+D/47/o/+4/83/2v/i/+P/2f/T/8j/u/+p/5D/d/9g/03/Rv87/zj/P/9J/1z/cf+H/5P/o/+r/6v/p/+h/5T/fv9t/0//OP8q/x//I/8f/yX/Jf8p/yr/Mv84/zb/NP8o/yj/Gf8W/xb/HP8g/xz/If8a/xT/DP/+/vf+8P7j/uH+3P7a/tr+2/7e/ub+8f74/gX/Dv8N/w7/DP8F/wr/A/8F/w3/Ff8n/zD/Nv85/0H/Q/9J/0z/Sf9O/1P/VP9M/z3/Mv8v/yr/I/8c/xD/Cf8L/wb/Af/+/v/+A/8P/xb/Gf8e/yH/Jf8n/yn/Iv8b/xX/EP8J//3+9P7p/uP+1v7S/s/+xv7H/sL+vv6+/rn+wf7H/tH+2P7Y/tn+2f7Z/t3+4P7g/tr+0f7G/sD+tP6i/pf+jP6C/oL+fP58/n/+iP6M/p7+sP7B/tn+5P7w/vT++P7x/u/+8f7w/uv+4v7m/uv++P4H/xj/Jv8s/zr/SP9S/1X/W/9Y/07/U/9P/0n/Q/9E/0P/Sv9U/1z/Zf9o/3P/ff+C/4z/lf+b/53/pf+u/7v/x//R/9z/5v/v//f//P///wIAAgD+//P/6//h/9z/2f/R/9b/0//Z/+X/5//n/+f/4f/g/+P/4P/e/97/4P/i/+7/+P8AABEADwAUABUADgATABEACwAHAAoACQAUABsAEgAVAA8AEQAUABUAFAAZABkAIQA1AEAATgBmAHgAggCMAI4AkgCWAJUAjACDAHQAbwBjAFMARgA0ACYAGQAWABIADQALAA4ABQAFAAgACQAFAAAABQD+//3/9f/x/+j/3f/Y/8b/v/+2/7T/r/+t/6z/rf+y/7j/wP/E/8j/yP/F/8X/w//F/8T/xP/J/8z/1v/i/+///P8NABkAJQAqAC4ANAAxADcAOAA3ADIANQA/AEgAUwBXAFsAXgBsAHMAegB8AIEAiQCHAJAAmgCaAJ8ApgCrALYAuADAAMkAxQDFAL4AvQC/AMEAwwDKANEA1gDlAPMA/QAIAQ4BEwEdAQMB8QDtANkAvgCuAJQAgwB1AGEAUwBHAEIAOgA9ADsAOwA5ADkAPAA6AEAANAAxACgAFwAXABcAGwAbABwAFQAMAAEA8v/n/9T/wv+z/63/rf+v/7T/sf+3/8H/x//W/+b/8v/9/wgAEwAXABgAFgAaAB8AIAAkACYALAA0AEIATgBVAF8AZQBtAHkAgwCOAJkApACxALkAwADAAMAAvAC3ALYAtACtAKUAnACVAIoAfQB0AGsAagBkAF4AXABhAGIAbAB6AIMAkQCXAKUApwCpAKcAnQCZAI4AhwB+AGwAXABQAEwASQBTAFQAUQBaAEwARgBGAD4AKAAZAAkA9f/r/93/2f/R/8j/yf/N/8//1P/W/9b/0//S/9D/zf/O/8n/xf+9/7T/rv+m/5b/hv99/3T/Yf9T/0//Sv9F/0T/Qv8//zv/Ov87/z//Rv9M/1X/Wv9f/2v/dv+G/5T/mf+Y/5H/if+C/33/f/+E/5X/of+t/7b/wP/G/8T/wv+4/67/p/+d/5f/nP+j/63/tv+//8z/1f/d/+b/6//m/+H/4v/j/93/2P/Z/9//4//q//j/DQAdADAAQgBKAFQAVwBZAFkATgBFAD4AOAA3ADQALwAwADUAPABBAEcATQBUAF8AaQBvAHkAhACNAJkAnACeAKAApQCmAKMAnQCdAJkAlQCKAHkAagBXAEgANgAsACoALQA3AD4AVABpAIAAlACqALoAxADKAM8A0wDPAMMAvAC4ALsAwADFAMoA0gDRAM8AzwDQANMA0gDPAMsA0gDRANYA4QDrAP8ABQESARIBFwEYARgBFwENAQUB/gAAAfsA+QD4APQA7gDvAO8A7QD2APQA7wDxAOkA1wDIALYAowCZAI4AggB7AHYAdwB+AIAAhgCJAI4AjQCPAI8AiACFAHcAbgBcAE0ASgBHAEkAUABdAGwAfACLAJYAoQCnAKwArAClAJcAhwB2AGcAWAA+ACoAIQAgACEAJQAmACkAKwApACYAHgASAAYA9//n/9j/w/+v/53/kP+H/3z/dP95/3n/ff9//33/ff98/3v/cv9y/23/Yf9S/0r/Pv8//0H/P/9E/0r/Tv9b/2z/dv+G/4L/kf+M/5P/oP+Z/7b/xP/P/+b/AQARABQAJgAqADEAPwBGAEUAQQA/ADwAOgAxAC4AKAAeAB0AGgAYABYAHAAnADYARwBYAGsAfwCSAKcAtwDFANMA3QDeANUAwwCrAJMAfgBqAFMAQwA8ADoAPAA/AEYASgBTAF0AZQBtAHUAewB7AIIAfwBuAGcAXQBMAEcARAA/AEcASwBKAFIASwBLAFEAVgBRAEUAOgAnABoABgD1/+X/zf/E/8T/yf/Q/9v/4//z/wMAFwAoADgASABUAFwAVgBNAEEAOgA1ACoAIwAoAC0AOwBOAF0AbwB2AHsAdwByAGQATQA8AB4AAQDj/8L/pf+F/3b/av9g/1//X/9j/2r/b/9x/3H/av9i/1P/Sf86/yr/Hv8P/wr/Bf8C/wr/FP8i/zL/TP9g/3H/eP94/23/Xv9O/zr/Jv8O//f+4P7P/sP+xf7E/sv+3f7s/gH/FP8o/zH/P/9I/0X/Qf86/yz/HP8U/wT/+v7z/u/+9v7y/vT+7/7r/uL+2P7T/sf+uf6m/qH+kf6N/on+i/6P/o7+l/6Y/p/+ov6k/qv+qv6t/q/+sf6y/rL+ov6e/pz+nf6m/r3+0v7m/vb+//4E/w3/B/8L/wv/BP8O/w3/Cf8D/wj//P4F/xH/D/8h/zP/Nf85/zD/IP8d/wr/+f7m/sj+rv6r/qD+m/6o/rz+1v79/h3/Mf9J/1L/V/9a/1D/Q/8v/xX/CP8A//H+9/74/g7/I/9D/3D/jP+o/7L/tv+7/7j/sP+r/5v/kP+G/4r/kP+b/7j/yv/v////DwAgABgADgDq/8//qf+G/2f/R/87/y3/Kv8x/0v/bP+U/8D/5v8DAA4AEQAEAO//3P/B/6H/hf97/3X/ef+L/6T/y//1/ycAUABxAHwAfAB2AGoAVgApAAQA4f+//63/qf+q/7D/wP/T/+n/BAAeAC4AMwAqAB0ADQD3/+b/1v/J/8r/y//U/+n/AQAkAEwAaQCFAI8AjACAAG4AUwAyACUADgD7/+3/1P/T/+X/7f/9/wwAFAAhACgAJwAYAAYA9f/s/+T/x//F/73/u//Q/9T/5v8HADAATwB2AHwAdgBvAFsASwAsAAsA5P/S/8L/wv/N/9X/7f8MAC4ASwBmAH0AkQCgAKIAnQCdAJAAiAB4AGYAVwBNAEgARgBRAF8AbgCBAJEAlACWAJMAkgCSAIYAgQB4AHQAcABwAGoAbgB3AHsAigCdALAAwwDMAM4AzADGAMQAwgC7ALcAtwC1ALgAxgDQAOAA9QAOASMBNwFHAU0BSwFKAUABMQEjARAB/gDwAOMA2QDWANEA0wDiAOoA7gDwAO8A6wDzAOUA3ADMALcAuQCoALEAvgDEAN0A+AATATUBTgFdAXUBbAFlAVgBPAEqARoB+QDrANsAzADXAOMA7QACARkBLAFIATwBMwEyASQBEQH0ANsAxACrAJgAiQCBAHoAbAB9AI8AigCJAIUAfABqAF4ASAAxACQADAAIAAMA/P/9//b//P/8//X/8P/o/97/0f/D/7T/rv+m/67/sv+2/8v/3P/z/wYAGAAlADAAMwAwACQAFwALAP//+//3/wEAEgAqAEsAaQCNAK4AyQDgAPEA+gD6APIA4ADIAK4AjgBxAF4ASABCADwAOwA8AEEASQBOAE4ARwBGAEAAOAA1ACwAHQAdAB8AKQA3AEUAWgBgAGkAZwBbAE0ANwAcAP//6//S/7z/qv+Z/5D/iP+N/57/tP/L/9H/1//e/+D/4//d/9z/2P/Q/8X/uP+z/6r/q/+s/7H/vf/F/9L/3P/j/+f/6//s/+n/6v/o/+X/2f/S/8X/uv+0/7T/tf+w/7b/sv+y/67/r/+x/6r/n/+O/37/bP9a/0r/OP8p/x//F/8a/yH/Lf9A/1T/av+A/5T/p/+7/8j/zP/L/8L/tv+n/5T/fP9n/1T/TP9I/0f/Uf9i/3T/gv+Q/5X/mv+a/5H/hv94/2v/X/9Y/1T/U/9Y/2L/b/+C/5H/ov+2/8b/2f/k/+7/9f/6/wMADQAVABwAJgAuADYAQABFAEgATABIAEQAPwAzACgAIwAbABcAHwAoADcATQBmAIIAowDAANcA5QDzAPIA7QDeAMkAsgCWAHoAYABVAFgAZQB5AIgAnQCwAL8A0wDeAOsA8wDvAOcA2gDNALwAtACuAK0ArwCzALcAxgDRAM8A0ADIAL8AsQCiAJIAiwB/AHQAdgB0AIUAjwCbAKYAqACuALAAtgC0AKcAmwCSAIgAgAB5AHQAcQB3AIEAiQCcAKUArQC1ALoAuQCzAKsAoQCXAIgAewBuAGkAYwBhAGYAagBrAGsAaABmAGcAVwBLAEAALwAeAAYA+//z/+3/5v/r//z/EAAtAD4ASwBWAFcAUQBGADkAJAASAP//7P/Y/8b/tv+o/6P/n/+a/6D/pv+q/7H/qv+k/6D/mf+T/5D/jv+M/4n/ff95/3b/d/94/3f/cP9n/2b/X/9c/1X/Tf9I/0X/Uf9V/2b/eP+L/6L/sv/K/9//6f/0//L/5P/k/9b/yP+9/7f/tv+3/7//yP/b/+n/8/8FAA8AFwAaABoAGAAJAP//7v/p/+f/5//t//T/BwASACkANABBAEoATgBTAEUAPQAsACAADAD8/+7/3P/Y/9X/1f/P/9L/1f/b/+P/7P/x//D/+v/4//v/AAAIABMAJAA8AEoAWwBtAHsAjQCaAKIAogCgAJcAjACCAG4AYQBVAEwASwBGAEUARwBQAFoAZgB6AJEAogCjAKcApQCaAJAAfQBzAGgAXgBZAFUAVABTAE8AVgBeAGYAhQCVAJ8AqACsAKwAqQCnAJoAigCBAHcAaQBeAFQASQA/ADsAPwA6ADUAOAA3AD0AOgA0ADAAJwAoACIAHAATAA8ACQAMABEAFAAgACYALAAqADIANQAwACoAHgAJAPv/7v/b/8r/vP+2/7j/uP+//83/2P/r//P/+P/9//z/+//w/+T/1//M/7//tv+t/6v/pv+h/53/j/+M/3z/cv9n/1v/T/9E/zf/KP8i/xf/DP8H/wj/C/8T/xn/Hf8n/yj/If8b/wr/8/7X/sH+pf6S/of+d/51/mv+Z/5z/nz+hf6Z/qb+p/6s/qj+qf6r/qn+oP6g/p7+m/6h/qX+o/6p/qr+q/6w/q/+rf6y/rL+uv6+/sj+2/7k/vr+CP8i/yr/K/8u/yj/H/8L//r+7P7l/tv+2/7f/uv+9P4L/yD/N/9I/0//Wf9g/2H/Tf9J/0X/Pf85/zX/Of9G/1j/af+K/6P/xf/n/wcALQA/AFcAXABdAGEAWABNADcAJQAbAAsABgACAPz/AwAPABkAIgAoADIAPAA5ADMAJAAVAAcA/v/z/+z/5//Z/+X/6//2/wYACgALAAwADwD///z/+P/u//D/6f/r//T///8PAB0AKwA8AEkATgBNAEcANwAlABYABwD3/+v/3//M/8T/tv+0/7b/vP/U/+v/BAAVACcALwA1ADsAOQA6ADkAOgA2ADIAMwAsADIAPgBFAFAAYgBnAGIAXwBVAEsANgAoAAYA6f/Q/7b/pP+S/4//jv+U/6H/sv+//9j/7P/y//z//f/1/+b/1f++/57/if9z/17/Wf9Q/1D/Uv9Y/2L/c/+D/47/mf+l/7L/uP+9/7z/vf+//73/uP+0/7D/uf/E/9f/6f/0/wAAAwALAA4ADwAMAAUAAgADAAsAEwAWABQAHQAoACwANwA2ADEAMQAwACoAHgAcAB0AGgAYABYAHQAqAEEAVwBnAHgAigCfAKUArQCpAKQAowCfAJQAiQB+AHcAfwCFAIcAlACkALgA1wDwAAQBFAEZAR8BHAEXARABCAEDAQEBAgEJAQ8BEgEXAR4BIQEuATMBLQEiARcBBAHxAOAAzADBALsAtQC6AL4AyQDZAOAA6ADwAO4A7QDlANgAwwCxAKAAjgCEAH0AdgB6AIwAlgCkALsAzQDTAN8A5QDoAN4A0ADKAMgAvgCtAKkApAClAKEAmACZAJQAgQBsAFkARAAwACgAHAAPAAoABQAMAA4ACAAAAAAABgAIAP//7//2/wcACwACAPj/+v8DAAMA/P/+/wIABQALABIAGQAhACAAIQAcABcAEwAUABQAIQAsADYATABeAIAAnAC7ANIA5gD7AAEBBwECAfUA7QDhANQAxQC6AK4AowCoALEAvwDTAN0A7wD+AAkBCwENAQoBBgEAAfQA7gDsAOkA6gDvAPQA/AAKARcBHQEnASwBJgEfAREBAwH7APIA7wD2AP8ACgEXASMBKwExATMBMQEoASIBEAH/AOYAygCsAJgAjAB+AHoAfACIAJQAogCtALoAxgDLAM8AygC4AKQAhgBqAE8AMwAhABEACAAFAAUACgAWACEALQA2ADkAPAA4ADMAKAAcABAA/v/s/+D/0P/K/8X/vv/E/8v/1f/g/+z/8v/v/+b/3f/T/8X/u/+u/6D/j/94/2b/U/9D/zL/JP8U/wb///70/u/+6f7f/tj+yv69/rb+p/6a/o/+i/6G/oj+iv6O/pr+pv61/sD+zv7V/t3+3v7W/tL+yf7H/sP+wP6//rv+tf6u/qj+pf6f/p7+nP6c/pX+kP6T/pb+o/6w/sH+zf7W/tn+1P7Y/tf+2v7e/uD+3/7R/sf+vP6n/pj+gP55/nH+af5p/nH+f/6N/qD+tP7G/s/+2P7g/uD+2/7T/sr+w/6+/rr+wv7I/tv+9P4Q/yP/M/9F/1H/Wv9f/13/V/9M/0D/Mv8p/yj/If8i/y//O/9P/17/dv+K/5T/oP+o/6f/rP+u/6r/qv+q/6//u//N/9f/6v/w////EwAVACAAHAAXABUAEwAJAAkACwADAPn/6//r/+f/5//2//X/AgAOAA4AHgAnACsAKwAxAD4ANAAuAC8ALQAqABwAHAASAA8AHgAjADQAOABIAFUAYwB2AHgAbQBpAF8AXwBZAEoASwA8AD0APABCADQALgAbAAgA8P/Y/8j/pf+k/4n/fP+B/5D/rf/C/9j/5//4/wIACQAPABAADAD9/+r/1P/D/7f/nv+S/4v/i/+K/4v/j/+Q/6T/qv+1/8D/w//Q/9T/3f/d/9L/2f/i//X/CgASABkAIQAgAB4AHgAVAA4ACgAGAAEAAAD9//v/+f/8/wQABAD///3/8v/f/8z/vf+r/5r/i/95/3L/a/9s/3P/h/+Y/6T/sP+0/7r/u/+9/7P/qP+Y/3v/bv9m/2L/Wv9i/3D/fv+R/6X/w//d//X/+//3//v/+v/w/////v/q/+P/1f/Q/9P/2//Z/+L/8/8CABQAKQA9AEQASgBGAEIAQAA3ADAAKQAgAB8AJAAsADgAQwBNAFYAXgBpAG4AawBxAHgAgQCIAIwAmACbAKIAqwCtALEAtQCxALIAtgC0ALIAuAC1ALIAtwC4ALIAtACuALMAvQDGANAA1wDhAOkA8gDxAO8A6QDXAMwAwQC9AL8AvwC5ALMAtADCANoA8gALASEBMQFCAU8BWQFbAVUBUAFMAUgBRQE8ATwBMwEfAQ4BBAEAAfkA8wDwAPIA7QDcANUAxQC6ALEArACuAKkAsgC1ALQAtgC8AMEAwwDBALMAoQCLAGkATAAvABQAAADm/9z/0//I/8j/zf/S/9D/0P/N/83/y//E/7j/qf+f/5b/lv+b/6P/rP+y/7v/tP+2/7n/tf+k/5b/iP97/3f/cf9t/2n/ef+A/4T/ef90/2//Z/9i/1L/U/9H/0r/Uf9L/zr/Kv8r/yL/FP8C//r+9P7u/uT+2v7S/sz+x/7O/tX+3v7l/u3+9v73/vv+9f70/vD+8f7r/u3+7v72/v7+DP8b/yH/Kv8q/y3/Iv8b/xD//v73/uL+4P7f/t/+4v7w/gT/F/9B/2L/h/+n/7H/uP+v/6f/lP90/1v/Of8f/xT/Df8H/xn/LP86/1j/Zf9+/5L/rf/B/8n/2P/W/93/3P/a/9T/xP+8/7z/v/+//8r/yv/L/83/z//M/8H/tP+o/6D/m/+i/5//of+p/6n/of+S/4L/bf9U/z7/Lv8h/xf/G/8i/yz/Qv9Y/3P/kf+w/9P/7v///wMA/P/r/9b/wv+k/37/XP9K/z3/Nv8y/zf/Qf9V/2//f/+S/5//pP+u/6v/n/+P/3//bv9h/0z/Ov8u/yr/Mf81/0f/Uf9h/3T/hv+V/5f/jP98/3n/av9c/0v/OP8s/y3/M/9B/1H/ZP+F/6f/zP/n/wUAGQAmADIAKwAlAB0ADwACAPz/9////w8AIgBAAGAAggChALYAwQDNANAA0ADPANUA0wDTAM8AyQDNAM0A1gDgAOwACgEXASUBMAEtASMBGQEDAe0A1ADJAL8AugDCANUA7wARAS8BSAFmAXgBkAGhAaMBpAGYAYsBgwF2AWUBWQFMAU0BUwFmAXcBjQGoAcUB2QHqAfkB/QH5AfMB5wHTAcABuAGuAbABtAG6AcQBzQHOAdYB0wHbAecB3QHaAdQBxwHDAboBqwGiAZIBgwF+AXIBdQF4AYABhQGHAYsBkwGQAXEBYgFWAU0BMwEmARAB+gDiAMwArwCSAIgAegBoAFsATgBJADoAKwAfABEAAQD8//n/7//m/9j/0//L/8L/s/+k/5X/hP92/1z/S/87/y//Kf8p/y3/Lv8s/y3/K/8s/yr/JP8k/x7/H/8T/wj//P7r/uX+3f7f/tz+5v7s/vP++v79/gL/Av8D/wb/Bf8D//3+9v7u/uH+1/7S/sn+y/7P/tb+4f7j/uT+3f7d/t/+3v7q/u/++v4H/xn/Lv9M/2L/g/+g/7P/yf/O/+D/3f/Y/9H/wf+1/6D/lf+R/4j/h/+N/5f/p/+9/9T/4v/3/wgADgASABMACAD9//f/6//g/9n/0v/I/8T/wf/J/9P/1f/a/+H/2//X/9j/0v/T/9n/3f/c/+L/6P/w//f//f8JABMAGQAfAC0ANgA9AEMAQQA/AD8APAAwACcAGQAHAPn/7v/r/+z/8//5/wAAAgAFAP//8v/o/9b/y/+7/7X/q/+e/5f/if+D/3n/cv9w/3P/df9x/2b/XP9Q/0X/Ov8v/yz/Jv8j/yX/Lf8v/zT/N/88/0H/R/9M/1L/Wv9Z/1//Zv9p/3X/fv+P/6D/r//B/9P/4P/x/wIAAgAJAAYA+v/z/+f/3v/S/8f/w//G/87/1f/g/+j/7P/1//v///8BAAEA/v/4//X/7v/r/+j/5//t//T/AQAGABIAHAArAD0ARgBOAFUAYABlAGcAXgBVAFIAUABWAGEAagBtAHMAewCJAJcAnwCgAKEAoACXAI8AggB/AIAAeQB2AHYAeAB7AH4AfAB9AHwAeQB5AHgAfAB+AIEAhwCFAIsAkwCZAJ0AnQCkAKUApgClAKQAnwCcAJoAjACHAIEAdgB2AHIAdgCAAIsAkwCgAK0AuQDNANQA3QDnAOoA6gDiANkAwwCvAJoAhQBuAGAAXABbAGQAcACBAI8AngCrALoAyQDMANEAzgDLAMUAuwC1ALYAvgDFAMkA3ADrAPYA+wACAf4A9gDlAMsAtwCcAI4AcgBfAFMARgBGAEsAVwBkAHEAdwCHAIoAhwCIAH8AfQB4AHMAZABfAGIAXgBiAGgAZABrAHMAcQB6AH8AgACDAIcAgQCDAHgAdgBzAGwAeQB5AJMAnQCjALsAywDaAN4A4wDbANcAzgDFALoApQCVAIcAfAByAHEAawBlAGkAZgBiAFwAUwBOAE0ATQBIAEMAPgA9AD0APgA7ADcANQAuACIAFQAFAPL/4v/V/8n/x//N/9b/4v/p/+//8//5////AQABAP7///8AAAEA+f/s/+n/3//R/8r/wv+6/7b/tv+t/7L/rv+z/77/xv/Q/9H/3P/d/+L/5f/l/+j/4P/i/+r/8v///w8AIgA0AEsAYgB3AIgAmgCoAK4AsQCxAKwApQCdAJQAjQCLAIYAjACWAJ4ArACzAL0AwwDGAM4AzwDVAM8AzQDNANEA2ADgAOsA/AAKARcBKgEzATwBRQE/ATkBMQEuAR8BEwEEAfMA6wDkAOcA6ADvAAMBDgEgAS8BQwFQAV4BXgFbAVMBQQEzASABDwH8AOYA1gDPAMMAvAC4ALgAugC+AMQAxgDHAMQAxADBALwAuACzAK8ArACuAKgAqQCuAKsAqwCoAKQAoQChAJoAlQCNAH4AcgBbAFAAOwAyACwAJwAmACIAKwAtADMAOQA4ADwAPgA2ADUANQAwAC4AHgAVABoAGwAWABUAEgAEAP7/7f/W/8z/tv+m/5j/hv98/3T/b/9y/3n/f/+L/53/qf+5/8b/yv/E/6//nv+K/2//VP87/yP/Df8A//X+8v70/vj+Av8O/xz/Jv8t/yz/Kf8k/xv/Ev8E/wX///4E/wf/Cf8f/zH/Sv9a/27/bv9u/2//Zf9b/0L/N/8r/yf/I/8c/x7/I/8w/0r/Zv+D/6T/uf/J/9v/5//m/+P/1P/A/7T/o/+R/4v/if+L/53/rf+6/9n/9P8KAB4AMAA3AEAASQBLAFAASQBAADsANgA1AD0APQA0ADIAMwAxADAAKwAdAAsAAQD4/+z/3v/U/8v/xv/O/9f/3P/b/9r/1v/O/8b/vv+y/6L/mv+U/5P/lP+U/5r/nP+f/6j/q/+s/67/rP+i/43/df9d/0r/Pf8z/zP/OP9D/1b/bv+F/5v/rP+6/8T/yP/G/8D/sv+e/4z/eP9g/03/Ov8t/yz/LP81/0L/UP9c/2v/df98/4T/fv94/27/Yf9R/0j/Pv84/zb/Nf89/0n/V/9s/4P/kv+i/63/tP+4/7f/r/+q/6X/m/+R/43/k/+e/7L/y//q/woAKQBCAFYAYgBnAGkAYwBhAFoAUgBLAEcASgBXAGoAlgC9AN8ABgEtAUkBXAFmAWgBaQFkAVEBSAFDAToBNQEuAS8BLgEtATQBQQFPAVEBVQFVAVgBVwFVAVUBWwFXAVABVAFYAWMBagFnAWkBagFlAVQBRgEzARYB9gDUALYAnACNAIcAhgCNAJsAswDGANsA8AAFARQBHAEdARUBCgH6AOIAygC1AJwAiwCAAH4AgQCLAJUAngCtALcAwwDMANoA2wDeANwA0QDIALwAtQC5ALUAsQCyALkAwQDKAM4AzwDUANMA1QDPANUAzADLAMIAtACwAKkArgCwALUAswC2ALkAtACyAKMAmgCPAIYAegByAGsAWABNAEUAPwA8AD0APAA9AEIASQBKAEcARwBFAEUAPwA6ACoAHwATAA0A///x/+n/3f/Y/9T/zP/G/8X/xv/F/7//vP+z/6z/qf+i/6L/n/+X/4//if+F/4D/ev91/2//b/9s/2r/bP9r/2n/bf9w/3P/ef+A/4z/kv+Y/5n/lP+N/4L/dv9l/1X/TP9G/0H/Of8+/0H/Rv9R/1j/Vv9X/0z/Ov8u/xj/AP/r/tn+zP7A/r3+wv7I/tL+4v70/gf/GP8r/zv/R/9O/1H/U/9Q/1D/Uv9U/07/Sf9I/0n/R/9K/0//Vf9b/1//YP9f/1r/WP9Y/1z/Xv9g/2D/YP9j/23/eP+D/4//kP+R/4v/hP98/3T/bv9l/2b/Xv9b/1j/W/9j/2b/cP95/4X/lv+i/6//wf/P/93/5//3/wQACwAXACAAJgAjACIAIQAfABsAFwAYABsAHgAgACYALQAzAEEASgBTAF0AZQBoAGcAZABeAFcATABDADgAKQAcAA0A///y/+j/3v/W/9H/zv/M/8z/zP/N/8//zf/L/8z/yf/L/83/0f/W/+D/5v/t//P/9f/y/+n/4v/V/8n/wP+2/6j/nP+V/4//jf+O/43/jv+N/47/k/+W/5T/kf+U/5r/pf+n/67/t//A/8P/wf/I/8b/yP++/7H/p/+T/4L/a/9f/1X/TP9G/0D/R/9O/1r/Y/9q/3j/fv+G/4b/hf+B/3f/bf9o/2L/W/88/yn/Hf8W/xX/DP8L/wv/Cf8M/xT/GP8l/yb/Jv8u/yb/Jf8f/xv/Ff8Q/w7/A/8H/wX/DP8N/xD/Jv83/0n/Xf93/4X/jv+c/6P/qv+v/7n/wf/G/87/2P/i//T/BQAJAAwAEgAkACwAKQAnABwAFwAMAAEA+f/v//j/+/8FABsALgA8AE0AYgByAIIAhQCSAJoAlgCWAIsAjACPAI0AjwCVAKUApACqALkAwQDLANgA0wDUANwA1gDdANoA0gDWAMcAxwDAAMAAzQDJAOcA8wD4AAYBFgEbARgBGQELAQIB+AD0AOYA1wDFALcApgCRAIgAfQBpAGQAXABUAEwAQQA9AEEARQBGAEUASABKAEwASgBDAD8AOwAxACIAEgD9/+//6v/n/+T/5v/x/wAAEwAhACwAMQA5AEQASABKAEwAUgBXAF8AXABTAFQATwBEAEQAPwA2AD4AOgAxADkAKQAqADkAQgBEAEIARABDAEoARwBGAEgAPwA8AEMARABKAFAAUgBVAFkAZABuAHcAfAB+AHwAdwBzAGYAXQBTAEIAOQAzACYAJAAqACQALgApACwAKwApADEAIgApACIAHAAgAB4AIQAeACQAMwBBAEsAWQBgAG0AfwCEAIUAgQCDAHsAeAByAGYAZABkAGkAbwB3AI8AoAC0AMkA4ADyAAQBBwEIAQMB9QDsAN8A1gDLALcApgCfAJEAjQCIAIIAhACCAIcAkQCUAJIAmACVAJYAlQCSAI0AggB8AGoAYgBkAGAAZwBkAGgAagBxAG0AdAB9AH0AhQCDAJAAjACTAJ0ApQCoAKgArACeAJYAiwB3AGwAXgBMAEIAOAAtACkAOwBEAFAAVQBZAGgAdQB0AIEAhQCBAI0AhgCPAJoAmQCkAKMAngCfAJ4AmgCcAJcAkgCXAJcAlQCOAH0AegByAF4AVgBMAD4ANwAzADAANAA2AD4ARABOAFwAYABpAGsAagBsAGQAWABNAEUANgAvAB4AEQAKAPn/+//z/+3/5f/b/9X/0P/K/7z/vv/C/8//1v/W/9r/1//U/9X/0P/N/8f/uf+s/53/kf99/2z/XP9L/0X/Ov81/zP/N/87/0f/WP9j/3n/hP+N/43/jf+C/3b/cf9o/1//Tf9E/zT/L/8y/zf/P/85/zr/Qf9D/0L/TP9L/0H/Qf87/zT/Kv8m/yT/If8h/yP/JP8j/yj/K/8s/zD/Nv82/zL/Ov8//0P/S/9Q/1b/Xf9n/3L/e/+A/4r/j/+K/4L/fP90/27/a/9l/2X/Yf9i/2f/a/9s/3H/b/9v/3H/b/9u/3L/cf9t/3D/dP9w/3b/df90/3n/d/99/4b/h/+L/5b/l/+f/6X/ov+n/6b/pf+n/6z/r/+4/8L/y//j//f/CgAnAEIAVQBnAHEAeQB/AIIAfgB5AG4AaQBdAEwAPgAvACQAHAAeACIAJAAlACgAJwAmACYAIQAaABMAEQAJAAQA/f/8//r/+P/9//j/+P/1//T/6//o/+X/4f/c/9b/1f/T/83/x//C/8P/v//B/77/wP/J/83/1P/d/+L/6P/x//T/+//5//T/9P/x//T/7//n/97/2P/L/77/uf+v/6T/l/+P/4f/hf+E/4f/jP+O/5X/nv+g/6n/sv+1/7j/sv+w/6v/pP+f/5X/i/+A/3r/d/91/3T/cv93/3z/f/+J/5D/lf+c/5X/kv+S/4//iP+E/3j/bP9k/1j/S/9E/z7/N/82/zL/M/84/z//S/9S/2L/bf92/3z/fv+D/4z/l/+c/6b/qP+o/63/q/+v/7D/sv+2/7z/xP/M/9L/0//W/9b/2f/h/+v/8v/3/wAADgAbACkANQBFAFYAYABuAHUAfQCGAIoAkgCVAJ4AoQCnALUAwADLANYA4ADrAPMA+QD8APoA9gDuAOoA5gDaANEA0ADOANMA0wDXANwA5QDtAPkABAESARQBGQEkASIBJQElASYBHwEcAQ4B/gD3AOoA3gDUAMoAwgDEAMUAxQDSANgA2gDqAPAA6QDwAOsA3ADbANIAwgC6AKkAogCaAJUAkgCPAIMAfAB2AHMAbgBuAG0AbABzAHgAfQCGAI0AlgCdAJsAlgCUAIgAewBsAGMAWABJAEEAMAAiABgADAAGAP7//P/9//7///8CAAgADAAVABwAHQAcABIACwD9//X/7P/m/+3/6v/o/+T/4P/Y/8//xv+9/7b/rv+k/5z/mP+V/5X/lf+T/5j/m/+e/6X/q/+o/6b/qP+n/6T/ov+h/6P/ov+k/6z/t/+8/8f/0f/X/+D/5P/l/+L/1//O/8X/uv+z/6n/n/+Z/5P/kv+T/5f/n/+o/7X/wv/N/9n/5P/t//P/8//w/+3/5v/f/9X/zP/E/7z/tv+v/6f/oP+a/5b/lf+Y/5z/pf+z/8D/0//l//v/DQAgADAAOwBEAEkASgBDADUALwAnACAAGAAPAAoABAAAAPn/9f/1//X/9v/z/+3/7v/n/+b/5P/l/+n/5//p/+P/5P/k/+X/4f/b/9v/1//a/9X/0v/P/8j/wv/F/8j/yP/M/8//1P/g/+f/5P/m/+X/5f/p/+v/7P/t/+z/7P/u/+r/5P/c/8//wP+3/6j/nf+V/4n/hf98/3f/ff+B/4P/if+P/5D/k/+P/4f/f/91/2//av9k/1z/XP9j/2v/dP91/3f/ef+E/4//mP+e/57/oP+e/5v/mf+U/5P/kf+P/5D/kv+R/5H/mP+a/5f/lf+V/5T/k/+N/43/kf+W/6b/sP+7/8v/zv/P/9P/zf/L/87/xv+8/7j/qv+k/53/k/+M/3j/cv9n/2P/bP9u/4X/mP+p/77/0//g/+H/5f/g/93/3f/a/8//yf/C/7r/uP+t/6b/nv+S/4//jf+K/4f/if+J/4//lf+W/5r/oP+i/6n/sP+v/7D/sf+u/6j/mv+I/3j/bf9r/2X/af92/4f/mv+n/7L/t/++/8b/y//Q/9X/4P/o//b/BAAHABMAGwAXAB0AIAAaACAAHAARABYADQAKABAAFAARAA0ACwD//wAA+//3//j/8f/0//v/+//+/wIAAQAGAAoAFQAgACsAMgA6AEEAQABAADoANQAzACkAHwAbABQAEwAZABYAIAAdAB0AHQAfACoAKAA4AEEASgBdAGgAcwB7AIIAlACaAKQArwCzAL4AxADFAMUAvQC6ALIAqgCkAJ4AmwCaAKYApwCtAL8A0ADlAP8AHgE6AVYBZAF0AXQBdAFqAWcBWwFXAUoBPwE9ATYBNQEzAS8BMQExATYBQQFCAT8BSwFJAUsBRwFCATkBKAEfAQ4BBAEBAfwA/QD/AP8A/QD7AO0A6ADeANkA1wDSANYA0ADOANIA0QDRANAA0wDLAMwAzQDMAM4AzwDOAM0AxwC+ALUAqACYAJMAjACGAIkAjACLAIsAiACDAIgAgwCIAI8AkgCiAKcAnQCZAJQAggB6AHEAXABQAEcAOQAuAB0ACwACAPX/7P/o/9r/zv/M/8P/vv+//73/v//J/8//1//d/93/3//f/9z/2f/S/8n/xf++/67/pP+X/5H/if+K/43/jf+O/4//lP+Z/53/o/+w/7X/uf+8/77/vf+7/7z/tP+z/6b/nP+Z/4j/fP9r/2H/V/9Q/07/Tf9Q/07/S/9K/03/Uv9a/2X/Z/9q/2z/bP9q/2X/Zf9e/1f/Uv9U/1b/Wf9g/2n/dv9+/4z/mv+l/6j/rf+0/7n/vf+7/7r/tf+3/7X/uv+5/7v/v//F/8r/0v/Z/9//4//k/+r/7f/w//P/9f/6/wEABAAEAAgACwARABcAFwAUAAoAAAD0/+n/4v/b/9//1v/P/87/wP+8/7//uv+6/7f/s/+x/7D/sv+v/7T/v//O/9r/2P/i/+r/7P/2//X/9v/9/wcADgAXABYAEQARAAsABwACAPz/8//v/+j/5//r/+n/7P/w//X/+P/4////CQATABsAIQAmACgAKgAkABkABwDz/93/xf+v/5v/hv90/2b/Vf9L/0L/QP9D/0f/UP9U/17/Zf9v/3L/dP94/3T/c/9w/3X/dv90/3X/dP91/3f/eP97/3r/d/93/3P/c/9y/3D/cv95/4D/i/+Q/5b/nf+l/6z/sP+1/7n/t/+u/6X/mf+N/3//cf9p/1//Tv8//zP/LP8u/y3/LP8n/yb/Lf8m/y7/Mv8z/zj/P/9E/0T/Rv9E/0X/Q/9D/0X/Qf9F/03/Tf9O/1X/Wv9p/3T/e/+H/5P/nP+n/7b/vP+7/8D/x//K/8r/zP/A/8f/yf/B/7//rf+//9X/y//V/9n/3//o//L/9//x/wYAEAAkAD0ATwBjAG4AfwCGAIUAhQCKAIQAgwB5AG8AbwBkAGoAYwBgAGsAdgCAAIkAjgCWAJwAmgCYAJYAlACOAIkAjACKAJAAlACaAKUArACzALwAwADEAMcAyQDMAMcAwgC7ALIApACWAJMAhgB8AHYAcQBuAG4AcABzAHcAdgB/AIMAhwCRAJMAjwCUAJ0AoQCqAK4AtACxALAArACjAJ0AlwCNAIcAiACFAIIAfgB4AHMAbgBsAG0AcwB2AG4AaABlAGQAZwBmAGcAawBrAGYAXgBYAEwAQQA3ACoAIAATAA4ABgD+//n/9f/1//T/+P/8/////f8CAP3/9v/y//H/7v/l/+n/4v/e/9f/1P/Z/9T/zf/I/8f/w//A/8P/wv/C/8D/xv/Q/9z/6//8/woAGQApADkATQBiAHkAkQCiALQAxwDWANsA4QDjAOAA4wDmAOUA6QDxAPoABAEMAQoBDwEQARABDAELAQgBAwH7APcA8gDsAOkA6gDsAO4A8AD1APcA/QD7APoA+AD1AO8A7QDmANsA1ADLAMIAuQCuAKgApQCdAJoAngCfAKIArAC0AMEA0wDgAO0AAAERASEBLwE3ATwBOQE6ATABJgESAQIB9QDnANUAxgDCAL4AxADKAMcAygDHAMUAygDFAMUAwQC5AK4AnwCTAIYAfABzAHAAaQBeAFMAUgBMAEAAOAAoABgADAACAPf/+P/3//T/9//x//f/+f/1/+3/3//U/8j/v/+y/5//jf+B/3P/Zv9b/1T/Tv9Q/1L/Uf9J/0j/Tf9L/0r/Rf8//zb/NP8s/yj/Iv8a/xb/Df/9/vT+6v7h/tj+zv7E/r/+uP61/rP+sv6v/qv+rf6r/qz+rv61/r7+yP7S/tb+2v7c/t7+4P7g/t/+3f7e/tr+1f7Q/s/+zP7O/s/+0/7V/t7+5/7v/vr+AP8I/xH/GP8c/yL/Jf8m/yj/JP8k/yP/Iv8g/xz/HP8a/xb/E/8T/w//D/8L/wj/Df8N/xf/H/8r/zT/OP8//0X/SP9M/0z/RP8+/zn/Nf8y/y//K/8t/zL/M/87/zv/N/84/zf/M/8w/yz/JP8b/xj/FP8T/xL/Ef8P/xX/IP8j/yn/Kf8x/zP/Mv8y/yv/Kf8n/yT/Hf8W/xH/DP8G/wb/BP8E/wv/C/8M/xD/FP8c/yT/L/8y/zb/Pv9N/1b/Yv9w/3X/fv+G/4r/kv+U/5f/l/+c/5v/nf+g/5//nv+d/57/oP+g/5//m/+e/5//o/+p/67/s/+v/63/q/+n/6b/o/+k/6X/pf+n/6X/p/+p/6n/sP+4/8D/0//d/+j/9f8EAA4AFQAjACYAJwAuADcAOwA/AEYATABLAE8AVABVAFYAXABhAGwAdAB6AIMAiACVAJ0ApQCqAK0ArACwALMAtAC3ALcAtgCzALUAtgC1ALMAqgCiAJwAlACIAIAAdgBvAG4AbgB1AIAAhgCPAJgAnwCkAKUApAChAJ0AmgCXAJIAkQCOAJMAmACZAJoAlACTAIsAhwCEAIMAhwCNAIwAjQCOAI0AkACSAJIAkwCYAKAAqQC1ALsAvQDFAMwAygDJAMQAwADBAMMAwADDAL4AvgDHAMoAzwDaAN4A4wDpAOkA5wDmAOIA6QDuAOgA8QD/APsA+AACAQIBCAEUAQoBDQEHAQEBEQETARIBGgESARQBFwEhASMBIQEnAS0BMwE1ATABNQE9AUEBTQFVAWABbAF2AX4BggGEAYABggGCAXkBYwFfAVsBUQFGAUEBQgE+AT0BPgFJAU8BWQFiAWoBdQFyAXYBbwFsAWkBYQFQAUEBNgExASIBFAEHAfYA8gDsAOIA2ADMAMQAxAC/ALoAsACnAJ0AmgCSAI8AhwBwAHEAbQBqAGYAXQBRAEgAPQAkABwAFQAQAA4ABgAGAAcABwAEAP7/+//3//D/6//h/9r/x/+2/67/p/+c/5f/iv93/23/Vv8+/yj/HP8Y/xH/D/8H/wX///75/vz++v78/gL/Bv8L/wf/Bf8C/wP/CP8J/w7/GP8Y/xn/HP8d/yL/J/8v/yv/Kv8p/yb/I/8h/yT/Jv8r/zL/OP87/0z/V/9c/2v/cf90/3X/eP92/3H/df9w/2z/cP9o/2f/Yf9l/2L/bv92/3X/fv9//4r/jP+T/5T/l/+g/6D/ov+g/5v/pf+t/7X/xP/M/9P/2//l/+n/8v/z//H/9v/3////AgAAAPT/9v/5//L/8//y/+j/4//o/97/1v/b/9n/0//Q/9D/0//a/+T/5f/m/+3/8v/2//r/+P/x//D/5//c/8//w/+0/6L/lv+O/4L/ff97/3f/ef99/4H/gP97/3n/bv9o/13/Uv9E/zn/J/8e/xP/BP/7/vf+7/7n/uL+2v7O/sT+vv6x/qb+n/6a/pn+lf6N/oX+hv6G/of+if6K/of+ff52/nP+Zf5c/lb+Sv5J/kb+RP5J/lP+W/5i/mv+d/6B/o/+l/6g/rD+tv69/sf+0v7X/uD+6f70/gb/E/8j/yv/K/8m/yT/Iv8h/yj/K/8t/zb/Pv9L/1H/Wf9e/2v/ef+B/4n/hv+N/57/rv+1/7v/xf/X/+T/4//l/+r/7v/s/+3/8f/0//P/7//t/+3/8v/8/wUAGgAuADYARABTAGQAdQCHAJMAnwCpAKwAsACtAKUApgCnAKgApgCnAKcApgCuALQAuwDFAMcAzwDbAOIA4wDiAOAA4QDfANwA2wDdAN0A2gDYANIAzgDOAMsAxADHAMAAugCzAKcAmwCRAIsAiACLAIoAiwCLAIUAgQB2AG0AXwBUAEkAOQAzAB4ADQD5/+7/8P/x//b//v8JAAwADAAPABUAGwAeACcAKgAlAB0ADgAAAPL/5v/j/+L/5P/n/+7/9P/9/wcAEAAbAB8AJQAmACgAKQAoACkAJgAmAC0AMQA5ADkAOAA+AEIARABCAEcASABGAEYARABIAEoAUABWAFgAVwBTAFEATQBIAEIAOgAuACUAIAAdACAAIwAlACcAJAAjACUAJwApACcAKgAxADkAQwBNAFoAYQBzAIEAjwCgALAAtgC7AMAAvwC/AMIAwwDDAMEAwQDBAL8AvQC8AMAAxADHANEA2QDnAPEA/wAGARYBHAEiASQBHgEaARQBEQETARQBFAETAREBCgH8AOwA2gDVAM4AwwDDAL8AvQC6ALwAwQC/AMEAwADAAL8AvACzAKYAlgCIAHoAawBcAFUAUQBNAEUAPAA3ADQAMAApACIAHAAUAAUA9//r/9//0f/L/8j/w//B/7z/vv++/7z/uv+8/7r/vv/A/73/vv++/7//xP/O/87/1f/T/9j/4f/Y/9X/1P/L/8P/wf+5/7H/sP+n/5r/jP9//3r/cP9x/3L/b/95/3T/c/92/3P/bv9p/27/a/9j/2D/Vv9O/0T/Pf8w/yn/J/8j/y7/L/80/z3/RP9R/1r/YP9k/2j/bP9t/2n/Zv9g/2P/Yv9m/2H/Yf9e/1z/Wv9T/07/Rf9E/zn/N/83/0H/TP9Y/2L/Y/9o/2v/Z/9k/2L/XP9T/0f/Pf8z/yn/GP8N/wP//P73/vH+8P7u/u/+8f71/vn+/v4I/w7/FP8c/x//Kf8x/zv/RP9J/07/Tv9N/03/SP9I/0b/R/9J/0X/Q/9E/0H/Qv9I/0n/UP9S/1L/UP9J/0L/PP83/zj/Nf8u/y//LP8v/zH/Of9C/0n/U/9Z/1v/YP9o/2j/Z/9p/2L/ZP9o/23/df+G/5b/ov+s/7n/zv/g//T/AAABAAQACgASACcAOAA9AEYAUABeAG0AgACIAI4AmgCdAKAAqQCtAK0AsACvAKwArACpAKcAqwCqAKsArwC2ALsAwQDAAMAAvQC8ALkAtgC4AL4AxADIAMsAzgDOANEA1QDVANMA0gDKAMIAuwC0AK0ApwCgAJoAlwCTAI0AiQCDAIMAhwCHAIkAhwCHAIUAgwB8AHIAYgBMADwALwAlAB4AGAAMAP7/9P/u/+7/6v/o/+X/4v/i/+H/4P/e/93/4f/l/+b/5f/g/93/1//J/7r/sf+o/5//m/+X/5H/jP+D/3r/dv92/3z/hf+H/4f/jv+T/5X/m/+i/6r/rP+y/7T/sf+p/5r/k/+L/4b/gf97/33/ff99/3//iP+P/5f/n/+i/6j/sf+3/73/xf/M/9P/3v/p//r/DAARABEAHgAoAC0ALgAlACQAJgAsADUAPwBOAFcAbAB1AIwAnwCkALIAwwDPANMA4ADlAOkAAAENARIBGAElAS0BOQE/AUsBUgFbAWABZAFnAWoBbQF6AYIBhgGTAaABqAG1AcgB0wHhAe0B/AEJAhgCIwIpAi8CNAI2AjcCQAJBAkgCTAJOAlMCWAJfAmQCZwJzAnUCdgJ0AnMCbAJrAmICXwJTAkoCPwIzAiUCGQIEAvUB4gHRAcEBsQGeAZIBggFwAWYBVwFJATwBNAEpAR8BFgEOAQcB/wDwAOEAyACzAJ4AiwB1AF4ATAA6ACcAGAANAAIA9f/s/+P/2f/S/8j/vv+2/67/oP+Q/4D/a/9b/0v/P/83/yr/Kf8m/yT/J/8t/y//M/8z/zj/Nv8x/yf/G/8L///+8f7k/tL+v/61/qX+lP6L/n7+bv5l/l3+Uv5M/kP+P/49/jn+OP42/jv+P/5G/kj+Rv4//jj+L/4o/hz+Ef4K/gn+CP4K/gv+DP4H/hH+GP4k/iv+Lv4x/jz+RP5O/l3+Y/51/oL+kP6l/rv+y/7e/u/++f4F/w3/Gf8b/yD/G/8Y/xX/EP8J/wf/BP8C/wP/A/8I/w7/F/8k/zb/Qv9M/1P/Uf9U/07/T/9L/0n/UP9O/1L/Uv9O/0v/SP9I/0n/SP9J/0r/S/9M/1H/V/9b/2L/Zv9r/2//d/9//4f/if+J/4v/jv+P/43/h/9//37/ff99/3z/gf+J/5X/of+s/7j/xv/R/9//6v/u//P/9v/4//v//P/6//X/7P/f/9b/zf/N/9H/zf/N/9H/1P/g/+n/6f/v/+3/6//m/97/2v/V/9T/0P/U/93/6//0/wkAGgAiAC0ALgA1ADgANgAyAC4ALQArADMAPgBDAEoATwBSAE8ATABLAE0ATgBRAFcAWABVAFEAVABaAF0AXwBjAGUAZgBlAF4AWgBYAFoAYwBpAG4AcABtAGoAaABnAGIAWQBVAFIATgBHAEMAPgA1AC8AKgAoACgALwA2AD4ARgBLAFUAXwBoAHAAeQB9AIEAgQB+AHgAcgBlAFkAUABDAD0AOwA0AC0AIwAYAA8ACwAMAAoACAAKAAkACQAMAA8AHAAjACgALQAsACwAJQAYAAgA9v/h/8f/s/+m/5f/h/97/3X/df93/4H/iP+R/5b/mv+Z/5n/mP+X/5T/jf+F/33/df9q/2P/X/9b/1//YP9g/2b/ZP9g/1//XP9d/2L/Z/9n/2j/ZP9m/2P/Y/9l/2T/Y/9Y/1T/Uf9S/1P/VP9T/1D/Uv9O/07/Tf9G/0D/Pv8+/0L/S/9S/1n/Yf9n/2//c/94/3z/f/+B/4f/i/+M/5H/kf+T/5X/l/+a/6D/pv+n/6j/pP+h/57/nv+h/6X/q/+s/7T/uv/A/8n/zv/T/9j/3//o//D//v8IABEAHQApADcARQBUAF0AXwBjAGIAYABjAGMAXwBiAGEAXQBgAGAAZABqAG4AcwB9AIgAjgCXAJsAnQCjAKkArgC4ALsAwwDFAMcAygDOANAAzgDJAMcAwwC2ALIAqACiAJ4AlQCSAJIAlgCYAJwAngChAKUArAC0AL4AwgDCAMEAwQDBAL4AuACwAKcAmwCOAIMAeQBzAG4AZgBiAGIAZQBpAGsAagBpAGgAYwBgAF0AWABTAFAATwBLAEsASwBNAE8ATQBRAE4ATQBKAEgARQBCAEAAMAAqACEADgAAAPD/3f/V/87/xP/F/8z/0P/e/+n/9f8LABgAGgAXAA4A+P/h/8f/qv+J/3D/Xv9R/1H/U/9c/2f/d/+G/5f/qP+w/7r/uf+z/6T/jv94/13/R/8w/xn/Df///vr+9v73/v7+Bv8S/x3/Jv8k/yb/G/8H/+/+1/65/pj+ff5k/lH+Pv42/jH+LP4w/jz+S/5e/nn+hv6V/qP+qP6r/rL+rP6f/pj+iv55/mn+Vv49/in+GP4M/gf+AP7//QD+Ev4k/kj+Yv57/p3+sf7D/s3+0P7I/rz+rP6R/m/+T/4x/hX+AP7u/er97P31/Qb+Gv4u/kH+T/5g/mj+bv5t/mn+Wv5M/jr+NP4n/hr+Gv4O/gz+Af7//QD+CP4U/h7+Pf5d/of+qP7H/uD+9P4G/xr/Jf8s/zn/Qf9K/1H/Tf9A/zf/L/8n/yT/Iv8i/y//N/9C/1r/bf+L/6z/0f/x/woAIgApADMAMQAhABIA8//b/8j/uP+7/8f/4f8FAC0AXwCSAMcA/wAtAUgBWQFUAUoBMwEVAfcAzwCxAJgAiQCGAI8AnQCuAMUA1QDqAPgAEAEZARQBDwH6AOcA1ADGALkAsQChAJsAmwCkALgAzQDiAPQA/wAKARQBEwELAfUA2gC2AJMAeABcAFAARwBGAE0AXwB6AJ4AwgDdAPoACQEWARgBCwH3ANgAtQCRAG4ATAA0ACAAFgAWACQANwBNAGgAegCPAJUAlQCVAIQAaQBSAC8ACwD1/9f/wf+z/6v/q/+3/8f/3f/3/woAHgAlAC8ALAAwAC4AJwAeAA4A///s/9j/yP++/7f/t/+3/8D/zP/Z/+L/BAAkADQAVABmAIYAlgCAAIAAbwBMAEYAMAAvAD8AOAA3AEYATgBvAI0AqwDtABgBQgFhAWsBWQEyARUBMQE3ARUB9gC6AIgAeAB8AIoAtwDOANIAyADSAPQA3QD5APoA6QDgALgArAChAI8APgBIAD8AXwC6AMoAIAE0ARkB/gARAUMBUwEtAeMAkwBIAEkAUwBWAFIAQABBAF0AjADAAAsBGgEXAT4BawGdAZoBdgEwAf0AvgCTAKgAmACAAJYAXwBxAL0AzAARATwBdQF0AXsBlAGHAXoBOgH1APkAEQH3AOsAuQBbADcATQBrAMQA7gDIAJ4AfwB9AH8AlwCLAIEAWwBIAHEAbwBAAAkA3f/c//P/QwBzAEYAIADT/9//GABmAMIAyQDnAMMApgC7AKsAvgC0AKIAlACDAHQASQA/ADwASgB6AM4AQQFzAYABfgFnAYEBsAHeAfgB5AGoAX8BfAFCAUgBTQEwAV4BXgF7AYoBkwGEAYsBvAEYAqACbwJGAgcCrwGkAZYBygENAgQCxgHWAewBvwEGAloCcgKzAqACgQKAAjsC6gF+AR4BPwERAR4BEQHgANsAoQDqAC8BigG9AdMBzAGEAXMBQwHyAL0AhQBwAEoA7v/Q/6n/jf+r/9D/PgCtAOIA0QDPANAA0wABAQ4BKgETAfAAuQCWALIAWgA6ABoAMwBgADEAVgBYAEAACAA9AKcADgExAdMAtQBOACgATgBgAHkAhgBCANP/yv/b/9T/wP+y/6L/m/94/1//c/9S/yn/L/8Y/yX/QP9A/z//Kf8Q//T+4f77/vf+2f6Z/mr+Mv7z/Qj+2f3O/c39wP3T/df98/0j/tj9m/2f/YL9WP0+/Qf9pvy9/Lz85Pwy/WL9cv2N/ZL9pf20/YT9dv1L/TP9A/3k/JP8GvzO+4z7kvvZ+yz8YvzD/B79Yv2k/aX9s/3P/c79mv1Q/RH9mPwI/K77HvuQ+mH6efq6+h/7fPu8+wf8IvyU/Dn9uv1a/m/+Tv4A/nH9C/2M/Cz8NPx3/Ov8af2C/Vz9b/2P/dL9Nf6G/rn+Af/2/q7+Wv7C/ZD9wv0g/qT+MP9M/13/Xf8K/xn/TP+u/w4AIAAiAM//Wv8L/7D+x/4R/37/4v8XAFkAWABMAD4AewDMAFYB/gH9AbkBIwF4ACkAMAB8ANAA6wAPAfgA0wCsAKkAOwHAASsCiwK4ApMCKAKQAVIBcAG3AU4ChwKGAo8CTgIoAkMCwgKCA2EE4QQ9BQMFqgROBJMDKwMFA18DXAMjA8oCFQKhAbQBQAIXA00ERgX3BREGxwWOBVsFRgVuBWUFQwXMBOQDBQPJAVEBOQGVAVcCLgMzBH8EpgSIBIME7gRyBegFIAYGBn8FmwS8AxUDigJ+As4CTQPDA8sDxgOQA0ADZgPMA4cEGwVQBQcF0ARABLMDwwOvAxsEVwQMBKsDMANNAoEBJAHFADIBdAE2AeQA9v8h/7X+Uf6L/lX/6f90APUAKQFOAbYBNALhAj8DJQM3A3cCYAE1APn+Qv7T/e39Vv7u/i//ef+z/7j/NgC/AKgBUwJ/ApUCIAJdAVMANP+H/kH+Gf7T/Wb9+PzQ/Nn83vw9/cj9jf5z/+r/IwAWANr/jv8n/7/+b/4q/gH+r/39/Gf8Nfz9+xX8pPww/bj9zf3P/fT95/0K/ij+Tv6Q/qv+nv5s/jP+xf1p/Wr9rv1E/s/+3f62/sf+aP4h/m3+kP6N/nL+Tf53/g7+y/3Z/dT9Jv4U/kP+nv11/Tv90vxE/G78YP3H/q0ANQF1ASwA3v5J/pz9Pvyl+1X7jfpm+y78jfwC/av94/1B/h3/DQADARgBdADC/xf/x/7G/qf+Nv6E/Vf9U/3E/dH9XP0D/RP8pfuq+5r7Cvzg+7X72PsF/JX8g/3B/in/wv8AAE8AUgDo/1r/Mf4x/Q38tPvH+xf81fx7/TH+RP6c/nL/Uf8HAJcADwFLAREBpwFhAcEABQCJ/2X/uf+f//j+T/4s/bz8j/0k/sP+dwDOARwD6QNvBIEENAQLBOoCzQG6AEUATwA2AKr/Vf9S/2H/zP8gAJUAtwCDAVMCcgPnBHIFJwWaBHgEKARkBD4EBgSZAzkDSAMtA8ICDgKyAbwBkQKyAx0FFgZpBlsG7QUfBXIEiwRtBGMEqgTOBLMEywQHBQYFNQUuBeYFegZ3BnQG6QWfBSUFOQWcBVAGdAaTBhIH5geHCLoHCwfFBqAGbga0BrAG8wUQBuwG1AYPBwkHbgewB1kH3QcbCFYIZgiMCBMIUgd1BlkGugWKBXUGpQbUB2UJTAqKCm8KHgq8CW8JsgnHCWkJxQhsCPUHpAfEBykHoAZpBpQGzwVHBXcFYgXWBZAG/AZeB0cInQmECuUKXApeCXsIbAc/B2MG0gU9BTUETwMhA9IDOwT4BBsFegXuBeoG1QclCCAIFwi4B5MGTwadBVEEtQORA8UCbwKgAbEAIADb/2UAjwC2AOEAcwFwAW8B+wD6APsA1wD2AM0AWwEwAcEAbf/1/bb8xPte+0D7AfvV+vn6IvuP+4n7jfyW/aP+fv8HAOEATwBr/wD/9P0W/kv9/vwg/bn87Pyl/OD8x/sN+7b5LPnK+K34DPmh+Cj57PjZ+Xr6BPuO+8v7ffxL/C/8RvuL+aX3+Pao9n32Jfd39+j3JPi0+G73Ifd59h/2z/b99sb3KviW+WD5Sfoc+mj6bPrN+Tn67vmM+nf52feM9Tb0iPN/88DzrfOn8/TzyfQ/9b314PVW9rv2FvhP+Yn6u/po+rz5jPiV9yH2APXW8w/zz/Ka8rrxEPFp8Z3xtPHx8jX0S/XE9tb3sffb94X4kfgi+cX5O/nx+Lr4a/gP+az4Gfis9/D28fW09X71/PSK9I/0v/SQ9IL1f/ZZ98D3q/hV+Tz6YPqV+gv6WfnT+Jz3T/hA+Az5jvlz+eD4rvj4+GD4Wvgb+PT3Tfhm+ef54fp/+177e/v4+zD8G/yG/Fv88vsW+xL6NPk8+KL3+Pd8+NT5Nvvj+0D8sv0L/uf+nf/3/vr+1v7S/+v/PAAWAAcAMQAqAN0AggBtAA8Am/8o/zz+zP0d/Tr9tf1o/nP/rQBmAawCswPBA7kEvwTQBKUE1gQdBAEEZwODArECLAI8AzUDngNPA3ADOQPoAoQCQQGqATkCwQMNBbkFpQXkBdUGiAfNBwYI1AfIB40HPAigCM8HrAe7BoQGeQbbBhMHfQfUB4UHQwjsB1sIaAiWCA8JnQmwCp8K2AqVClwK6wl4CR4J/QiOCOgHEwihB74H7gdOB2oHxgcyCKAI2wjfCLYISQj3B1YIVAiZCMQI0QgOCS8JjwlBCRwJpAhxCP8HpQflBwMHowZ7BiYGLQbVBmUHjQf9ByEIIggvCFoIMQirB3sHXwdLB/EGbAYbBqYFDwXEBCAE5gPyA+YD/QOlA18DDQPBApsC8QJ8Aj0C8QGcAUMBOwGDARcBWwExAd0AdgDf/wH/hf5l/iv+Ev72/c79a/0b/U78IfyM+8/6ofpB+rP5+/jG+Jn4LPiJ9y73cPZb9mD2UfY39zL3kPe695737vff95H4uviO+T37KfwM/eH9+/1F/jH/wv9wAOUAjAGuAXwBNwG7AQACEQLbAYIBdQGgAGUBSAIyA48EAQUkBswHPQnPCngMyg0ED8oP2A+dEOQQgRFKEmES3xLeEuASABOyEjUSRBLyETMSohK4Eg8T3BN3FBYV9hUBFiYWQxbmFjEXBBfPFkYW+hVpFfEUXRTdExMTPBJUEYgQ3g9JD4AOyg16DXIMBQyjC0wLQgsQC6AKYgowCvUJkAltCAgHlgXxA1QCewC7/ob8jPnm9h30YfFr7zPuju197QTt3OyC7Ibsiuyi7Lbsluyb7HzsQ+zB663r7+qn6jPqpOnN6BDouufm5sPmdubT5WPlN+VF5XPlWOWB5XnlX+Vb5Rzl0+SD5HvkfeSS5N3kLOW/5ZvmN+fw5+nozOnf6vDrtOyc7Tzu8+5e79jve/A18SryEPNt9Hb1mfbE9+r4HvpJ+3b8wP3s/jsA0gEOA7cEWwa+B60JPAtqDdYP2BFRFNQWohg/GpYcDx4pIHYieSTFJvYnPCn1KVsrbCwZLX8tGC4QL70vsDECM10zXzMBM/IxKTBULVsqJyd7I3whRB9wHBgZ9RSID7AJkgQfALv8Sfk+9q3y0e/q7sTvxPHW9O/44/xCAS0FdwmNDMgOYRBsEBUQ/A4aDG4IGwQ5/5r67fVB8vnuW+w96tXoL+jW52boOems6sfsQu/78Zr0ffey+Zj7H/2a/VH+wv6W/ub9aPxI+mL3MfRd8ILsrOjd5I/hT97x2szX+tRp0qrQnc8nzybPUtAA0jDUp9bL2OTbGN9k4t/lWuhq6ibr1Or76U7o1Oaq5XrkyOOd4sfhgeAt3ybextw43H3bkdsD3E/d7t514ZTk4OdY7HrwpPRD+Iv7mP5XAuYFCgujEDIVDhmFGjMc0BstG8sbmhviHJkdAx5HHkIegh57HqcetB5mH1MffR/lHjEeoh04HQogLiRhKkcxxjcSPMhAk0QZSOZKXksgSmtFMD4dMxAnjhnyCwL/2/Pi6S7hpNp61FjQt82wzfDPdtWR3LDkmewJ8534af3gAnEIUA+qFf0bpyA0I3sjgSFSHrIalhcDFJ0QEgzJBk4AWvmy8gHtauiJ5HDhAt+O3Dbao9iN1+TX4Nm23U3jLepT8WX3HP3jAcoFUwnKC9QNJw5FDVcKkwVz/8X4LvKd69XlSOBZ29vWNtO5z+zMlcqvyYHKM8w0z0HSANZs2oPe4OOp6cDvdvbX+0cAlwJHA/UChAJMAlcCXwL7ATwAN/w492vwSelZ4jzc0dZL0wzRP9C+0aHSeNSx16rbU+LH6o30qv/kCnwVUh5+JjguRjMIONE5LjrSOQc3WzQlMb8ttijpJPMgDx5SHFsZ+RexFZITphFCEQQRvhKLFcIZdR9BJuUuwTUIPctBwkXNSJdKZkzCTCtNOkwnSsVGhkHSOosxSyfZG68PpgPp9jnqv96902bKJMNtvle+asFIyZLTFd8Y6tv0fADaCokUEB2TI1ooOSozKVgmOiE+G7QUhw4/CS4FcgNpAn4Bx/9E/e35RPYl84zv2euW53Ljt99c3BHaQdke2uncfeEa5xft2PIP+OX8WwFaBUII1QnNCcEICweaBPYBL/9h/BT6a/gt9+D1MPRE8jfwIe7/6wLqyuc05Zzi0t/v3Anaddep1RbVDtYy2BzbaN7H4eLkb+eX6UnrP+xd7OPrvOoS6d/mPeSn4SnfvNyT2u/YANj7173Ygdpc3VjhEeYQ63TvFvMO9Un2D/dL97v4d/qt/Cb/rAACAZ0BKgMpB2sOAhd1IJ0nuCvUK+kopiMrHhgZOxXfE7UT7RWlGIsb1B02IJEkICtbM5w8EUV9SihNVE4LTzRQPVI6VMNU9lIbTq5Gsz2vNGcsCSWkHkEY9hDSBxv94vHU5k/dA9dF1OrUgdcE22/eneGv5Mfn9evO8Ir1ovny/DT/7/96/0L+9/xg/AP9Tf+OAgYGXQn5C5kNZw5eDmIN4wvJCegGpgIJ/Tv26u4k6G3ibt7e2+/ag9sl3WzfCOLJ5ITna+ow7aTvdfED8kjxVO/Y7FXqLOi55vzlaOYz6EzrXO8F9KT48fxoAPMCxwTeBfgF6ARaAuT9D/h08cvqj+SO33nbU9hZ1onV69XK1hvYONli2kDbsNua2xPbmNqR2hnbSdzf3T3fwuBG4jjkq+ad6b3snu+X8lb1sPfO+Vr7EPzm+8L6cvkt+Ar3hvYx9u/1jfX+9Fz0FvRi9AX1H/bj9uX3lfhv+Z760/u7/UsA0wMyCCUNTBJ6FyEcaCGoJ0Eu4zWEPCRBFUO4QZY+hTthOo48JUFjRrVL8U5UUEVQRE+vTqdNUUyjSu5GgkGkOVYvTSQoGa8POAdSAKT68PU98tvtluqQ6OzmYOYU5qPlVeVg5IXjHeMD497jM+UO593px+x38KH09viv/QICFgaECUwMgg4fEEgRmxGMEQIRmw+9DZ4LFwmvBqgEHQPuAQcBNwBM/27+of0h/cH8dPw1/Ob7q/tw+0T7Dvu1+k76w/kF+RP4FvcV9kT1tfQ29MXzTfPe8lTyXvEX8GjuU+wD6nPn+uT54mLhD+DX3rjdw9w03BfcM9xD3JrcKd243TveXN4z3t7dT91v3IzbE9sY27vb3dyJ3tDgUuPQ5RLovOkH6xnsLO197gvwpvGR83H1lfZE91D3K/c991L34vdf+Zz7r/4+ArYFJAn1CwMPBhJbFUYZJh0gIdckOifDKBIqLSuULMEtgC/EMUM0EDcCOqo9pEFpRUNJsEwQT85QYlHAUDdPkkwVSX5ETT7XNnou7SUYHpMXURJFDjgLGgiQBBAA7Pqb9VPwxevq55DkR+HU3Vza69bD017RF9D2zwfRWdPI1kTbLeBs5dHqTvB29RH6Wf4XAsIFEwkpDBgPqhHDE1wVpRZxF7kX9xdOGFcYrxgHGSsZ+Bg1GOsWORV/E9MRBxBVDtIMSguoCacHZwWPAlH/vfvK9xH0bPDw7PPpaud25WLkxOOA4wLjleIe4kDhV+Aj3xneV92R3C3bRNkG17DU9tLT0XbR/dFL0+bUtdZX2L/ZINtq3J3det5Z3xLgluAs4b/hOOLs4rvjWuQl5T/mr+eG6ZPrl+2v78jxtfNs9bj29ffS+Jf5t/p2+5n87P1F/7sATALjA9sFLQiwCqUNlxC6E3gWzBiUGskbRB2pHj0g0yLXJRAphiybL7QyvTWyOAQ8VD8ZQudDjkQNRwRKAU33TxdS+lJVUmtPS0q2Q0k7uzIoKk0iShvwFJAOhggcAw/+dPoy+N72ifVN88jv++qM5X3gxty22qnZpthM14bV4tOE00zVYNlJ38Hl1uvG8Jr0n/dq+lr9KgDAAokEcAW2BagFuQVZBp8HzwmbDKwPshJRFZwXoxmKGw4d/R3xHakcEhpKFqIRkwy9B3IDvf9v/GL5ffbE82bxxe+y7gLuVO1p7BTrCOmS5rrjy+Dq3Rbbm9h31qjU/tKZ0ZbQ+M+Qz3vP18+J0FrR7dE30lLSQtIL0uTR/dFZ0unSptOy1PnVmteP2c/bTN7M4C/jGOXI5nPoEOqR69Hstu1k7qju5u5q7zXwjPGN87719fdr+tf8Of8nARkDdAQSBXcFpQViBgwI2Ar0DrsTdxjAHAYg2CLAJd4opSwKMbY1gzkuPMU91D4RPxBAQUKWRdtJf00LUCxR/1AmUAZQo1DHUeNR/0/KS+FELjzUMlkq3CIXHEwVIg7DBqf//fka9svznvIE8Vru0up85nHiXt+d3aLcmtsv2oLY6dYp1gzXY9nc3NjgFuVo6drtefIA92T7Vv+XAsoEIwbwBiMHTQeAB+AHtgjlCeMLaA4GEZUTtxWZF0sZqhrLG0YczhsVGvAWtRKgDU8ISQPs/mv7k/gL9uTzC/KS8G/vnu4l7rftLO0r7G/qFeg85SPiCt8r3LHZn9fc1WPUJ9ND0rjRstE70kvTj9Sw1XrW9tY+127Xp9fJ1+LX7dcW2GTY79jg2S7b7Nzg3tTgu+Ka5FXmBuiJ6dPq2+tw7KrsduwJ7Hzr++q16rvqBOuE61PsQu1T7mPvb/CQ8bTy6vNZ9Qr3V/iA+YX6k/vi/Cj+ogBqA8UH8gzvEU4XDRwjID8kACgxLP8wyjUPPM5BjEY+SkxMuE4CURlU41gbXuJjCmgxaZdojmbIYxlgilq5UmxICDyALw0kcho6EjUL4ARY/vr34fGK7DzowOS/4dDer9uX2A7WtdSz1EPVUtbh12HZhNub3oTiWOdV7DTxIPZ1+rz9ZADeAnoFQQg1C8INxg8MEmcUihZSGEMZtxkWGukZAxnlF/MVkRNJEagOtQuYCKkF7wKgANj+Jv3M+6H6qvl++RD5Efi59rL0ePLm79vsC+rC51Pm6eW35XblGeUM5Snlx+VQ5v7lJuXd4yji9d9Z3VXaJNf507rQhs03y/jJNMl7ySHKccsezZXOatAR0oXTatTk1ILVIdaH1tfWQteD1wLXs9Y111nYi9ob3YPgSuSs51rrfe478YD0j/bI97D4afh/+GT4Qfd19gf1RvNl8+HzbvXY+Cr99wKcCK4N7xG7FNsWVRhLGWIbHB2QH0YkFSkhLocytja2PJ9EyU1lWEhjO2ydc3h6G3//f/9//3/vfVl3m2x6X2JRzUQKOiUw+icCH2YW+Q3xBD3+e/gR893tKOgC4lnajtJgzDHIQsXUwkXBF8ErwvbFicwn1U7fI+lR8jz6zgB0Bv0L4hGcFoYa8xx7HVIech+wIDAiHCPjI18k6iRmJWglViV9JBEjeiCDHPcX9BIuDdsGtv9o+FbyF+0o6Q3mqeMV4tbgQeCC4B3iAeUA6LzqHO2j7mzvBvB/8PLwWPE58anw+u+97hjtOOsY6cPmQuRt4U3ebNvQ2C3WZ9NC0BXNEcp3x2nFs8OiwqHCDMO/w5XEuMUdx3rJbcwnz/nRrNTq1lHZWdta3XXf7OBl4m7jVeTa5SznK+jt6P7o6+hw6NPnWufV5ojmJOYN5uPlcuaE52bqke3j8OP09fhK/s4D1ghtDUoSRBcLHKkfiiI+JfklSSemJ7UoAysALaMvuzOYOEA+ckYPTnVXJWB1aX1zMXv/f/9//3//f/9//3+JfSt1v2u8XzlSREOxMxYkKRMCA3LxKuIU1gzNbMmXxvnE2sQ8xQrGrMeOyq3OHdMb10vaZdyB3ojge+OV50LsivAS9oz8sgPOC2YTZxucIj8ori0kMsQ03DW7NPkxiC2lJ/4gxBnyEkcMPgb1AK77Bvd489HwBO/G7dvs5uu46jjpfuew5YHjQ+Hc3mncI9qO2PDXgdh52vLc1N824wXn4utM8Y72IvsM/28CKwU6Bx0IxQeyBRoCMf1Y9+PwIuqi4w3dKtYNzz7I58FVvRG6W7ixt923c7j3uZK85b8NxE7Irsyw0OHUe9iV203em+At4j7j2OOx42/jd+NK5E7lf+a256DpKOwR75HygfXm+DX8wv4EAb8CVARoBdkF2gVwBbEEAgXABFAFvwV/BtoI8Qr8DQ0R0xQVGYgcvB+lI6wnCywlMV42+zs1QZNGwkytUxNbOWI1aZ1vnnVKe/x//3//f/9//3/MfXR1MmosXpRQc0KnMowjEBScBL33Quo44f7aK9ZV05/RrNES0xDU3dW917HZSdyE3cfev97Q3S3dlt2u3s3g2+Ov56ft9fQ5/Z0FHQ46Fv8dQCUsK+AwbDUROP845jfgNX8y9y7oKoYlmiBmG/0W3BLNDpALXAiABFoAvvv49gPzou4L66vmvuFH3dHYrNV703LSctJG0y3VNdgN3Czg0OSz6SjuRvKu9YH4mPpu+zb7GvqW9wv09O+d61fnUOOM38vbYtgY1VzSVdAMz43OU859zubOOc+pz/XPM9BE0E/QdtAw0NHPk89Ozy7PTM9dz5bPTNAo0ZbSpdQt1x7aKt1O4JHjBedw6tLtWfG/9K/3qvpE/c//VAI2BGUGeghJCikM6w3MD8IRrBN9FV4XehmoG+kd1B+3IuMkcyeSKugt9zH4New5bj6hQ59Is0/1VjhebWXma+hxzHUPeXp5r3gzd45yZ2y3Y/hY7kw2QKsyTCR/F7gJ0vwB8vXm6t5F2JrS089OztfNWs7KzwjSrNSo1zHautxC31rhKOPg5ILmLega6grsLe4f8Wz0P/jo/FEBaQb+CzgRVxd/Hfsi0yfAK9guVzG9MkYzKzNlMScucinGI00dRBY+D9sHmQA1+efxOOtF5T3gA9zI2GDWndSJ00LThNOy1HLWeNjt2mnd99+E4vbkV+eb6VLrNuyv7MTsVeye63/q8ej45pnkHeKs3z/dm9o/2EHWZtTi0pbRjdAR0A/QRNCx0IfRXNJc03nUq9XY1rjXg9gO2XfZjNmC2ZfZtdm82cXZBtqd2knbHtxx3f7e/eAb46Xlnuii6zLvifK99Tb5Ofwq/xcCpARDB4EJPwsLDUkObw/+EDISVxN8FCMWPhgXGsgcXh9uIg8mnyl5LuMzxDmeQJdHn06nVd9bCWKjZ4VswnAlcxR1C3Vbc19xyG3YaNJi/lrcUQJI/jy1MV8muhsEEfIGL/368z3s2OTl3yfcetmZ10XWrtXT1VzWl9dq2dTbRt7k38/hUOMD5ePmaOkk7CLvBPKu9B74vvtXACwF0AqzELMW1hxQIuYn/SyuMc01+jhlO8A8Ej1ZPO45bjbsMbssDyeZIJYZCBJUCrsCJfsH9J/tv+fw4mreetoe11DUrtIP0nfSftOW1OvVwNe62bHbyd2o3xzhTOIW437jheMC4xLi3+A23xfdAtsi2XnXPtZF1VrUmtMB07rS+NKt06vUwtXy1iXYT9lT2iXb89tu3JzcwdyV3EjcFNzK21zbHdvM2ofayNo72xncZt0X3xThO+Oh5RvoAOv37QDxePS898j68P3+ADgEdwcXCtgMkA/OEf4TyhV2FzsZ3hqQHBUe4x8sItck8ydOLBAw9TPjOFU91kKrSL1NDFMOWCFcgmCaZChoXWtNbRRuGm0Na2lnH2PsXkZZdlIUSpRAWzavLA4j4xjGD6gFt/q38EjmNd0u1vfPwsu0yIrGDsWqxHLFI8dlyhPO89Gc1v/aLd9o45bm2ukZ7dPvnvJd9eX3OvrP/BL/jQFmBGMHLAvTDwEVHRrHHuUiDSckK+cuOzKuNPw18DVkNL0xwS3+KLkjoB3QFh8P5QZ//n/2Cu/q51/hctu/1TfReM2yyiLJQMhCyLzItskby5/Mks720ELTHdXG1jXYMdkV2o7aq9pi2o/Zpti419jWodWO1LbT7dKM0k7SPNJq0gLT1tPT1PrVANcX2E3Zmtrj2+Tc1d2j3kXfxd8J4Gng7OBk4erheOIr4+/j2OQU5p3noenG60XuTfGw9JP4c/xPAG4EKggQDP4PnxMeF90ZChy9Hd8eJCAiIfUhbCKUIoQjVyRMJfEmNyhjKhct9C8ONHo4hz0xQ21IeE0TUtVVXVlKXGhe01+GXydfK14cXO9ZCVeZU5dPkUqlREQ+4zZQL7Mn4R+3FywP0QVC/B/zyuk/4YLZPdJjy0LF7b/fu3u4gLb5taq2g7j8ukK+PsLwxjHMa9Lj2IzfVOa+7Nny+fi6/ioERQmjDZ0RHhUqGEIbOB7IIPkioCT3JfomCigjKSYq5ipBKykrgCp+KSIomSaRJPIhkx6AGhwWjBHLDMYHagKn/LD2svAO6+vlNeGs3FnYadTC0KzNQ8uCyY/IEMgGyFLI2siKyW7Kksu6zM7N4s7vz8vQf9EX0njSwNLb0q7Sa9IS0rLRXNEx0SnRN9FQ0V/Ry9Gn0vjTy9XE19jZINx23uDgeeMF5n/o8eoq7Rzv6fCd8i70rPXc9uH33Pip+ZH6xfs5/av+MwDNAZIDiQWDB/EJXwwnDwQSvRR6Fzga9RyBHy8i0ySJJx8qxCxPL5ox4jMLNkc4aDpiPC0+BUD4QQdEOEamSNxKnkw9TmxPdFBoUZFRkVGFUUxQrk5xTDtJOkVjQFg6VDMRLPIjPhsKEjMIPv6r9EfrrOLo2qjTdM0hyLPDPsDxvde8jLxiveC+DMHww1PHj8sj0LrUT9nr3ZrieOd57HbxPva7+pD+FQKfBfsIOgxZDyYS1BRmF7AZ8RszHjUg4iFXI3IkXSU3JusmOicCJy4mmyS4IoQg2h3eGm8XrhOGD+8KFAYpATb8Q/dl8obt1ehx5EzgndxJ2TvWc9Mk0U7PDc4zzbnMhcx0zIDMpMwPzZXNHs7DznLP7c8/0F7QWdBp0HPQa9B00HrQeNC20BjRt9GE0m3TjdTu1XTXM9kV2xndRt9o4XfjceVQ5y3pAOum7D7usu8B8UXyjPMm9M30ffUl9ub2m/eM+Kj5DPuT/F7+SABgAoIE/QauCY4Mbw84Ei8V6BelGmkdLyADI6QlQCjfKlgt/C+yMoE1ojjOO/w+CkI8RXdImEvMToRR1lPEVVpXi1g/WYJZ6ViCV0RV7FHGTSZJE0RIPgk4DzFtKWwhDRmnEH4IZgBj+NDwQelO4tzbMdZ40aTNmMoKyDbGPcXIxEjFzsavyAbL4c0y0f7UItlk3f/h0eZo6/vvsfRE+df9KwIgBssJKg1QEDAT2hU7GEka1RsRHSweHR/WHzsgbCBQIBwgrx/nHvQdtxw0G3cZmxdwFRATdRCBDVkK+QZ4A97/T/yo+Of0H/Fb7fbp4OYS5IvhSt8k3THbsdmF2HbXwdZG1tPVkdVy1XHVqNXv1ULWgtau1tLW+9ZE15LX2tcE2BfYO9hl2KTYCdl02brZG9qi2ifb39t83A3dut1d3urebd8J4L3gbOEB4pTiL+PJ43LkPOUW5v3mAOgq6YvqGOyw7XjvZfFg84311/cr+qb8Kv+5AXMERwdGCk4NXBB0E2kWchmeHPIfWiO7JvQpJi1LMHsz1jaIOug990AORKVGU0kjTFlOlFC2UipUelVfVr9Wu1ZBViRVa1NBUUdOB0tIR6tCqD3SN1oxjCqaI08c7BREDVgFgv3Z9ajum+c/4Y/bZ9b40R7O9sqvyAfHQMZ2xlLH0Mgey+bNLtHe1PTYht1v4njnTOwN8aL1J/pt/qsCoAYLCj4N9w9mEr8U6RbDGGsaqBugHGQd8R1/HuweDh/UHlIemx3BHLYbbxrtGAMXxxRcEsMPGw1ICjIH6AOBAPz8afkB9rfyoO+g7L7pDueg5Ivig+DJ3lDdz9t52l7ZX9iY1/LWhtZa1hbW8tXt1RHWlNYS17HXcNj92KrZodrH2+rcMN5g31bgceGN4qvjAuVW5lPnOOji6E7p2+l66hDrmuuo63jrPOs0613reeuW64frOOsB6wzrO+ul6wfsUuy97CPtqO2P7pfvrfDZ8Q3ze/Qg9gz4Nfpv/Lj+5gBRAwIG9QgDDBIPJhIHFfYX/hoDHkQhdSRVJ0EqGi3PL3QyKzUOOK06+jwTP+ZAykK1RJtGlUgKSuRKiUvrS9ZL20sUS8NJAUgCRdtBRD5NOs81mzC7Ks4j9BwyFloPCwlPAhv72fOD7PvliODJ25XXlNOKzwLMqsmXyHrIf8m4yvLLmc2hz//SWdfy26Lg6eQO6Tjt3PHv9hL8+QDUBPIH9goPDhkRRBTJFoIYjxn+GX8aMRvFG9QbKxv1GYoYLBcQFvYUjROzEW4PKQ1QC8MJOghpBk0E9QHI/+r9Qfyf+q/4UvbH82nxWe+F7a7rmOkh53fk/eHr30Destwa22jZxddm1kPVq9SC1EDUE9QO1EfU7dTo1RjXdtjI2QXbbNz93bDfjuFl4yPlyOZT6PPpmOs47cruLvCC8cny8vMw9WX2g/ds+CH5pPkU+nH6nvrQ+u/62vqo+lf67Plx+ff4d/gj+Nj3kfdi9zT3K/dF9633ZPhC+UP6UvuE/Pf9mf+RAZoDwwXjB/0JUwy3DkUR7hODFv4YXhujHQUgdyL1JFonmCncK/4tHzCCMu00IjdbOSA7zjycPh9AlUHSQodD00OsQzhDc0InQZ0/aT1tOtg2gzLRLfMoxiNrHqQYfRICDEgF3v7g+Grz+u3T6M/jIN9Y2wjYqNX404fSg9Hz0PHQktEP0yLVltdC2irdUODH43/nX+tr71fzNfe6+h/+gwGsBM0HggrwDBkPzxBjEqoTjxQhFWEVPRXXFEEUaBM7EtkQXw/KDTgMkwrvCEAHjQXWAycClQAR/5H9Ifyq+h/5i/cT9q/0UvPf8UbwlO7h7C/rbem05/flGeQ04k7gbd6Q3LnaAdl91xXWydSg07PSCNJ90TXRQtGN0SLSGtNJ1KzVbtdb2Y7b6N1i4P7ineUl6K/qKe2B78/x4PPg9Zv3NfmX+p/7m/xg/dz9Nf5n/p7+kv57/nn+Zf6D/qb+7/5l/+L/jQBGASMCMAM7BGEFqgb7B1sJpwrkCwgNAQ70DhwQGhHvEbISNBO9EzkU2xR/FTcWDBcAGA0ZGxqAGxAd2x7rIAwjCiUWJ0gplSvdLSIwWDIiNKs1HzdlOGM5LTqBOmc69zn9OK43FTYbNK0xty5GK3YnTCPkHkoaexVrECUL3AWhAIr7dPZz8ZDs2ed444XfGdxP2czWsdQS0/DRcdGA0S7SWtPd1MbWKNn12/3eXuL05ZHpPu3a8HX0APhe+4j+aAH7Az4GKAi6CRoLKQz2DIANyA3ZDaoNVw3GDAsMQwt6Cq4J0Aj3BwgHFgYyBXwE5ANJA6wCAAJWAakA+P88/2n+cf0//NX6PPmF9671vPOi8WDv8+x36vnnkeVK4/zg1N7d3Afbetky2DbXg9Ys1irWj9Y+1z7Ygdn82rbcjd6T4Lri8OQT51bphOuA7XzvSfEH86X0G/Z095v4qPmg+mn7H/zL/GL98f1v/gj/kP8PAJcAJQG7AUoC9QKmA08EAAW3BW0GKwfjB6AIWgn/CaYKSAvwC5IMHA2gDSYOrQ4zD68PIRCpECQRtRFxEigTFBQLFR4WYBesGEEa6hv3HSMgaCLdJEMn2CmSLF8vKDLdNC83TDk7O/A8XD40P4k/TD95Pj49qjtuObQ2ZDMwL5AqYSXkHzkaORTSDRoHHwAO+Ufyiesn5Q3fQtn00yXPFcvVxxnFGcPawW3B28EiwzXFEchhyzDPn9Nq2Ird8+Jo6KntvPJy9xD8dwBrBPQH/ApeDUUPyRDoEbESAhP3En8S3BEyEW8QjQ+DDm8NZgx6C60K+gldCcIIJAiDB/QGcwb0BWcFtATRA78CfAEDAHH+rfyP+jL4svUP81TwbO1o6nPnhOSy4QHfetwb2vfXINas1HrTr9I+0jHSg9JC01nU1tWp18zZPdzI3mfhEuTI5oPpO+zV7lPxqvPQ9Zr3OPmY+sX7t/x2/Qn+fP6y/rz+s/6Y/mH+E/7L/Zb9ZP0w/Qr96/za/N38EP1r/dH9Q/7H/nL/LAD8APQBAgMOBBwFLQY+B2MIigmzCuALIA1QDnUPqxDkERETTBSJFd8WVxjeGYAbNB0KH/UgBiNwJf0nmSpVLRQw2TKvNVM4/jqbPc4/w0FQQ2ZE+ET4RFVEFkMeQWg+GjsYN1Ey8SzaJlIgchlBEu4KbgPT+1P0Au385Yzfd9n+0zDPDsurxxLFNsNKwi7C48JYxGjGFMl2zGjQ29SX2XnereMF6WLun/O3+HL97QEGBsQJKQ0KEIISiRQdFlQXMxi4GPUY4hiDGPEXLhdcFnoVdxRZEyES3hCXD1IOCA2wCykKjQjnBkMFoQPsARAACv7r+6v5T/fq9InyJfC17Trrw+hd5iLkneFE3z/dStuV2RfY6tb/1U3V2NTJ1ATViNVb1nHXsNhD2hvcKt5X4KHiGuWV5w7qhuzk7jTxj/PG9dj3v/mB+wL9Uf5i/zAAqQDVAOEAzAB4AO3/QP9+/qv9vvzQ+//6MfqC+fv4jfhG+Cr4Qvik+GL5ZPqX++z8a/5BAGMCsgQZB4UJCgynDj8R4ROeFl8Z9RtAHl8gfiKVJJwmgyg+KssrTy3jLr4w/zIaNQ034TjSOhI9Mz9ZQY1DgkXZRtxHm0gFSQJJA0hERlNDiz/VOm41ly/GKPEgAhiDDvUEpftI8u/o9t/31m/O1cY3wNW6abbJskiw366trr+v1bHAtIC44Lz2wavH782o1F3b+uFr6M/uGfVe+yMBZgYaCzoPExOkFuMZuxz4HqAg4CHNIoojFSRVJEIk3CMWIyUiECHeH3AekxxnGgoYgBXYEgcQ7Ax5CZsFfQFO/Rz5+fTJ8HTsGejh49zfK9zn2BHWmdN10cfPts4kzhjOis5Tz23QBdIW1IDWIdnd27veh+E45ALnx+lq7Mnu6vDr8rP0NPZ193z4Rfnt+VT6hfqS+o36TPrA+Rb5Xvig99L2+PUT9e7zv/Ku8bnwwe/G7uvtLu1q7M7ri+tm61TraevJ64nsiu3i7pvwtfIs9cT3ivqH/eoAiwQkCLMLNg+ZEuQVVBm2HNQfiyLwJA0nCikHK/cs+S7rMNkyxTTNNgs5bjvwPbZAikMlRrBIO0uGTYFPAlGYUVRRQlDFTphMREmoRAw/RzioMF0oXh+1FZULOwG49hrsrOEN2J/OA8aKvgK4hLJQrmCru6nbqDip86qyrYmxVbbdu9/BScjdzvLVH91P5NHr9vJ7+Zz/IgUWCnUOAhL3FD4X2Rg5GjAbtRvoG9kbtBtxG3AbmhuSG1cb9hqUGggafBm5GL8XTBZTFAkSRQ8tDLoI7AS4AD78g/ej8sjt5Ohe5EPgftw72ZfWqdRp0wzTftO61IrW/Nj020Dfw+KV5qHqpO6E8hX2T/n7+x7+0v8OAZwBigHRAJb/Bv49/DH6APjR9ZzzhvGX7//tr+ym6/fqjOpi6mjqkerN6h3rS+tW61zrUesa69jqj+o56sTpSukN6evo/+h16WXq2+vD7S3wGvNu9jr6ZP4RAxUIBQ2hEdgVzRllHeEgBSSWJr8oUipUK0wsaC0pLvQuyy+vMP4xpDOBNdQ3iDo/PVtA3kOGR1pL7E4eUpFUXlYmV2lXUFfMVS5TAk85SSBCOTovMfwmGRxPEBkEz/el69zfBtXcyu/BNroStNOv8ax3q0Crn6zBr020U7oUwQvIBc/O1RPdXeR96+PxefcS/Kf/1wJmBbQHhwl+CgILaQsBDLAMgA1XDmcPthADEqwTuhUZGEsa1BvJHEMdZB1GHXkcmhqzF4kTlQ4ACQ4D7/yF9tPvxegL4pnbCNak0RnOgct6yWDIdciDydvL4s580oTW8drf3/HkCOrY7lLzHfc0+rv8gP5V/yX/+f00/OP5D/fm82Twp+zm6FLlFOJL3wTdRdvm2R3ZsNir2B3ZBNpG23ncx90y37fgE+I94z7kCuV+5cDlB+ZA5m3mlObE5gnnpee56BLqkOtw7W/vg/Ei9Mn2wvmt/GL/CgJpBKoGDAkBC2IMsw1+Dm0PFBD1ENERwRL3E3YVfxdJGikeoiIMKOUt+TM4OtBAtke+Tn9VMFuhX2ZiY2TPZZhmX2YFZUFig148WudUU0/NSIJBWTmWMHMn6h3eEwsJu/3u8VHmSNsV0RTIicASuQWzka4DrDCsMK4MslS3QL0gxI7M6dUG4CLqP/P7+v4BRghLDhMUnBgxHJge6B9WIdUiSCSqJVsmoyaaJrkmMCdmJwon2SXKI9QghR3jGQwWbBG1C/gElf1B9mzvW+mk40/eSdnh1HPRDs+/zXfNxs3Vzs3Qd9O31oXaiN6e4q7m0uol73rzbvf1+v/9XABSAtcDzgQEBWUEEAP/AB7+qPpj9oXx9esg5kvgz9oY1qLRyM0TymDH58XGxfPG3shny2bO59Gt1d/ZFN4A4nvlSugk6oHrduz27ALtVOx263nqcenV6LPo9eiN6ajqWezO7vXxbvVg+fv8UQAlA4EF0Ae7CWQLmwyYDYQOfw9QEFURkhL0EwoWnxg7HGkgDCXBKUgu7TLYN1U9PEPpSKVNFVLqVUNZXly5Xttg52L8Y9tkgGWSZRpl4WOMYeBdg1lEVOtN80UYPJUw/CNhFgAJXPyM72Xjhdeny13B3LjcskOvK64Yr0+x5bT1uWDB9sld0wrdEOah7qD2Yf7iBekM7hLQF1QbnB6iITEk0Sa/KFwqaCsqLOEsZy1zLYcsRSqaJuUhhhyNFvoPcAjw/8D2hu225HPcN9XbznTJ0MSVwaq/Xr+4wDnDpsZVymnOxNJp10XcQeEN5gLqn+0L8Rn0Qvc0+uz8Ff+4AF8C1QPIBAYFswSZA78BJ//C+6H3rfIo7SvnqeAy2u/TGM6dyPPDtr8xvI65zrdVt3a3cbgOugi8SL6AwBrDh8X0x1rKysxGz6vR+tMw1o3Yn9ra3MLeZuAE4jDjheQY5qXn1Okb7DXukvBA8hL0a/bb9175Q/rx+s/7Rfwf/br9nP6i/3MAqAKSBUoJIg/PFP8a7CAlJoorWjDmNNo4NTyOP7VCG0U/SAZL7Uw9UDNT+1ZJXMNgnmW7aXps524IcSly7nKpcMtr9mMDWJJKIzqFKG8VKQJM7m/ccc34wEi5R7QjskezebWOukfDEs2d2RvmG/Em+1QDrgkBD3oTZBbPGGAaQBtzHK4doB8LIogkNSdqKf4r8i58MdYz1TSXNBszEzAaLG4nYSFWGiYSqghX/wb2ye1l5pbfhtnW02fP9cvqyZbJPsriy5fO09FV1Z3ZO94045Ho5e3y8tL31vv6/psBMgMyBPQEngVQBv8GZwctBzIGrASPAi4Aav3z+UX2Q/K27Y3ofuIx22nTE8x2xRLA0LsZuIS18rO+srCyebMYtSy4l7t8v6/DN8iczMTQhNSk18naot1/4GnjHubL6F/ruu0X8FbyQPQy9qX3afis+Aj4vPbD9ALyyu6B61Xnh+MI4IPcVdog2CPXV9jP2jnf3OXp7Dj1s/14Bn0Q9xlFI38qGTEuNig5XTyrPfk+lUBNQfdCiUXgSD9OB1X2Wxlj42kmcKd1zHmpfJt88Hv5eAJ0YW+YZwFfcFRgRiM3dCZjFUEGw/gH7lflst1r12PSNM/JzVXPwNLK17zd8OOY6tXwOPY1+/H+cgIkBlAJFA2dEDYUhRdXG90feSSUKT8uVzIfNvs40jq8O6g6DDjXM/UtlCezIF0ZkRECCWgAg/ft72vpEOSX4DTd6No22dnXtdcQ2JzYlNmq2rjbiN3i397iG+Yv6S7sM++K8iD4Rf8dBqgMdRLLFmMatxxdHf4cVBqLFqoRfwsmBVr+ePfQ8JDqvOTB36Xb/df/1JzR7c3Fyb/EEsAsu+q2rrPYsIOviK/5r9CyKbYiupm/PcWBzMrTstoz4XPmwOrD7YjvN/C97+Tuju007AHri+lm6EHnVOZ+5cnkguRv5K/kfeTT44/i2+Cf3hzcg9mF1wfWVtUD1iDXZ9lY3GHgX+Ws6hDxrPce/sYEagqLDxUUERhYHIYfxyIPJgopsizFMSo3nT1NRaZM31WfXWBlTGsEb1tx028Fbnhpf2RIX01ZdVRKTnRI5ELoPfM5ITZTMzQx/C65LcwscCltJDEdwRIBCKz82fGA6IHgztp11rPTwtJ51G3YBN4J5cHrnvJy+Nr9uQL+BY0I9glQCksLWAxMDaAPzxElFU8Zex3TIoknjCviLcQt1yvjJ/4iHB1BFpoOLgaC/Zz1Au/O6QvmY+M24uTh7OJB5Dfl0OVc5ZjkVOMJ4vDg49/83mzeX96r3nng1uPp6E7vMvZf/UQEfwq8D38TIxX+FAUTDRA8DKgHlwLo/EP3K/LC7U3q/+d05p3laeQL45vg6d0C2yjXndNez0LL3sfYxLrCJMFzwNrAiMKxxQfKus7n01PY39y04Evj1eWx5nPnTOf15l7mkuWL5ZblOuZP5+vo4+pp7W7vjvEd89LzafQg9OzzOPNl8qnxqfA38ILwUvHb8h/1XPec+n392ACNBL4HqwoHDfQO2Q9UEBwQ/w+RDqYNgQ2NDD4O8Q9REgUWPRnyHDkhdCXOKpUwIDV7Opw9bz+GQHk/2T5iPRs8njwbPSs/a0KdRI9Hz0kXS/JLCEtSSQ9GE0GHPBg3IzFHKoYi7BkkEUsIv/8v+ITxRuyw6GHm4uVV55Toz+tI7kDwGvI98t3y8vLw8xf2Pvmj/bIC7QfqDboSaxZzGTEa1xqBGjga3Bl1GQ8ZqBf+FWYT5A8LDGAHJQKs/Cf3QfIS7hvrAek456XlPuQx4rzgG99z3XjcZttk27Lbtdx23nrg9uKh5XXoXuvK7pny6Pav+0UA0ATKCOELIg5GD2YPbA6+DJcK8AdDBcoCRwAj/nn80vqG+fD31PVI8zXw4Ow16Z7lFuJE3h3bI9g/1SHT6NBPzwvORc30zD3NOs6Sz1fRQ9PF1HfWPNjR2ZvbCt253oDg6uLo5Q7p2Oxl8NDzx/bh+Iv6efu6+7r76vrb+Zb4Ffce9i31CvXy9Ab1hvUV9vv2yPet+JP5PfrN+nH7xvtO/JX8Bv1n/XT9PP74/vj/AAICBLkGQArNDdkRQBWoF5gZhhqqGu4bxxxxHqYhgCSOKRgu7zIyN2g67DziPb4+xT6bP9RA3UJWRXpHhknDSsxLZkzTTM9MPk2cTbJOtVDwURhSZFAyS0xDkDgzK3gd0w43AqX2Ou395lziYuB93zzfM9/E3hreSd3X3NbcNN003l/fU+F443PlHOjQ6vft3/Fk9kT8wAIqCWUPQBSwF4EZthmMGJkVqRHqDLoHRgOK/7b8pvrV+HX3zPVx9Ofy5/DE7unrPOl65gPkNuLD4M/fUN/03v3eqt+t4M3ijuXW6MXs2fBK9bD54/2+AYYEgQa4BwsI7weUBygH/QYLBx8HOgcDB10GigUSBDsCwP/Y/Nj5hvay857wR+3m6dHlsuFj3SjZh9VR0gLQos4AzkfO084E0HrRJdM/1QfX5tjl2vfcWt+z4WXkKOer6VTscu6M8J7yJfSv9c72sPd0+LP44viQ+DL4kfdV9gP1WvOB8cvvPu617DXrM+pt6cXos+iG6JToF+mJ6Xzqq+uJ7TXwpPLT9Wr5ffwDADoDxQVKCLgJAgupDLUNnQ9eEecSMBXIFnQY6hlVG9Acgx17HjEfqB+tIJchHSPTJGsmDCn4K4kvdjPhNwU9Y0HCRe5JZk2CT8VQ2lALT5BMPUliRTZBSz07OQg21DLCLzstbSloJacgzxpTFHENnAZaAPj5lPQg8CbsXulb5/zl2OTE44jj4+Nn5NfloOeS6e7rm+5L8Tr0lvfa+lr+2AFeBQkJpAxLEFUTUhWhFmIWOxVCExwQuAzACBQF8gEq/2X9Rfxw+yX7xPor+iL5kvfV9ZjzGvGs7kDs9Onc5wTmvuTV46jjTeSo5c/n0+pe7mTyrPa3+oz+lQGsA/QEbAVMBZEE6ANZA+4CCwNbA9cDZwTZBPcEVgTiAt4AKv7++n/3tfOq737rgeeY4zDgT93H2qrYMtc61nbVVdXR1W/WTdf+157YV9ng2UDa29qx2xHd5d4s4efjzuZE6oLtgfBd86n1sfdm+YD6KPtk+2v7/fpc+nH5Bfi09kr1vPND8tTwme+j7vbtju1H7S7tQO1l7ZntzO3+7XXuJ+8F8B/xUfLZ83f1GffY+Jv6dvxN/ioAAQK5A3AFPQfbCI0KHwyRDRMPkhALEnQTDhW6FmMYHRrrG9Idjx+dIbQjsCXTJ7opzCsJLkAw7jKcNXc4VTtoPZE/vUBAQVFBEUDOPtQ8dTpZOKU1JjNtMC8t7ykWJj4iAB6DGRYVWhAUDNsH4gM0AAj8+/eS81/vSutP5wTkMeEC3wbe1t0n35Lhe+T86HXtbvIT98z6gv76APYCQgQjBUoGBwcxCBgKPQt0DMENiw7mD6cQIhEoEXUQUw+IDakL5gg4BZABF/0e+Bzzp+5d6/To2ec95zbnsOeB6EfpRuqq6kHqyeni6DDoo+fd5wvpwer+7Njv3PIO9qj5Mv3mACsEqQaUCLcJMgr8CUAJ4QfHBZUDSQET/y/9lvsx+jL5VvhQ91j2v/S/8i/wEe2q6Q/mBeNU4EfeRt1Q3Ancjdwy3Vfepd8V4XHiU+Tp5ULnuujA6cHqiet57C7tuO1t7izvEvBe8bLy+PNW9Tf21vbB9gH26/RG83Xxeu957f/r3Ooj6gHqGOrT6pHrPewD7a3tUe6b7uTuHe9M76PvTfAh8STyL/OK9DX2lvey+dL7CP4CAY4DWAY7CaQLYQ57EO8RWxMIFHIUkhVcFqkXaRm4GlkdCB+bID4iVSOeJOUkoiWJJjQnoihcKp4s1C64MJ8ydzSwNrs41zpkPT0/fkCXQWlCk0HlP0c9GDkcNCUuByhGIowd8hjNFR0T9A9mDeYJrgbxAir+c/l29JHvAOsp52DkM+Iq4U/hAeKJ45/lGehg6wjvofKR9jr6Df1y/5cBNgOHBAAGXQfYCI8KWgx5DroQvhJGFDsVfhWUFCATXBHIDuoLxwgtBXMBvP0w+s/2ifON8IPtv+qI6IXmL+U85EHjeOLe4X3hVOF44STi7+IH5KzlvudS6lvtnvAw9NL39vr7/aAAmgJhBLYFwQZrB7oHEAgmCFcIhwhvCEoIwQfsBtcFRASJAqMAgv4s/MH5KfdH9IPx2e5H7Onp6+ci5qfkluPr4pjij+LC4j/j3eOM5HrlieaX59ToIepf66Dst+3c7uTv6/Dg8aDyX/MJ9In0CvVZ9Uv1KfXT9Cv0iPO/8sPx1PAD8FXv7+667qzuxO4X76jvRPD38JDxO/L58qrzcPQr9en1yfau98H49flO+/f8vf7AAOQC5ATVBpsIIwppC3IMUw0hDvIOyw/IEOwRPRPEFEkWthc2GX8arRvuHM8dwR6OHygg6CBVId0huiJfI1sk2CVeJ2QppCvgLUowNjLQM7k09jSvNGEzkTHyLscrNSg5JDgg6RvGF7YTyw+NDGkJ1gaqBIkCswCD/in8OPnJ9T7yfO5X65To/+W85E7kz+TQ5ivpWuwW8JzzP/c8+rP8hv7D/8IARwHQAWYC8QI/BOMFlwelCWwLFA18Dk8Pfg/nDqcNzAttCYsGOgPP/2P8Lflk9vXz8/FV8CTvM+5f7ZDsp+uW6pfpq+i/5w7nquaW5vLmxOc06STrde0X8NnyjfU6+L76Dv0r/+cAXQKgA50EXAXqBVMGjAakBrkGpQZ3BikGnwX5BBwE5AKFAdH/0/25+3T5Nvfm9OLyCfFb7zruM+1s7O/rlut362HrYOte63brouvP6xvsYuyx7DTt0e1z7kfvFfDm8MfxjvJA89fzXPS+9Az1UfVf9WH1efWM9an11PXv9SD2WvZ39or2h/Z79lP2HPbt9bP1mfXD9f/1ZvYB96P3ePiD+Yn6mvvP/PD9CP8GAAkB8QG4ArUDlwSPBaEGnge3COgJ9womDEgNbQ6QD3oQexE5EtYSeRPcE2AU6BRbFR4W2RalF6gYjhmaGqcbjRxpHRIekB7pHgQfpB89INQgCSJZI+4k0yZ0KP0p9SryKo0qGikBJ4QkJyHSHYYa8hajE18QcA0CC6QIygY7BRME+AIVAhgB3P/f/ZT7yPhy9a7y7e/i7d7s6Oyb7WfvHPLA9L338PrR/f//2AE+AxwEvAQxBVsFPQUsBSwFCgUWBRYFGQVDBZ4FLwaSBukG8wZ/BpIFOQRRAi4A0P14+2v5fvcc9jL1wPSf9Nv0GvUp9fr0nPQJ9BvzOPJC8WTw4e+j78jvMfAA8e3xBvNp9KX1+fYD+O74u/k++qb6sfqh+k760Pln+aH46veO9z73P/eJ95r3Nfeh9tr1q/Qx84/xje+E7dzrQOrZ6LXnBefX5gTnoudr6F/pVOpz64vsYu0I7pPu/O5E76/v8u8u8Hzwy/D+8DTxd/HJ8T3yx/JZ8+TzZPTe9EX1dPWU9X71DPVm9Kjz8PJR8v/x8vEX8nbyBvOb8xT0rPQW9X/11fXa9fD14PXg9e314/Ub9mz2B/ca+Hj5KPtC/VD/hwHAA4UFNQesCM4JygqaC0wMCA3yDSUPuxAxEkwU9xWYF3sZgBppG0gcsRwdHdAdfx6SH2AgqSENI6EjciTGJA4lqCVjJownHinPKmYstS1yLokudS2lKxApoSWsIY8d8xmMFkUUxBKQEe0QLBBxDzMOpwzPCm4I/AVwA8UA9f0T+0X4svVm86Pxh/Az8MzwJvJe9F73mPr//RIBlwMmBWYFrgQoA3YBz/+2/j7+f/6h/2QBvgM5BpsIowoADGcMGAz7CiYJ9wZ1BOEBMP+l/Gf6fPhE96z2gPa09jf37/eo+Dj5iflt+cj41feS9gL1kfM18kHxuvCd8PjwjPF58t7zRvWF9qv3lPhX+Rj6c/rT+vH65foa+xv7SPuP+7f7Cvw4/D78J/yi+xb7HvoV+fv3jPYb9YTz/vGu8ITvc+6z7Tztyuxr7F7sBOy3633r+erJ6oLqReo26knqluoV69PrvOyv7aHune9j8CTxl/H+8WjymPLF8tLyDfMp81jzkPOe8+fzOvSA9K707vRM9Vz1ePV+9XX1dPUy9S31L/Uh9Wj1zPU89gv35/fL+N/5wfq7+3r8MP0E/nT+Mf8LAM8A0QHIAvYDJQU/BqgHwwgYCmkLVgyMDXYOVA9CEOwQxRHBEm8TQBQhFRMWGRcXGD4ZDhpAG/AbTh3DHRce4h6YHgsf7x5KHrUeRB8gH5wgOSFvIhUkBCRaJQ4l0SPkIRMfHxxpGNAUCRHKDbIKhQjUBoEFsARMA14CqQH//9n+rf3I+2/6Fvnl91f33va39t/2R/fu94X4V/kY+tL6uvuj/Br90f26/hn/wf9wAN8AigHrAWECOAOdAzgE3QQBBVMFmgWuBbgFdAXtBGwEnQOjAsMB1gDM/7b+w/3o/A78Mfte+qP5Avlp+Kf3KPeN9uH1mfUz9ev0DPUr9XH19fWL9kD3/PfJ+Jj5XPoT+4n7CvyH/Oj8Qv2S/cr96v0d/kH+Uv40/uj9kf0N/VX8afto+hr5y/ee9kX1F/QI8wDyKvFi8L7vB++B7hnuf+1C7ezsoOyl7KLssuz07Fbtsu0/7vrup+968GbxJ/Ic89XzM/TX9An1G/UY9Tz1XfU89YP1jvWY9cH15fUE9ij2KvY49gX21vWa9fz0mPTq83nzDfOU8nzycfK+8jTzIPQX9VP20/cb+aX6Fvxm/cr+CAASARUC9ALcA6QEAAWYBf0FRAatBh4HeAcWCO8IAgq6CxUNvA7xD6QQoBGUEVURjxCgDwMPXg5yDtoOqQ8+EdQSdBQcFh8X4ReAGMAYSxiMF4gWIhUXFHETRRNKE5MTLBRhFLwU5RSXFDYUWRMZEpEQpg6ZDJMKnwjiBm8FEwTiAr0BtwDq/yr/Xv61/Q/9Z/zv+0n7yfog+n35R/kC+Sb5kvkk+kX7dPzz/Xv/8wCKAooDYwTcBPIE1ARmBCsE1QOUA48DrQNOBB0F6QXiBq8HTQizCMIIcQiIB4AGIwVLA30Btf91/mn9tvwk/M/71Puc+337Dvs9+in5F/j19pX1rfQJ9O/zFvSW9KX1x/ZE+Lz5Cvs//P38s/0u/mz+tv6T/q/++v4U/3z/0P8TAHkAuQDTAKEAIwBW/y3+1vwv+4358/cz9nL0/fKt8Y3w3e9u713vhu/T7xDwT/Bn8C7wBvCY79/uI+6K7TftDu107RjuDu+E8OzxPvOC9Jr1SvbE9hD38fa79nz2VPZk9nr2lvbo9mf3/fej+HT5QvrN+n376vsG/A78zPuT+1X77/q3+p36yfo4+8P7nvyE/Y/+l/9yAD4BvwEBAhMCCQLEAW8BOwExAV4B0AFOAisDVQRTBWwGXAczCM8I/wgjCeoIoAiCCFUIfgjcCFUJagqUC9UMPQ5CD3wQURG1EQ8S6RG5EV4RERHbEHwQahCNEKsQ2hA2EYsR4hEwEjES6BFdEX4QUg8EDoIM1QotCY0HCQaaBHUDlwLoAXcB/gByAMP/5v7Y/cz8q/uO+p/50Phi+D/4dvgf+ev51vrU+6P8YP35/WX+2/4f/1T/qv/1/10A6wCZAXUCKQPrA6sEMgWoBdsF1gW2BVIFwQQfBGADlgLXAS0BlQAQAK//d/9Q/zT/Dv/J/lf+wf0G/Rf8Ffv++R75Xvje97f3zfdG+AH5C/pF+5f8zP3O/qL/RACqAMAArgB5AFgATgBqALIACgF0AeUBQgJ2AnICIgKfAdMA2f+e/iX9q/sn+rz4h/db9mL1mPT485DzPfMM89nyr/KK8mvyT/IP8r3xXPEK8cLwkPB+8J7wBPGs8aLyq/Pm9B32SPdx+EX5B/p1+rD66/rv+gP7F/s7+4776Ptm/Ar9i/0X/oD+uP7r/t7+pv5K/ub9hf0h/db8ofxy/GT8gfyg/NX8BP0h/TT9M/0l/RT98/zc/N/8CP1d/dT9g/5L/zcARgFFAkMDLwT+BLQFUga9BjEHmgcKCKQIQgn8CcIKlAtKDPcMlA0JDlsOlw7NDuEOCA8fD0IPcg9/D6sPpg93Dz8Plg77DTUNOgyDC38KvwkNCVwIBgifB1sHHgfKBnkGDAaOBfkEPQR/A7kCBgJnAe8AqgCIAI0AsgDyADgBggHAAfcBIgI3AkcCSQJNAkQCIAIMAvAB4AHsAQcCSwKxAiwDwgNtBCAFrgUcBmkGXgYGBm4FmwSUA4QCgQGOAMv/Qf/x/tv+6P4Z/y7/L/8e/9T+YP67/ez8Mfxo+8H6YvoX+ij6fPoK+/D7zfy0/aT+Yf8JAHYAqQCeAFQADAC2/2H/Cv/G/qP+oP7I/vv+Qf+B/6P/vf+a/0z/vv7v/QD96/vC+pn5ZfhX93T2rfUm9az0U/QM9NjzuvOP83TzTvMs8yjzPvNr87nzHfS19Hj1NvYY9+r3sPiF+TP64/qC+/z7d/zu/Gz9D/59/gv/rf8qAMEAPQGpARQCUQKKAtAC5gL9AgUD+wL9AtkCvAKLAlYCKQICAugBzQG5AakBrwHAAc4B0QHUAdcB1wHNAdEB2gHgAfYBIAJdAqIC+AJhA80DPASiBBMFcgXBBQ8GTAaHBrIG3QYUBzkHbwelB9YHCggrCE4Ibwh5CHgIXQgrCPIHqwdfBwwHrgZaBgcGqwVJBeUEhgQeBLkDYQP5AqQCTwIIAtUBnQGAAVUBOAEGAdwAvACEAGMAKgAGAOX/yv/O/8n/5P/8/xAANAA4AEIANQAdAAEAyP+a/1n/H//v/rP+h/5b/i7+Bv7Y/bL9hP1M/Qv9x/yB/D/8+fu7+437X/s8+x/7Afvu+tL6vPqp+o/6f/p5+nj6hPqf+sX6+vo2+3z7vPv8+0D8e/yu/N/8Df04/Wj9nP3S/QL+Rv6C/rf+7/4b/0j/c/+V/77/5P8FACsAUACDALcA5AAcAUoBeQGoAdQBBAIkAj8CWgJvAogCoAK9AtcC+wIdA0cDcQOhA84D7QMIBBgEHwQcBBcECAT6A+oD3QPWA9YD3wPoA+wD7wPxA/MD8QPkA9gDvgOjA4oDcgNdA0IDNAMfAw4D/ALrAt4CywK5Ap8CfgJbAi0C/wHOAZQBWwEqAfgAzACkAHYAUQAsAAkA5f+7/43/XP8m//n+xf6U/mj+M/4N/tb9of1w/TD98/y5/Hn8RPwT/OX7w/uk+4/7evtr+1v7UPtG+zj7K/sh+x37FPsS+xP7Fvsh+zH7SPth+3v7j/up+737y/vg++v7AfwQ/Cn8Rvxf/IT8pPzJ/O38DP0q/Un9Zv2A/Zr9uf3X/fv9H/48/l3+fv6k/sX+5f4M/yr/Tv9x/4z/tf/Z//7/KQBMAHgAowDIAPUAIQFMAX0BqQHZAQsCNgJmAosCqALPAukCAgMdAzQDSwNiA3cDjAOeA68DvwPLA9YD4QPnA/AD9APwA/QD8gPvA+wD8AP2A/sD+QP3A/oD9QP2A/ED7gPlA9UDyQO7A68DlwOIA3wDbwNaA0UDMQMbAwcD8gLdAsQCpgKKAmwCTwIwAhEC7gHQAa0BjwFvAVEBNgEeAQoB+QDlANAAwwCzAKQAkwCHAHsAaQBaAE4APgAqABgAAQDq/9P/uv+l/4j/a/9O/y//EP/2/tf+vv6k/on+dP5f/kz+Qf4y/iz+Jv4f/hn+F/4V/hP+EP4O/g3+DP4T/h/+Lv4//lH+Zv57/o3+ov61/sT+0/7i/u/++/4F/xD/HP8p/zX/QP9P/1z/af94/4X/kf+g/6//uf/E/8//2//m//H//P8CAAUACgASABUAHQAhACUAKgArADAAMQAyADEAMQAvADAAMQAvAC0AKgAoACUAJAAjACMAJQAlACUAJQAkACIAHAAVAA4ABgD8//L/5f/Z/8f/u/+w/6j/of+b/5n/k/+O/4n/hv+D/4D/ff93/3T/cf9y/3P/d/99/4f/kP+a/6X/rf+2/7z/wP/B/8T/xf/H/8j/xv/H/8X/xP/B/7z/u/+1/7L/r/+q/6n/o/+f/5v/mf+X/5P/j/+L/4j/hP+E/4X/iv+Q/5P/mP+a/5z/nv+g/6T/pv+r/6//tP+6/8P/zv/Y/+L/6//2//3/BQALAA4AEwAVABMAEgAQAA8ADAAKAAYAAAAAAPr/9//z//D/7v/s/+z/6f/m/+T/6f/r/+//9//5/wIABAAJABEADgAQAAkADgARABIAEQAdAB8AJwA4AD8AVwBWAGYAdAB1AH4AewCAAIAAegBsAGcAXABOAEUALAAkABUABAD4//T/6v/U/8v/wf+u/53/kv+E/3b/a/9v/2v/aP9j/2T/bP9h/2P/Uf9I/zv/Lf8m/xf/Gv8Q/w//Ff8U/x3/IP8w/zf/OP9I/0//YP9p/3L/gf+N/5b/l/+Z/6n/tP+3/7f/uP/K/87/yP/N/9P/zP/M/8b/xf/F/7z/u/+4/7X/q/+p/6b/nf+W/5H/j/+J/4H/f/97/3X/af9b/1L/Q/83/y//Iv8R//7+9/7o/t3+2P7R/s/+yf7L/s/+2f7h/u3+/P4N/x3/I/80/z7/TP9Z/1//aP9t/3b/fv9//4b/iv+O/5T/mf+g/6H/rP+w/7H/uf+7/77/xv/R/9n/4//p/+3/8f/w//j/AAD2//X/3v/e/9r/yf/R/73/y//L/8P/xP/A/8z/yv/N/8j/w//G/8X/wP/H/87/2v/f/+L/9f/6/wIAEQAcACYALQA7AEsAVQBiAGoAgACTAJwArQDHANcA4wD8AAcBHgEhASUBMQEzATYBMQEuAS4BLwElASYBJQEYARwBGAETAQ8BFgEZAR4BKgEzATEBQAFIAUoBUgFPAVkBXQFcAWYBdQF9AYEBjgGZAaUBqQG0AbsBvgHIAckBygHOAcsBzgHSAdgB2AHWAdgB0wHTAdABxQHDAbsBtAG0AaEBkgGMAXoBbwFeAU4BRAEwASIBDAEAAfAA5QDSAMQAwACyALAAmgCfAJkAlQCXAIUAjgB9AIEAfgB4AHoAcABrAGMAYQBZAFgAUABKAEQARQA+AEAARABQAF4AZwByAHMAdgB/AIoAlACYAKEAogCiAJ8AlgCXAIoAggCBAHAAbwBoAGEAZABcAFsAWwBSAE4ASgBCADoANQAvACcAGwAYABsAHQAqADMAMwA5ADsANwA0ADUAKQAiABkAEgAPAAoACQACAAcA/v/+/wUABgAJAAwADQARABEAAgD5//H/7P/n/+T/3P/T/8r/wv+3/67/of+S/4z/hf96/3P/bP9f/1X/Sf81/y3/If8d/xz/Gv8Z/xz/MP85/0z/Yv92/43/mv+s/7X/u//B/8b/y//R/9f/4f/l/+r/8P/s/+7/8f/z/+z/5//p/+D/4f/k/9//5P/u//H/BAATACgARwBkAHgAdgB1AHQAaABgAFkAUQBMAFoAZgB3AJoArQDCANQA1wDZAMMApwCAAGQATwBUAEgAWAB8AHsApwCgAIcAlABqAD4APQAlAFcAgQDCAB4BRwFyAUkBMgHyALUAewBcAFMAcgCgALoA5gDaAPEALQGLAQgCvwKHA18EGgXFBU4GZQZiBuAFHQU2BBUD/wEEAVwAy//N/7b/GgCfAOoAuwHuASUClwHYANr/7P76/Vr9X/1o/TH+wP5b/8T/AwBUAGEAAACV/wf/Nv6F/e/8OPzV+8z78fuE/Bn94v2j/nr/GQClAAUBygE5AogCLwNQA6cDpgObA3sDIQOhAh0CdAHaAGIAzP+U/1b/aP+V/6X/4v8ZAAgA3/+e/17/HP+x/n7+Cv63/Vb98/zT/Lf8AP0z/ar9+P1u/qP+g/6H/kr+H/6y/Xz9Uv01/Uv9gf3X/S/+gv6W/pr+fP5X/hH+y/2g/ZH9kv23/Qf+Qf6N/qj+1P79/tb+z/6y/qX+4P7f/vj+NP9W/2n/cf9q/2b/WP8i/zf/N/8m/1L/eP+K/5//kf+z/5T/aP9X/xX/7v4J/wv/3/4q/zH/Jf8b/zH/Nf8a/zL/K/8r/x//Jv8Q/wb/EP/9/hP/Bv8M/x7/IP9F/1f/av9z/4v/hf9+/5T/gv+U/5n/rv+7/7n/xf/F/+L/4/8JADkASwBYAF0AbQBfAEwATQA2AB4AIAAWABcAGwAgAD0AQABPAFsAWQBmAGIAXABhAFsAWQBrAG8AdgB9AHoAegBmAE4ASgBCACoANgA+AEMASQBPAFIARwA+ADcALAAVACIAIQAcAD4AVABdAHUAhwB9AHsAVwBCAC8AEQAcACYARwBgAHkAjgCYAKsApwCmAJwAjQCNAIwAiACOAIcAggB6AGgAWwBaAFYAXABsAGIAbAB5AGYAcABzAGYAdwB5AHUAjQCTAI0AqQCnAKoAoACjAKYAnwC0AL8AywDEAO4A+wDeAPYABQH7APIA4QD7AN4A4QDfANcA7gDoABEBDQEBAQkB/QDkAN0AuQCpALEAkQB3AHYAZQBsAG4AbQBgAHcAjwB7AHQAaQB6AF4AWwBoAGoAcQBgAGgAhABSAB0AJQAFAO//zv/n/xYABgD1/xcA9v/N/9f/qv+1/43/QP93/yX/af+M/1P/3f+v/9X/pf85/xb/C/+9/nv+pP7E/iz/6P4a/0T/DP9C//3+x/4m/xD/i/6e/vD+rv6e/sP+wP7S/rf+rf7S/q/+sv7P/rL+0/7S/tD+8/7S/qf+o/5k/kz+K/4Y/j3+QP5U/k7+b/5W/k7+Vv5N/kv+Pv47/kj+a/5r/pb+pP60/tL+yf7Q/tr+7v4A//v+//4D/wb//v4Z/yT/OP9Y/2v/pf+y/8v/xf+2/8D/r/+o/6n/of+J/43/if+F/4X/cP9w/1//Tv88/07/VP8//0b/O/8r/x//PP82/y3/Lv8//zn/Kv8u/zv/Uv9H/1T/Ov82/yH/DP8B//H+/v7x/vn+Bf8E/w//FP8x/0f/Vv9r/3D/e/99/4j/jv+a/5b/jf+K/47/lf+T/5b/k/+t/67/uv/M/8P/vf+8/7f/u//C/7j/0//N/9T/6P/i/+7/3v/Y/8X/sv+t/6H/of+b/6D/iP97/4r/ff93/33/X/8u/y//FP8U/xn//f4G//P+9v4J/wf/B/8H/xD/Gv8m/zj/Rv9f/27/eP+H/47/lP+r/7D/qP+o/5X/jv+L/4r/k/+U/57/rv+n/6X/n/+e/67/qv+x/8H/x//J/9X/5f/9/w8AGgAnABEA9v8BAMv/x/+s/6j/1P+y/9f/y//D/7//ov+a/5P/nv+h/5f/lf+o/7v/v/+7/9b/7v/u/wAAFAAnAC4ANQBJAEgASwBIAE8AagByAHsAkACjAKEAsQC9AM4A3gDnAOsA+QD4AOoA+ADxAPAA9ADnAO0A8ADhAOQA3gDWAMcA0wDOANQA7ADqAPgACAH/AP4A/gD2AAMBBAEAARMBIAEpATwBTAFXAWMBYwFjAWgBYwFjAWYBWAFUAVUBXwFhAXMBdQF0AYEBfgGCAYMBcAFjAWcBZAFpAWEBXwFdAVQBSgE3AS8BIgEcARAB9ADlANUAwgCpAKIArQCwALIAowC2ALQAsgC+AL0AxwDFANMA1wDYANgA2QDJAMQAvACqAI4AbgBjAE8ARQA3ABgADAD9/+//4v/a/9H/vf+z/6f/l/+D/3X/Z/9R/0T/M/8q/xv/Dv8K//v+8P7t/un+2v7Y/t/+3v7f/uL+5/7n/uf+8f72/vv+/v4J/xL/G/8j/yb/L/8y/0j/U/9j/3P/eP+B/37/fP9+/33/gv+K/5X/qf+3/8D/yP/S/9j/1v/P/9X/0//U/9X/zf/M/8v/zv/M/87/1//R/8j/uv+n/6P/lv+I/3j/bP9f/1//W/9U/1f/Qv9K/z7/Nf86/yr/Pf86/0L/T/9a/1z/T/9X/1L/U/9S/1H/T/9Q/1D/Uv9Y/1H/WP9W/1j/Xf9i/2n/bv90/3r/h/+O/5X/nv+t/7L/vf/J/8r/z//V/9b/0P/G/77/tv+z/7X/uP/B/9L/5P/5/wQADwAUABkAHwAXABIAEgAZABwAJQAvACwANQA6ADAANgA0ADAANgAuACgAMQAnACkALgA1ADoAMwA3ADUAOAAxADQAPAA7AEYATQBQAFkAXwBhAGkAawByAHgAgwCLAI4AmQCYAJ4AlgCVAJsAlQCSAJAAjwCLAIwAhwCJAIMAegByAGkAaQBXAF0AVgBNAE4ARwBFADkANQAwAC0AJwAeABQADwAKAAUA/v/2/+7/3//Z/83/wf+2/6f/n/+V/4j/hv+H/4n/jf+V/5z/ov+f/5z/l/+Q/4z/hv+D/33/eP9y/27/bf9y/3H/cv92/3b/d/96/37/ff+B/4T/g/+G/4n/iv+G/4b/gP+A/4D/f/+D/37/f/98/3//fP+A/4j/jP+X/5j/pP+n/67/tP+7/8L/w//J/8b/x//I/8T/x//I/8r/yv/H/8j/xP+5/6//rv+p/6T/pv+o/6X/pP+e/5n/n/+S/57/qP+p/8T/0f/Q/9b/5f/V/9n/3v/Q/9r/3//e/+f/4P/c/+r/4//j//D/6P/k/+r/5f/f/+v/7v/u//v/AQAIAA4ACQALAAsAAwD+//v/8f/1//L/5P/b/8r/yv/A/73/vv+4/7P/rv+v/6//sv+0/8D/wf/G/8n/0f/X/9f/4f/W/9r/xP+6/8H/sv+n/5X/kf+K/4P/f/98/4j/gv93/3b/ff+E/5L/pP+1/8j/zv/U/9L/yf/I/7z/sP+o/6f/o/+d/57/n/+n/63/uP+9/77/r/+o/6H/nv+d/43/h/+B/3v/ef97/3X/bP9j/2P/Zf9o/2z/b/93/3z/g/+F/4n/kf+Z/6P/sv++/8b/1f/Z/+b/9f/9/woACQAMAA4AFQAbABkANQA2AD4ATgA+AFMAaQBjAGkAZABhAGYAYgBsAHEAfACIAJoAswCuAMYAyAC9AMgAuwC+AMwA3ADoAP4A+QD3APcA8wD3AO0A2ADBALwArQCpAKoAogCjAKQApAChAJ4AnwCmAKkArQC1AMMAxgDPAMkAxQC9ALEAogCOAIYAcQBlAFkAVQBDAD4AOAA7AEEARgBbAGYAfwCPAKMArgC6AMcAyADTANsA5gDtAO8A7wDrAOsA8QDxAPAA7wDsAOQA1wDUAMYAtwCxAK4ArgCxALgAtQCxALMAtQCwAK0ArACqAJ0AiwB6AGgAUABAADMAIAAJAPL/3f/L/8n/u/+y/6j/mf+d/4n/jv+R/4r/lf+b/5v/of+j/5z/n/+W/4z/iP9//3b/ff91/27/dP9q/2//dv9y/3j/f/+C/4z/kP+R/4//oP+e/5f/m/+i/43/i/+S/4z/lv9o/43/of+G/57/jP+c/47/j/+D/3P/l/9u/4P/lP+N/5v/fv+X/4z/eP9+/33/aP9n/1b/PP9T/yX/Nf8n/xr/MP8Y/yT/H/8m/xr/I/8f/yP/GP8X/xr/Fv8g/xT/Kv8o/yv/Nf88/0D/Ov9A/0D/Q/87/0j/Qf9F/zz/N/83/yH/Nf8j/yT/Iv8Y/yP/G/8Y/x7/Jf8j/yv/Mf8o/zL/M/8W/zT/J/8k/yP/Fv8n/w7/CP/9/gD/9f7u/t7+0P7j/s7+0f7U/tH+zv7L/s7+xP7P/s7+t/6+/rX+rP6u/pb+n/6b/pf+m/6F/pH+dv5t/mv+Xv5Y/kb+Uf5E/jv+Qf5P/kX+Qv5Z/lr+Wf5I/mf+Xf5c/mH+dP53/mr+hP5n/oP+ev6D/o/+jv6c/o3+nv6r/qv+rf7C/sr+0v7W/uz++P73/gb/D/8V/x3/J/8w/z//RP9Q/1n/Vv9m/2f/X/9g/2L/U/9b/1r/UP9f/2b/Z/90/3v/b/+D/43/ff+O/5H/i/+k/5b/oP+s/53/tf+x/7n/u//A/9f/w//b/+T/1//u/+r//v8QABMAHQAmACkAKAA2ACwAKgA3ADEAMwA+ADcANQBRAEQAQwBgAFUAYABoAG0AdwCBAIsAiQCPAJ4AnQCqAKIApgClAJ8AqgClALAAtADFAMwAvQDKAMwA0ADgANAA4wDjAN4A5QDhAOgA4QDlAOQA7wDvAPEA4wDvAP0A4wDxAOsA3wDdANkAxwDeAN4A0wDvAM0A3QDdAM0A1ADMANMA0QDRANQA1ADNANQA1gDYANMA2gDYANcA3QDSAN8A1gDaANwA3wDdANMA2gDXAN4A0wDSAMsAzQDLALgAwgC4AK0AoQCYAJIAlQB+AHIAfABpAGgATABDAEUANQAoACEAJwApADMALwAwADsANgA4ADgAOwA+AD8AQQBBADoAOAAvACcAHAATAA0ADwAOAAoAFgAQABEAFwAaAB4AKQAxADAANQAuADEAMwA4ADsAOQA+ADQAPwA5ADgAMQAoACEAGAAiABQAEwAHAAkADAD9////+//n/+v/2v/G/9j/vv+2/8H/u/++/7r/vv/A/9D/z//O/97/1//f/93/2v/e/9b/4f/Z/9n/3v/d/+D/4P/0//n/EQAVACMAKgAuAEQAPABLAFYAZABhAF8AZQBgAGYAZwBiAFYATwBMAEYAPAA7ADUAMwA+ADUAPwA+AEgAWABhAHgAcwB6AHoAfQB7AHsAfgB2AHwAbwBsAGkAVwBaAFIASQBOAE8ARgBBAEYAQwBKAFYAaQB9AHMAfACCAIcAjAB/AIsAmwCcAKUArgC8AMoAxADbAOMA5wALAQYBDgESARYBGwEPARYBEAEGAQ4BBAH+APMA7wDnANcA0ADMAL8AtgC+ALsAygDEAMEAwgC7AMUAvADAALIAsACkAKgAqwCoAKoAogCiAJAAnACcAJgAkwCPAIgAhQCGAH0AegBvAGoAcwByAHgAjQCLAKIAmwCsALYAqgC2AKcArQCrAKAAnwCRAIQAhQB0AGwAXABDADYAHAAOAAQA/P/4//v/+P/2//L/5P/f/9r/0v/N/8v/wv/E/8j/x/+9/73/tv+p/6D/lv+P/5H/kP+N/5P/j/+V/6n/tP++/9H/3f/e/+z/7v/w//X/8f/s//P/8f/l/wIA/P/h//b/7P/t//X/5v/v/+//4P/0//X/9v8DAP7/BwADAA8A///w/wAAAwD+//H/4//k/+b/2v/i/9n/4v/p/+b/6v/s/93/y//U/83/vf+b/5T/j/+F/3n/av9w/2f/W/9W/1r/XP9e/1b/XP9i/1r/ZP9h/2b/bv9w/2j/Yv9e/2X/Yv9b/2L/Wv9p/3D/b/9z/3P/ev+G/47/lP+Q/5n/oP+m/67/tP+w/6r/vf+5/8r/yf/G/8D/t/+5/6f/rf+i/6b/qv+m/7T/vv/J/8z/z//Q/8b/yP+8/6z/qv+Q/4L/fP9z/2f/a/9g/1P/TP8x/yb/FP8M/xf/H/8l/yr/Nf82/z//Sv9O/2b/Z/+A/4//kf+S/5T/qv+x/7z/x//V/8X/uP/A/8L/xv/B/8j/tv++/8P/w//R/83/1v/Y/9//6//u/+v/BgACAAoADwAKAAsA+//7/+3/1f/Z/8j/uv/A/6r/p/+d/6T/oP+q/7X/r/+9/8T/2f/p//b/AQAOABcAFgAlABoAGwAyAC4ARwBKAFcAaQBiAHgAcAB6AHwAgQCBAIcAnACcAKcApQC2ALUArQC5ALIAqAClALAAqwCcAKwArgCmAKUApQCuALQAtwC1ALYAuwDCANAA0QDcAOEA6ADyAPgA8wD5APQA8QD9APoA7wDvAO8A7gD9AP0AAwELAQ4BFgESARgBGgEYARwBIQEkAS8BLgEoASQBIQEaASMBHQEZARIBCAEAAfgA9ADkAOIA1wDTANQAxwDKAMcAuQC6ALkAqQCwAKYAnACaAJgAkwCRAIgAgwCAAHkAgQCDAHwAhACNAIcAkACQAJIAowChAJ8ArgC5AKoAuAC1ALoAxQDBAMoAzgDDALoAuAC0AKYApgCkAJ4AqQCqALQAuQDBALkAuwC/ALsAtwCiAKEArACpAJ4AlQCPAJEAhgByAG4AZABXAEcAOQAxACkAFwASAAYA9f/u/+7/4P/o/+j/2v/d/9D/0//L/8f/uf+x/67/ov+Y/4f/ef9t/2v/X/9U/0z/P/80/zX/Mf8x/zH/Lf86/0D/Qv8//0T/Rf9J/0z/S/9N/0n/Rv9K/z//Pf83/zP/NP8p/zP/Jf8m/x//Ff8R/w7/Ef8O/xj/FP8c/yT/GP8a/wz/Bv/4/vf+8/7i/un+0f7K/rv+u/7B/sH+yP7K/tr+2/7c/t/+7v79/gr/F/8j/yf/MP8v/zf/Of89/0v/Uv9e/2L/cf9z/3r/gP+E/5D/kv+X/5z/of+l/6n/sf+r/7L/vv/C/87/zv/Q/9j/3v/m/+z/+v/5//z//v///wgAEQAkACwANwA/AEQAUABYAGAAaQBnAF4AXQBlAHAAggCNAJMAlQCRAJgAngChAKMAnAChAKkAqQCsALQAtwC4AMEAxgDMAM8A0QDHAL8AtwCrAKIAlwCSAIgAfwB2AG8AYwBXAEoARgA+ADkAOQA5AEAARgBQAFUAaAByAH0AhQB+AH0AeQB2AHwAgACEAIMAhACEAHoAcgBhAF4AVwBRAFIASwBOAEoAUABYAFsAYABgAGYAawBqAGsAZQBgAF4AXgBaAFIAVABdAGMAXwBdAF0AXABiAGAAYQBjAGAAWgBYAFcAWQBSAFUAVgBSAFcAUABMAEsARQBDAEoAQABBAEAAMwAzADEAJgAkACcAHgAiABQACwAPAPr/9f/x/+D/3v/h/9f/3//k/97/1f/J/8v/xf+4/8f/xP/D/83/v//P/9L/yv/I/8X/1v/R/8//2f/b/93/1//a/87/yv/Q/8v/3f/Y/93/6P/t/wMABgD5//r/+P/3//D/4P/Y/73/tv+s/63/k/+Q/3n/dP9o/2L/XP9B/0b/HP8h/x3/Kv83/z3/Rv9H/0//T/9M/1T/U/9X/1j/Uf9V/1r/Xf9T/1r/Wf9j/2D/ZP9t/27/gf+H/5z/rv+y/9P/5P/1/wIAAgAYACsAPgBTAFgAWQBkAGcAbABwAG8AbQBwAGwAagBoAF8AVgBPAFQAVQBOAEkAPwA2AC0AJQAbABIAEAAIAAEAAAD8//3//f8KAAoACgAQAA8AEQATAB8AHQAeAB0AGAAiACYALAAwAEYATgBdAGsAewCVAKAAsgC2ALoAuwDCAMMA2wDeANsA2wDVANgA1ADVAMsAwgDGAMIAvQDEAMAAvQC7ALMArgCxAKoApQCuAKUAqwCvALAAtQC5ALkAtwCyALAArwCoALAAtAC/AMEAxQDNAMYAxADHAL4AugCwAKUAnACPAIgAdwBvAF8ATQBEADAAHgAQAPj/7v/n/9z/1//J/8X/vv+7/7X/rP+f/4v/g/97/3n/eP9v/2X/Wf9V/1j/Xf9Z/1z/YP9e/2P/Xv9f/1v/V/9e/2L/Z/9i/1n/Wf9L/z//Lv8h/x7/FP8Q/w7/C/8H//7++f75/vz++f7//gH/+/4A//z+//4G/wr/GP8e/yf/K/8q/yv/HP8d/xj/Fv8U/wv/Df8K/wj/CP8R/xX/Gf8i/yP/LP8x/zb/N/82/zb/Nf84/zv/Rf9M/0n/Rf9I/1H/R/9E/zX/Kv8q/yv/JP8q/zD/NP9P/1D/Z/9m/2z/dP97/4n/ff+Q/4n/lP+s/67/qP+t/77/wv/E/8D/y//R/9L/0//c/+b/4//b/+f/7P/y//j//v8LAAoAEAASABQAEgAaABUAHwAjACAAJwAjAB8AGwAcABgAFAAMABAACwD+/wQA/P/6/wIA/v/x/+3/3//O/9z/z//N/9T/wv/F/7//t/+5/6b/nv+T/37/f/9z/2T/fP97/3H/gv94/4P/hv+W/57/nv+t/6b/v//A/8n/yf/F/8f/y//U/8b/0P+//7f/uf+y/7H/qf+b/4n/hf9x/3z/bv9i/2r/Zv9g/0v/RP86/yP/E/8a/xL/A/8I/wz/Cf8Y/x//K/84/z7/U/9f/2r/bf92/3j/fP+F/4P/hf+B/4r/jf+K/5D/kP+U/6D/rf+t/7//xf/J/+D/3f/p//P/8P8DABQAEAAWAAkABwAQAPX/+f/x/+b/5v/v/+//8//t/9n/9f/l/+n/8P/i/+n/5v/k//H/9//z/wEADQAdACMALgAzADYARwBDAEgAUwBOAEgAUABCAEEAOgA0ADkANgAyADsAPwAsAEAANgBDAEcASwBjAFwAZABjAGsAYwBpAGUAZgBzAGoAdgB4AHIAdwB5AHgAgQB6AIUAigCFAI8AmwCfALAArwCvALoAuQDIAM0A0QDUANIA1gDfANsA3gDeANoA5QDlAPEA+gD/ABEBIgErAToBRQFNAVABWQFnAWQBawFwAWwBeQF7AXIBdwFsAV4BYQFGAVMBSgE6AUQBOAE/ATwBNgEwAS0BHQEUAQ4B+QD2AOcA5ADcANYA1gDfAM8A1QDVAM4A1gDFAMsAwwC3AK0AogCbAJUAnACeAJ4ApQCrALAArQCrAKoArgCuALUAvAC9ALkAtAC2ALgAtACrAKoAoACUAIgAcABlAFoAUABQAEsARQA9ADYAMwAqACQAHAASAA8ACQAIAAMAAAD4/+7/6P/g/93/1f/a/9j/3//f/9z/4f/g/+D/4f/j/+X/5//o/+X/3P/T/8H/s/+s/6L/ov+h/5r/jP+A/3r/df9v/27/bf9s/2n/ZP9q/2n/av9y/2z/bP9h/1f/Wv9H/zf/Kf8Z/w3/+v7v/un+1/7O/s7+1f7a/t3+4f7l/ur+6P7r/uf+4/7d/uH+4v7b/uP+5P7g/t7+4f7p/vT++/4H/xL/Ef8S/xr/KP80/0f/UP9e/2v/eP+H/5D/l/+Z/5z/o/+h/6T/qf+p/6r/pv+i/6D/nP+W/47/h/9+/3P/dP9u/2f/Zf9s/3T/ff9//4X/g/9//4H/e/98/3P/cv9t/2z/a/9k/2T/Yv9d/1//Y/9m/2v/a/9u/3P/dv95/3v/gv+J/5D/mv+j/6T/qf+p/6f/p/+m/6T/oP+g/5j/m/+c/5b/l/+S/5X/j/+H/4X/ff9w/2v/Zv9U/1H/Qv80/y3/Iv8d/xP/Df8N/xH/Ef8R/xT/F/8V/xv/Gf8Z/xv/HP8g/yL/KP8r/zX/PP9G/1L/V/9g/1r/X/9e/2L/Y/9h/2X/Zv9x/3L/ev96/3f/f/9//4L/hf+F/4P/gf+A/4P/h/+I/4f/iP+I/4T/gf9//3//gv9+/33/g/+H/4v/kf+S/5f/nP+b/6L/p/+v/7j/wf/M/9H/2v/m//L/9//8/wgACgARABIAEwATACkARgA8AEsAWQBdAHQAeQCGAJoArACtAL4AwgC/AO4A4wDrAAYB/QAXARIBJwEdARoBGQEaARMBBwEbARcBLQEwAT4BOAE7AUUBRgFEAUMBTAE9AT0BPQE1ASkBHgEcARkBCgESAQoB+QDtAO4A3QDWAMgAsgCqAJoAlwCCAHcAZQBTAEwASQBDAD4APQAyADoANwAiACcAGAAOABEACQD2//n/8//Q/9T/x/+4/7b/qP+h/6L/nP+Y/5X/lv+Y/5v/jP+b/4j/l/+O/43/tv+Z/8D/0f/a/+b/6f/z//T/+//1//b//f/3//r/9P/l/+//9f/5//f/+//7//3/+//+/wIABgAWABgAIgAqACoAMQA2ADkARQBHAE4ASgBKAEYAOQA6AD4ATwBjAHkAggCbAK0AtADKANoA5ADoAOoA6ADyAAQBCwEHAQcBFAEKAQsBCwEKARABFAEKAQYBCQH+AAEBAQENAQQBAgEIAQkBDgH/AAYBCQEIAQwBEwERARUBDQEQAREBCgEJAQIBAgECAQcBAgH/AO0A4wDWAM0AuQClAJ0AhwCBAGoAaABZAEQAQgAoACYAGAARAA0A/f/u/+P/2v/T/9H/zf/H/7r/s/+0/6f/p/+c/3r/ef93/2H/Wv9P/zr/KP8x/yb/K/80/zT/R/8//0T/S/9U/1j/YP9Z/13/V/9g/1//WP9j/0r/Tf9I/zr/K/8p/xj/Fv8U/wn/C/8L/w3/F/8V/xP/F/8g/yL/D/8Z/xv/B/8W/xb/B/8Q/wv/Fv8Z/yL/K/9C/z7/Rv9E/0f/UP8//1r/Rv9S/1z/ZP9v/2f/cv9r/3D/dv90/3P/eP99/3//ev+D/4v/j/+S/5P/n/+b/47/kP+J/43/lf+N/5z/oP+h/6z/rv+4/77/vf/F/8f/yv/Q/9f/2//d/+D/1v/g/97/0//W/9D/z//P/8v/x//G/8T/y//N/9L/2f/c/+P/5P/m//L/9v/3//3/AQAAAAkABAAAAAYACAARABAAFQAOABUAEgANAAwAAAADAP//BgADAP///P/u/+3/6f/k/+L/2f/S/8z/xP/A/7H/pf+g/4z/jf9+/2v/af9Y/0r/Uf9P/0//Wv9j/3D/b/95/3f/f/+C/4X/if+H/4X/ff+B/3z/fv+A/3L/cf90/2//c/9x/3D/a/9x/3P/bP9n/2X/Z/9m/2n/bP9v/2n/b/9z/27/a/9l/2T/W/9k/2r/c/9+/4H/jv+W/5//qv+w/7L/sv+0/67/o/+e/5L/jv+J/4P/gv+C/4f/i/+T/4//jf+Q/5D/j/+R/47/kP+V/5P/mP+c/5D/lf+R/4f/if9//3r/eP9y/2r/av9k/2P/av9g/2n/cv90/3r/ff+E/4j/if+K/5T/n/+h/7T/vP+9/8b/x//I/8n/yf/C/8T/u/+2/67/oP+d/5X/lf+U/5r/nP+e/57/of+l/6P/p/+m/6f/rP+u/7D/tP+z/7z/v//H/9H/0//c/97/4P/c/93/3P/j/+z/6//3//j///8EAAMADwAFABEAGQAYACAAHQAnAC0ANwA7AEUAUABZAFoAXwBrAGsAewCDAIIAhgCLAI0AjgCZAJkAmgCeAKIAqQCyALcAyQDRANoA7QD7AP4ABQEGAQ4BDgERARMBEQEWARcBEQEZARwBGQEhASkBKwEsATEBLgE2ATkBNQEzAS4BPAFAAUgBRwFJAUcBQQFNAVIBRwFSAVEBUgFWAVoBWgFhAWUBcgF7AYwBoQGkAa4BogGnAagBoQGkAZ8BkgGGAYsBewFuAV8BSwFIATkBLgEhARsBDQH4APUA8QDmAOEA3ADLAMgAxgDGALsAuACzALEArAChAJkAiQB+AG4AZQBBADgALgAfABoADQD9//X/9P/v/+T/3v/T/8v/yf+9/7L/rP+e/4r/fv91/2r/Zf9f/1f/V/9Z/0//Vv9a/0r/Sf9J/0D/SP9G/0z/UP9M/0//W/9j/2n/cP9z/2T/aP91/3v/hv+E/4f/if+Z/6L/n/+l/53/kP+e/6P/mv+X/5L/jP+U/5X/pf+t/7H/sf/C/9L/1P/i//X///8SABsAGAApADUAQgBUAFkAWwBmAGgAZwBsAHYAhACTAKAArACuAK0AsAC6AMkAzwDfAOcA5ADjAPMABAESARkBGgEbAQ4BAwEMAe8A7QDeAN8A7wDUAOYA1wDSAM4AwQDDAL4AuQCtAKQAoACaAJgAkQCCAH0AeQB1AHsAdwBvAGAAUgBKAEMANwAxACwALgAnACIAJwAvAC4ALwAqACkAKgAlAB4AFAAGAPj/6//c/9D/v/+r/57/kP+C/3X/ZP9b/0//TP9I/0P/Rv9E/0H/QP82/y//Lf8m/yL/Gf8P/w7/EP8M/wr/Cv8H/wP/+v74/vT+8f7v/u/+7P7p/uz+7v7u/vT+9v76/v/+Af8D/wL//f78/gH//f4E/wP/CP8K/w//D/8S/xr/Hf8n/yX/Kf8x/yz/Mv8p/zD/N/83/0L/Ov9J/0r/U/9c/2P/a/9t/37/ef+C/4P/hP+J/43/kP+W/4X/ef91/3P/dv9r/2v/af9m/2T/ZP9k/2f/Yv9g/2r/Yv9h/1z/W/9b/1j/Xf9Z/1v/WP9b/1z/Yv9t/3b/gv+N/53/pv+w/77/xP/N/9X/2f/f/+f/8P/1//j/+//9/wAAAAAGAA0AEQAUABoAFAAZABQACQAGAAAAAgD9/////v/6//z//f8CAAUABAAEABIAEAAJAAwACAALAAkACQAJAA4AEAACAAoACQAGABAADgAHAAkABwAGAAoABwAKAA8ABAAQAAEACwAMAAwAMQAhADYASgBWAGUAZwBtAHAAdQB4AHsAggCEAIMAhQB+AHwAgQCFAIQAhQCDAH8AdwB0AHIAcwB8AIIAiACPAJIAmACgAKcAsQC2ALsAtwCxAKwAnwCbAJcAmQCkALYAxQDZAOkA8wD+AAcBEAETAQ8BDgEQARcBHQEdARwBJAElASQBJwEmASQBKQEhARsBGgEJAQYBCAELAQYBAAEDAQMBBwH/AAMBBwEFAQcBEAETARQBEgEOAQsBCAEIAQoBDQEMARABEQERAQgB/ADvAOUA0gDDAL0ArACjAJUAiQCEAHMAcABhAF4AWgBLAFAAQAAyACYAFgALAPn/9P/q/+H/zf/C/7T/qv+k/4n/dv9p/1T/Pv8w/xr/A//6/u7+6f7t/ur+9f7x/vH+8v74/vj+/P72/vH+6/7q/ur+4v7l/tj+1P7Q/sn+xv7K/sX+yP7Q/tT+2f7g/ub+7P7y/vb+//4K/xD/D/8T/xf/Ev8c/yj/K/8z/zT/O/9A/0f/Tf9e/2f/b/91/3v/h/+H/5n/nv+n/7T/v//O/8v/z//L/8f/zf/O/87/1P/X/9r/2f/m//T/+/8GAAAADwATAAYAEAAIAAsAFwAOACYAKQAnADoALAA5AD0AMAA4ACkALAAtACsALQAoACcAFAAmACMACgAIAPz/8v/+//n/9f////b/+v/2//f//f/6//j/7P/g/97/1//M/8b/yP+8/8L/uP+t/63/pf+u/63/sf+e/6T/mf+I/4H/av9x/3L/iP+S/47/if9x/2z/aP9l/2v/af9a/0j/Ov81/yv/J/8w/yD/Kv8g/w//Fv8N//3+Cv8L/wj/F/8k/zH/Mf84/y3/Nv85/z3/Qf85/zn/Lf86/zr/Pf9F/yv/J/8u/y//QP9O/1P/UP9a/13/Vf9P/0//VP9W/1//bP90/27/d/9+/37/f/98/3z/b/95/4X/lP+i/6b/s/+8/8v/2v/p//X//f8HAAkAAgACAPv/+P/w/+3/8f/0/wQAFgAnACUAIgAnACoAMQA4ADwARQBLAEsAXwBpAF0AbwBvAHIAfQB4AH4AfAB3AHEAdgB3AIwAkQB+AI0AjgCMAJcAkwCcAKQAngCgAK4AsgC2ANUA2QDUANgAxwDFAL0AuQC5AL0AtwC1AKcAlQCKAHYAcQBuAHUAdgBuAGEAXgBRAEAAPAAxACcAIQAhAB4AFgAIAAQA9v/o/+f/3f/j/9r/y/+0/6n/nP+e/6v/pP+z/6z/rf+q/6L/qf+K/5X/kP+J/4//gv+E/4L/gP98/4f/jv+R/4P/gv+L/37/kP+O/4P/if+M/4j/jP+c/5z/lf+V/5b/mv+Y/5b/pP+q/7T/xf/U/8//0v/T/9f/2//i/+r/6//u/+r/4//v//P/9v8FAAcADgAYACMAKQBEAEoASQBMAEsASwBUAF8AUABUAEkAPwBUAE4ASwBdAGAAYwBhAGcAYwBjAGgAeACFAJoArAC7AM4AxwDdAOYA5gD2APkA9AD8AA8BBwEIAQEB+wD7APIA/gAEAQcBBgH6AP8A/QD8AAEB+gDnAO0A9wD1AOkA5gDeAN4A2QDUANEAwwC5AKwAngB9AH4AdgBlAGEATwBCADwANwAwACoAJgAgACAAHQAYABYAFQAOAPz/8v/s/+b/5v/k/+H/7P/4//3/EgAdABkAIwAtADIAPwBCAE4AVwBbAGIAaQB0AIAAigCQAIEAhwCRAKAAtgC0ALwAvgDXAOoA7gD6APIA2wDpAP0A+QD9APMA4QDkAOMA8ADuAO4A4gDkAOwA4ADmAO0A7gD5APUA6wDyAPkA/gD/APQA9QD2AOkA3ADTANsA3QDjAO8A7QDgANEA0gDbAM8AuwDAALgApgCWAJwAsQCqAKEAnwCaAI0AggB/AFoASgAyADwAQQAqADgAGQATAAMA7v/t/+H/3//E/7X/tP+v/6r/l/+D/3//ef93/4T/f/93/23/Yf9a/0z/Q/88/zb/N/8t/yb/Lf8w/y//Nv81/z7/Rf9F/0X/Q/88/zf/M/8o/yH/FP8N/wj//f72/uz+4v7d/tX+1v7U/tb+3P7a/tn+2P7N/sX+w/65/rH+pf6e/pz+oP6g/qP+qP6s/qn+pv6o/qn+rf60/rz+wf7I/tT+4f7s/vr+BP8Q/yD/Kf8u/zD/LP8s/zb/Nv89/0D/Sv9N/1L/U/9W/2H/Z/93/3f/fP9//3j/ev9u/3j/gP+G/47/h/+S/47/kv+U/5r/nP+g/63/pP+n/6P/o/+i/6b/qP+p/5P/g/96/3n/ev9t/2f/Yf9c/1j/V/9Z/1r/VP9T/1P/TP9I/0T/Q/8//0H/QP88/zr/Mv8y/zH/NP87/0T/Sv9U/2H/aP9z/3z/gv+J/4z/kf+V/5r/nv+n/6z/tv+9/7//xf/N/9//6f/2//////8HAAUAAgAHAAkAEwAfAC8AQQBIAFEAWQBnAHIAdAB+AI4AkQCWAJkAlwCeAKIAsAC7AMgA0wDRANkA3gDkAPEA8wDrAOwA6QDoAPEA8wD1APQA5wD3AOwA9AD5APsAFwEQASEBNgE8AUcBSQFRAVUBXgFbAV4BYAFhAWUBaAFkAV4BYQFfAVcBWQFYAVQBUwFQAUoBTwFOAU8BVwFYAVcBWQFbAV0BXQFcAVoBUwFKAT0BLgEoASABHgEfAS8BQQFSAVoBZAFjAWMBYgFXAUkBQAFBAUEBRgFCAToBNQEsASABHgEYAQ8BDwEIAQAB/QDpAOIA4gDgANcAzgDNAMsAywDFAMIAwgC5ALQAuAC3ALgAuQC1ALEAqwCtAK8AswCzALQAtgC1ALIAqQCiAJ0AkACGAIEAeQB0AHUAbwByAGwAagBnAGUAbgBjAG0AaQBgAGUAYQBiAFgAWQBhAGYAZgBiAFsAYABjAF4AWQBQAEsAPwA5AC4AIwAfABwAIQAmACoAOwBGAFEAXQBtAHwAigCPAJQAlACRAJAAjwCVAJgAkgCPAI8AigCSAJMAkwCYAJgAoQCqALAAtADAAMgA0gDfAOoA8gD1APIA7gDtAPIA8gD5APYA9wD1APUA8ADuAPIA+AACAQIBEAEOARQBGwEeASEBHAEeARMBDAEIAf0A+wD4APIA7QDjANsA0wC9AKwApACZAIsAhAB6AGwAXABLADwANgAoACYAJAAgACcAIwAbABQADgD+//b/7v/i/9//3P/Y/9X/0P/M/8v/x//E/8L/vP+1/7H/rP+n/6f/p/+n/6n/p/+n/6b/o/+k/6H/mv+S/4v/g/+B/3r/b/9j/1j/VP9M/0j/Rf8+/zv/Of84/zX/MP8v/y3/Kv8n/yL/Hv8Z/xX/Ef8K/wL/9f7x/uj+2v7R/sP+v/66/rT+tP61/rP+sf61/rv+wf7K/tX+3/7m/ur+8f76/vz++v78/vv++/76/v7+Af8F/wr/FP8h/yb/MP83/zz/Pf9B/0X/Rf9F/0D/Qv9B/0T/Sf9K/0f/R/9J/0z/Tv9O/0//Uf9Q/1T/Wf9d/2H/Zv9p/2//dP94/3z/fv9+/4D/gP9+/3r/dP9x/2//bv9u/3D/eP91/3T/dP9w/3H/df9z/3H/b/9r/2f/Zf9m/2X/a/9x/3X/ff96/4D/hP+D/4L/gf+C/4f/jv+Q/5P/j/+M/47/jv+P/4v/hv+A/3//ev96/37/f/+F/4v/kP+T/5T/mv+h/6n/sf+x/7b/tf+4/7b/sv+r/6X/n/+Z/5L/iv+E/37/fv97/3z/ev96/3v/ff+E/4X/jP+M/5D/j/+N/4v/hP+B/3v/ef92/3L/cP9r/2r/a/9q/2v/a/9s/2z/bP9s/2//cv91/33/hP+M/5P/nP+m/7P/u//F/87/2v/n/+//9//+/wUACgARAB0AKAAtADcAQwBRAGYAdACCAJAAmQCnAK0AvgDNANcA5gDwAPgAAwEJAQ0BFwEWARcBGgEZARkBHQEYARYBFgEQARQBFwESAREBDQEJAQoBGAEbARoBIQEiASUBJwEmASABHQEiASABIAEXASUBLgEpATEBNwE9ATwBPwE/AToBQwE8AUQBSgFGAUcBPgE/ATUBJwEaAQ8B/gD3AOwA3gDWAMUAxQC3AK8ArwCsALIAtQC2ALoAvAC6ALgAtACwAKgApACiAJ8AngCdAJkAnQCZAJUAlACUAI4AjACKAIsAiQCGAIgAhACDAHwAgQB/AH4AfwB+AIIAggCEAIcAiACFAI0AkgCXAJ4AnwCiAK4AswC8AMUAygDRAMwAzQDNAMsAygDPANEA1QDgAOkA8wD4APwABAELARIBGQEhASABHgEdARsBGwEhAR0BHwEfAR8BHwEeASABHAEcARsBFwEVARIBFgEZAR8BJAEoASsBMAE0ATUBOgE2AToBNAEuASwBJwElARwBGgEQAQsBAQH7APcA7gDoAOAA3ADZANkA1wDVANMAzQDJAMcAwwC+ALoAsgCoAJ4AkACDAHcAawBfAFIARQA9ADAAIQARAAIA7//f/9T/x/+//7z/uP+2/7H/q/+q/6X/m/+R/4X/ev9v/2P/Wf9O/0T/Pv8z/y7/Kf8k/yf/Iv8k/yL/Iv8d/xr/F/8S/wz/Bf///vf+6/7j/tb+y/7B/rX+qf6e/pX+if6G/n7+ff59/n7+ff5+/oP+hP6K/ov+iv6I/on+hv6D/nv+df5v/mL+XP5R/lL+Uf5T/lj+Vv5X/lf+Vf5Y/lj+XP5b/lv+W/5d/l3+W/5j/mT+ZP5l/mX+ZP5q/m7+bP5v/mb+ZP5d/mD+X/5s/m/+dv6D/oT+kf6N/pD+jf6L/o/+kf6X/pX+l/6X/pr+m/6f/qD+qP6q/rD+tf65/s7+3v7u/v3+EP8d/yv/Of8//03/Uf9W/1v/YP9n/3D/ev+A/4T/iv+P/5b/pv+o/7j/wv/E/83/yv/S/9L/0v/R/9P/2//b/+P/4P/h/+X/4v/i/9z/2P/Q/83/y//J/8b/wv+//7r/t/+x/7H/s/+2/7v/vf++/77/vv+8/7z/wP++/7//vv+7/77/wP/F/8j/zf/R/9T/3f/d/9//3//d/+D/4//n/+D/3//Z/9r/2f/O/8r/wP+v/6H/j/97/3L/X/9X/1H/Sf9M/0P/TP9O/1L/Wv9g/2v/bP94/3z/fP+E/4P/iP+K/4//k/+V/5v/of+x/7f/xv/M/9n/4f/n//H/7//5//3/BwAKAA4AFgAYABwAHwAeABYAFAAQAA0ABwACAP3/+v////r//v/4//r//////wYA/v/7//b/8v/y/+//7P/j/+H/3P/Y/9f/0f/Q/8z/zP/L/8r/x//C/8X/xP/M/9L/2v/l/+P/6v/p/+v/7//s//P/9//6/wEABwAKAA8ADgATABcAGwAmACUAJAAjABwAFwAMAAgA/v/x/+7/4f/X/8n/xP+8/7X/s/+t/6z/qf+y/7f/vv+//8P/yv/P/9r/3f/k/9//5P/o//P//P8CAAYACAAOABEAIQAkACwAMwA+AEYAUABdAF8AaQBwAHYAhQCNAJQApgCpALwAwQDRAN4A3gDqAOcA7ADzAPIA+AD3APUA+gD6APsA/AD9AAEBBgEJAQ4BFwEcASkBKAEwATcBNAE3ATsBOgE9AUIBRQFOAVMBUwFSAVEBUgFLAUkBRgFAAUIBQAE6ATsBNQE1ATgBOAE3ATgBNgE1ATIBKgEjARsBDwEAAfcA7wDpAOYA2ADNAMoAvQC9ALIApgCiAJoAkQCRAI0AigCGAH8AfwB2AHMAZgBdAFYASwBAAC0AHgATAAsA/f/2/+//7P/q/+j/4//g/9v/2//e/9n/1P/J/8T/vf+y/6r/o/+c/5T/j/+K/4z/if+K/4v/j/+V/5f/of+k/6z/s/+3/7j/tf+3/7j/uP+3/7j/t/+4/7n/t/+4/7b/uP+8/8D/wf+7/7n/uv+7/73/wP/C/8D/yf/L/9T/2P/d/+L/4v/s/+n/8f/y//X/+v/6/wEACQARABYAHQAcABwAIQAbABYAEgAKAAQAAgACAAEABAACAAAA///+/////////wcADAALAA4ADwANAA4ACwANABIAEQAaAB0AKAApACgAMQA0AD4APgBDAEAAPgBBAEEAQAA8ADsAMAAqACYAIwAiABsAGAAUAA0ACwALAAkADQAOAA4AEAAPAA8ACwAIAP//9//t/+H/1f/G/7X/pP+Z/4z/f/94/3H/af9k/2L/Yf9g/13/Wv9X/1P/TP9J/0P/Pf88/zv/Pf86/zv/Ov80/zH/Kf8e/xP/Cv8A//X+7/7m/tz+2v7T/sv+xf7B/rv+t/6z/rX+sv6z/rb+tv63/rP+tf62/rn+vP6//sT+w/7H/sf+zf7R/tr+6P7v/vf++P4B/wb/Dv8W/x3/Jf8o/y//Mf85/z//RP9M/1D/V/9b/2L/Y/9l/2r/bP91/3n/gP+E/4j/jf+R/57/p/+v/7b/wP/D/8r/0//V/+P/6P/t//b/+v8HABEAFQAeACIAKAAyADQAPgBBAEsAVQBhAGwAegCFAJIApACqALEAvwDLAM4A1gDUANQA5wDqAO4A+gD+APsACAEFAQ4BEgEWAR0BGgEWARUBGgEXARYBIQEnAS0BPwFGAUsBVwFWAVIBVgFMAUYBQAEtASoBIgEVAQgB/ADyAOoA3ADLAMQAuACvAKcAnQCUAI8AhQCHAIAAdgBxAGsAYQBfAFgAUQBNAEQAQQAzACwAIAAXABIADAAHAPz/+P/w//L/6P/i/9z/zf/H/8H/uv+2/63/qP+s/6r/qP+i/6P/p/+l/6r/qf+l/6n/o/+j/6n/n/+f/5j/mv+U/43/kv+K/4f/fv94/3j/b/9x/3j/ef96/3//gP+D/4H/gf9+/33/ev90/3b/bv9r/2X/bv9x/3j/gP+J/5b/nf+q/7b/w//W/+L/8P/6/wIADAARAB0AIgAmAC8AOABDAE8AWABiAHEAfQCJAJMAnQCpALIAugDFAMsAzwDMANUA3ADjAO4A9gD2AAMBCwESAR8BJQEiASEBFAEbARgBGgEnASMBMgEyATQBPgE9AUIBPwE/AT4BPQE9AUEBRwFLAVEBVAFTAVgBXQFgAWABYQFhAWEBXgFcAVsBXAFbAWABZwFrAXMBfQGDAY4BkwGSAZMBjwGGAX4BcAFlAVkBTAE/ATcBLgEiARkBEwEKAQIB+gD1APEA7wDwAO8A8gDtAOgA4QDXAMwAwgC0AKUAlACFAHIAYgBPAD4AMQAiABYACQABAP3/+f/1//T/8//x//D/6//j/9r/zv/G/7v/rv+j/5P/hv97/3L/av9h/1n/UP9I/z3/NP8p/yP/Gv8R/wX/AP/3/uz+5v7Z/tL+xv6+/rf+rv6m/qH+mv6V/o7+h/6A/nv+dv51/nX+cf50/nL+d/5+/m7+cv5z/nP+cP5x/nP+df55/nv+gP5//nv+dv50/m3+ZP5j/mD+ZP5j/mP+bP5p/nH+eP58/oL+i/6W/p3+oP6r/rb+vP7M/tP+4P7m/vX+Av8M/xn/Hf8k/y3/NP8+/0j/Tv9a/1//cP97/47/kv+h/6v/rv+4/8H/zf/R/9r/3P/h/+7/+v8KABkAIQAuAEEASwBVAGoAewCOAJsAqAC3AMgA1QDdAPUABQERAR4BJgEoATgBQQFNAV4BYgFqAXEBcAGEAYABlAGiAa4BzAHCAdgB5wHvAQQCCwIXAiMCMQI2AkICTQJTAlwCYAJcAmICZwJsAmgCbQJqAmMCXQJRAkcCRAI/AjwCOwI0AisCJQIfAh4CFwIQAg4CAgL6AfAB4gHcAdMBzQHGAcUBwQG/Ab0BvAG7AbwBtwG0AaoBnQGUAY0BgwF1AWcBWAFGATUBKQEXAQwB/wDzAOcA3gDSAMkAxAC/AK8AogCYAJEAiAB9AHYAbgBkAFwAWABVAE8ASQBGAEEAOwA3ADQAMgAuADAAMgA3ADcAMwAuACgAGAAQAAQA+P/r/9//0f/M/77/uv+4/7D/tP+q/67/pf+h/6H/oP+i/5v/n/+f/6X/oP+f/5n/lf+U/4z/h/+D/33/ef95/3f/c/92/33/hP+Q/5v/qf+z/7r/wf/N/9j/4//r//H/+f8BAAkAEAAVABkAGQAdACIAIwAnACkALgAyADcAPQBCAEYASgBUAFsAZABpAGoAaQBqAGsAbgByAHUAdAB1AHUAcwB1AHYAcwB0AG8AbABrAGQAZwBhAF8AWwBUAFIAUQBSAE0ASABBAD4AOwA7ADcANQAzAC4ALgAzADwAOgA7ADsAPgA8ADsAOAAvAC0AKgAqADIAMQArADQALgAzADEALAAyACUAKQAmACIAHAAWABIACgAGAPf/7//o/97/2f/T/8//y//J/8j/yv/L/83/0P/W/9P/0P/O/83/zP/N/9D/1P/W/9f/2P/Z/9v/2//Z/9j/2//g/+P/4v/g/+D/2//b/9n/1v/Q/83/yf/B/8D/t/+w/6j/nf+W/4//iP98/2z/X/9O/z3/Mv8h/xT/DP8G//7+//7+/gL/Bv8P/xH/Dv8W/xT/Gv8Y/xv/I/8i/yf/Kv8x/zT/P/9E/0b/S/9P/1X/Wf9e/2L/Zv9t/3X/ef+D/4v/lf+g/6b/rv+z/7T/uf+9/7v/uf+4/7n/t/+7/8L/x//P/9b/4f/q//P/+v8AAAMABQAJAAgABwAGAAQABAACAAEAAwADAAQABgAIAAgACgARABUAGAAbAB8AIgAmACcAKwAsACYAKgAmAB8AHgAXABMADgAKAAgABQACAAEAAQD5//z/+//6//n/9v/3//j/9v/5/wMACgANABkAJAAoAC8ANQA7AD4AQgBBAEUAQgA/ADoANAAwACkAJwAmACcAJgAgAB0AGwAWABMAEAAMAAcAAQD9//X/7f/j/93/1P/M/8b/vf+2/6n/nP+M/37/dP9r/2f/Xf9Z/1H/TP9G/0D/Pv80/zP/LP8n/yX/H/8f/xz/Gv8Z/xr/Gv8b/xz/Iv8q/yv/NP85/z3/Qf9G/0r/T/9S/1X/WP9e/2P/Z/9t/3P/ef99/4P/iP+N/43/j/+O/4v/iv+H/4f/g/+C/4D/e/92/3T/cP9s/2j/Y/9g/2H/W/9c/1//XP9e/2D/Uv9P/07/Sf9E/0T/PP85/zj/K/8m/yT/Jv8j/yL/HP8f/x3/HP8k/yv/Nv83/0L/Qv9J/1H/Vf9i/2P/X/9f/2b/Zf9k/2P/Y/9l/2P/av9x/3f/ff96/4b/jP+R/5r/ov+k/6z/uf/L/9n/5v/w////DQAUACAAJQArAC8ANgA2AD4ASABPAFgAXwBnAHIAgACKAJIAmwCnAK4AtgC8AMUAzwDbAOEA6QDvAP0ABwEQARUBIAErATUBPgFKAVYBWQFiAWwBdgGCAZABnwGsAbEBtQG+AckBzAHXAdwB3gHoAfUBAwILAhYCGQIiAi4CLwItAiwCIQIeAiECIgIdAh4CHQIhAiACJwIrAiwCKQInAiUCIwIcAh0CGQIYAhQCCwIDAv4B9AHvAeIB1AHJAb8BrgGfAZIBiAGEAYABeQFrAV0BUAFCAT0BNQEvAS0BJwEdARkBFwESARUBDQEDAfgA5QDgAMwAwwC2AKkAsACcAJgAkACFAHwAawBkAF4AVgBLAEIAPwA8ADoANgAwACgAJAAgACEAHgAUAAYA/f/1/+3/4//c/9f/1f/Q/8//1P/Y/9n/3P/d/+D/4f/g/97/2f/W/9b/1P/O/8z/xv+//73/uf+4/7X/sP+u/6v/qf+r/67/sf+z/7f/uf+3/7f/uf+5/7r/uv+6/73/wf/C/8L/xP/F/8X/xv/M/9P/3P/k/+z/8//8/wUADgAWABsAIQAnAC4ANAA6ADwAPABFAE4AVQBfAGUAawBvAG0AawBrAGoAaQBnAGQAXgBcAFYAVABRAE8AUgBNAE0ARwBHAEQARQBEAD0APAA4AD0ANgA0ADIALAAmACIAHwAaAA0ABQD///7//f/0//H/8P/r/+7/8//u/+//7v/m//D/6f/n/+X/4P/k/9j/1//K/8L/uf+z/6z/o/+o/6f/q/+o/6v/p/+o/6r/pv+n/6X/o/+g/6H/oP+a/5r/lv+V/5f/kP+P/4b/gv95/3f/Z/9g/1P/P/80/y3/Kf8e/xn/D/8M/wr/A/8F/wb/BP8H/xL/CP/7/gL///4B/wT/Bv8L/w7/A//w/gD///78/v7++v7z/vX+9/77/gH//f77/v/+9P7+/vP+Av/8/v7+CP/1/hH/E/8g/yz/NP88/0D/Q/8//zf/P/9A/0H/Pv88/0T/TP9S/1H/UP9O/0r/Sv9O/07/S/9S/1L/V/9X/1n/YP9q/2//dv91/3j/df9y/2z/Y/9h/2X/aP9x/37/iP+W/6D/pP+w/73/wv/G/8b/w//I/87/0f/P/9D/1v/U/9P/2f/a/93/4//d/9r/2v/U/9r/4//s/+f/5f/s/+3/9P/w//L/9v/z//b/AAADAAcADQATABcAHAAgACQAKQAuADsAPwBHAEYASgBIAEwARQBAAEEAMAAwACAAGQAWAAUACAD6////AwD6/wgAAAD///3/+f/7//T/+v/4/+//6//q/+X/5//l/87/xv/C/7j/s/+n/5j/if+Q/47/kv+a/5j/ov+j/63/s/+//8T/yv/K/8v/yv/R/9P/1v/h/9n/2//g/97/3v/h/97/5P/t//D/+P8DAA0AGwAlAC4AOQBEAE0ATgBQAFIAUwBgAGUAYABqAG0AeACDAIwAjgCVAI8AlQCOAIcAjAB+AIwAiACKAJEAmACjAKEAqgCqAK4AsQC1ALUAuAC4ALUArQC6AL0AtwC6ALUAtQCwAJ8AngCZAJYAmgCSAKQAqQCnALQAtAC9AMEAwADLAMcAxwDNAM8A0ADMAMgAvQDHAMQAuQC7ALkAtwC9ALwAvgDEAMQAygDQANUA3wDhAOcA5gDiAOQA5gDeANkA1wDOAM8AxwC9AMAAugDBAL4AwAC6ALsAuAC1ALcArQCvALQAvwDAAMIAwwC8AL8AvQC+AMAAxAC9ALQAsQCtAKUAnACWAIAAgAB1AGcAawBjAGIAbgBuAHIAgACEAJIAkwCaAJwAngCkAKQApACcAJMAiQCDAHgAdwBwAFcATwBJAEMASQBKAEcARgBPAFUAVQBYAF0AYABdAGQAawBkAF4AXwBbAFEAQwA5ACwAHQAeACEAKAAuACsALwAzADYAPAA8ADwANgAxACgAGAAMAPv/7v/f/87/xf+//7z/uP+z/6X/oP+i/6P/qP+m/6X/qf+q/6v/s/+w/6X/p/+b/5z/m/+P/4v/hv9+/3r/eP9z/3f/bP9s/3f/df95/3b/df9//37/ff9//4f/i/+V/6r/qf+p/7H/rv+u/67/rv+n/6D/lf+N/33/bv9h/1v/V/9X/13/WP9Z/1j/Wv9a/1j/W/9T/1P/U/9Q/0z/Sf9K/0z/S/9O/1T/Vf9b/1j/WP9T/1r/Xf9n/27/bP93/3b/gf+B/4L/hv9z/3z/cP9w/3T/a/9z/23/bP9u/3f/f/9//3j/gf+F/4f/kv+I/33/ef9v/2L/Xv9a/0z/QP87/zT/NP8z/zX/Pf9A/0r/U/9d/13/Yf9l/2T/X/9c/1n/U/9P/0f/Qf9B/zb/L/8z/zH/Mf81/zX/K/84/zn/Pv9F/0r/Rf9S/1j/Tf9S/1T/Uv9a/1v/Rf9B/0L/PP8u/yH/Df8H/wr/Dv8j/zv/Vv9t/3j/bv99/4D/gP+K/4n/hP+E/47/jP+G/4L/fv98/3L/ev+E/4r/iv+H/5H/kv+O/4//kf+L/43/l/+l/6//wf/P/+z/AgALAB0AGwAgABsAEgAHAAMABQADAA4AGQAcACcANQBGAFIAXwBuAIAAjgCeAK4AvADMAM0A0QDbAOUA7gD4APsACQEYASQBNQFEAUgBRQFLAVABWAFiAW0BewGEAYkBiQGQAZoBmAGWAYoBgAGBAYoBlgGVAZcBiwGTAZ8BkgGHAXoBVwFNAU4BQwE7ASkBDgEKAf0ABAEFAQIB/wD8AAIB/AD/AAcB/wAEAQMB9gD4APsA+QD7APYA9QD1AO0A3QDSANAAzgDQANcA2gDSAMkAzADXANkA0gDbANkAzQDGANIA7gD3AP0A9wD2AOIA0QDSALoAuQCsALgAyQC9AMwAwAC3AK8AnwCcAJkAlgCHAIYAiwCLAIsAhAB5AHcAdwB2AIEAhACBAHwAeQB4AHMAbABkAGAAYgBhAGYAbAB0AH4AiACMAJQAmACZAJgAkgCNAIgAhQCAAHsAdwBwAG0AbgBuAG4AbgBwAHEAeACBAIcAkQCXAJoAnQCUAI0AiwCGAIEAeQByAHAAbwBrAGYAYwBeAFcATgBIAEYAQwBBAEEAQAA/AEEARABHAEsATABPAFQAVABSAE4ASABEAEMAPQA5ADIALwArACUAHgAZABYAEAANAAQA+//3/+//6v/j/+L/4//g/+T/3//h/+H/5//t/+//9//7/wYABAAHAAkABQD///r/9v/x/9z/zP+8/7b/rP+a/4z/ff90/23/bf9l/2L/V/9P/07/Qf83/yv/I/8S/wr//v70/u/+2/7X/sf+vf69/rr+s/6w/rT+r/6y/qr+pP6i/p3+l/6V/oz+gv6A/n/+g/5//nz+ef53/n7+e/6F/oP+fv59/m/+Zf5h/lT+Vv5Y/mP+bf5q/mH+V/5e/lz+WP5s/n3+gP6B/nH+c/50/nj+l/6g/rH+sf6b/q/+uP69/sv+xf67/sD+uf69/sf+w/7O/sH+uP7K/q7+wv6//tD+5v7Q/vj+Bv8C/w3/HP8g/yX/HP8J/wf/BP8H/wr/AP/+/vr+9v72/vj+7/7u/ur+4/7n/uX+7f73/v3+BP8I/wj/Df8W/xf/H/8c/xX/Df8H//z+8/7w/u7+7P71/gr/Gf8x/zX/Ov9D/0v/Tf9F/0D/N/9G/1D/Wv9Y/1L/U/9N/0X/Tv9J/0L/T/9D/zn/Q/8u/yz/Mf8+/zT/K/8y/y7/Qf85/z3/Sv9K/1X/bf97/4j/lv+V/5n/n/+p/7T/vP/B/83/2P/Z/9n/1f/X/+P/3f/b/+b/4P/r/+//8P/9//f/AwD+/wsAGwAKACkAKAAwAEoASQBZAFAAWgByAHAAeQB5AGoAfQCHAIsAkACPAJcAmACkAKUApwCwALUAxQDTANMA6wDzAP8AEwEoAUIBWwFkAW0BagFpAWsBagF0AXQBaAFhAV0BVQFbAVYBTwFXAVMBXgFwAW8BawFvAWgBXgFdAVwBUgFBASsBFwEGAQIB+AD9APkA/QD6APcA8gDuAO8A6wDzAO0A+wDxAPgA/wD/APwA7gDrANMAwgC0AKEAmgCSAI8AiACBAIAAegB9AH4AhgCOAJIAoACrAKoAqQCiAJwAnQCOAJcAmQCcAK8AsACnAKQAowCRAIwAgQBuAGsAZwBfAFkATgBHAEgAPgA3ADMAKwAnACUAHQAYABsAGwAdACEAJAAnACsALAAsACoAKQAnACMAIgArAC4AKQAnACUALwAxADsARQBNAFUAXABsAHYAgQCMAJwApQCwALUAwQDKAMsA0QDQANIAxwDKAM8AyQDEALsAuwC2ALQAsAC1ALgAtQCyALoAxgDNANcA6QDzAPQA9gD5APgA7wDkANwA1gDQAMsAzADMAM8A2QDrAPYAAwEJAQ0BCAEMAQwBDQEGAfkA9ADwAO4A6ADjANsA2QDXANwA3wDdANsA2wDbANsA3gDcANwA3ADaANoA2wDZANUA0gDMANAAyQC6AK8AnACLAH0AcgBkAF0AXQBQAEYANgAgABoAEwACAPb/6P/c/8//xP+9/7P/rP+r/6//rv+g/5//mP+N/3//bP9o/2f/af9i/13/Tv9H/0P/Qf86/y3/If8X/xL/DP8L/wv/CP8M/w//EP8M/wf/Cf8M/w//Ef8Q/xL/Ef8U/xD/CP/9/vP+6/7j/tj+yv6//rb+sv6o/qn+pP6g/qL+qf6y/rb+wP7F/s7+zf7O/s/+yf7M/sn+z/7S/tL+2P7b/uL+6v7u/vf++/4A/wT/Av8E/wT/AP8A/wP/B/8O/xL/Fv8d/yb/LP8w/zf/Pf9C/z7/Of8z/yz/Jv8g/x3/Gf8N/wb/AP/6/gH/Av8B///++/4A//P++f78/vr+Av8H/w3/Ev8T/xH/Ev8L/wv/DP8L/w3/Fv8X/xf/Hv8h/yv/M/88/0X/TP9S/1r/ZP9q/2z/b/9y/3X/dP93/2j/af9p/2b/Zf9R/2P/bf9i/2f/Y/9o/2D/X/9c/0//W/9V/2H/cP9z/3j/df98/3D/aP9f/1j/Uf9J/z7/Nv85/y//Nf8y/zj/Rf9N/1n/ZP9w/3r/hf+M/5H/lf+a/5r/nv+p/7H/vv/L/9n/7v/4/wcAEgAdACQAKgAzAD0AQABCAEIAQAA9ADUAOQA1ADQAMQAsAC0AKwAsAC0ALgAsADUAOgBAAEkASwBLAFUAXABkAG8AdQB9AHgAdgB0AHQAdgB4AHoAgQCMAJYAoACqAK8AuQDDANEA3gDuAPMA8wD1APoAAwEQARYBHwEnATIBNQE1ATYBMQEwASwBIwEfARcBFwEWARgBFwEWARYBFgEWARQBFQEPAQ8BAQH1AOwA4wDaAM0AzADAALsArwCmAKIAmQCRAIwAjgCPAJQAlwCXAJsAngClALAAugDCAM4A1ADbAOEA6ADuAPQA+wAFAQgBCwESARABBgH8APAA3gDRAMgAvgC7AL0AwgDJANAA0QDWANoA2ADYANgA2ADUAMoAxQDAALcAtgCwAK0AqgCpAK0ArACzALQAtQC6AL0AwwDKAMwAzgDRANQA1gDUAM4A0ADPAMoAxgDGAMAAvADBAMEAyADTANYA2wDgAOcA8QD5AP0A/QD8AP4A9wD0AOcA3gDVAMkAwAC2ALwAwADLANIAzwDSAM8A0ADZANgA2wDdAN4A3ADXANQAzQDMAMkAxwDGAL8AtgC3ALMApgCfAIwAgAB0AGwAZABmAGkAZwBuAGoAbQBqAGYAWwBQAEkARABEAD4ANgAvAC4AJgAfABsAGAATABIAEgAOAPv/7P/o/9z/1P/I/7T/qP+b/5D/hP92/2P/WP9K/zL/KP8X/wn/+/7v/uH+3f7P/sf+xP66/rP+oP6Z/on+fv5x/mv+cP5y/n7+fP6C/oX+gP6E/oH+f/54/nf+c/5s/mX+Xv5V/k3+Qf49/jP+Nf45/jr+Rf5D/kL+QP5B/j7+Rf5I/kT+RP42/jf+Nv4//kH+Qv5K/kn+Uf5S/l7+Xf5l/mD+Xv5r/mf+dv55/oz+m/6j/rP+vP68/sb+vf61/r3+tf62/r/+wv7C/sr+2f7h/vT+9/75/gL/+/72/vP+8f7t/uP+4/7d/tz+2f7V/tT+2f7m/un+8/7w/vz++/74/v3+6v7q/un+6/7i/tX+0/7N/sr+0P7P/sj+y/7H/sD+v/7A/sL+zP7X/tP+0/7P/tz+5v7z/gX/Bf8Q/xP/Ff8b/xv/Hf8W/xr/Dv8O/w3/Bf8J/wb/A/8E/wb//f73/vz+9/4B/wP/Ef8f/xf/H/8f/x7/Hv8Z/yX/LP8u/zv/Qf9J/1L/T/9j/2j/dP+R/5X/qv+4/8j/z//L/9r/2//X/+H/5P/k/+L/5//r/+r/6f/p/+L/4P/v//v/EQARABcAIgAoAEQASwBYAFgAWgBbAGoAeQCBAIkAfwCFAIgAoQC0AMEAzgDbAOQA8wACAQABBwEBAQMBFQEeAS0BRgFMAWkBcgGLAaABmQGnAZ0BoQGnAZsBoQGfAZkBowGfAZ4BmwGLAYcBeAFuAWIBXAFcAWEBVgFXAVEBQwE/AToBMQEsASwBKQEvATcBOgEwATQBMAEiARYBBgH5APYA8ADkAOAAzwDGAMwAyQDJAM0AywDGAMoAwgC9ALsAsACfAJoAkwCPAJ4AlgCIAJIAhwCJAIkAegB7AHEAZABtAGkAaQBqAGIAZgBhAGgAXQBUAFcAVQBSAEIANgA2ADQAMgAyACwAOQBDAE0AUgBVAEsARABKAD0ALQAQAAUA/f/s/93/1P/Q/8L/uP+1/7r/vv/C/8H/wf+9/7T/tv+w/6//r/+q/53/lf+R/47/hf9+/3v/cv95/3f/df90/27/cv93/3j/dv9u/3D/c/9x/3H/bP9m/2L/Zv9n/3H/bv9t/2b/Wf9V/0z/Sv9E/0f/Sv9S/17/bf9+/4b/kv+V/5n/ov+e/57/of+W/5X/lv+b/57/pf+l/6P/of+V/4z/ff98/4H/hf+K/4z/lP+Y/5j/m/+g/7L/vf/M/9n/3P/e/9z/4f/n//L/+f/6/+3/5f/l/+T/5P/k/+f/2f/X/9T/0//X/9T/0v/P/9P/2v/f/+H/9v/7/wMACgALAA8ACQAKAAMA9//4/+//6//t/97/2v/S/9b/0v/f/+b/5//z//P/AQAKABEAGAAdACAAGwAdABUAFgAhAB8AJQAoAC8AMgAsADAAJgAlABwAGAAUABAAFwANAAkA/P/4//X/7P/o/93/0f/H/8r/wP+5/77/u/+0/7D/r/+x/7b/vf+6/7f/uv+8/77/vP+9/7n/vf+9/7n/s/+u/6b/of+f/5z/lv+T/5T/l/+f/6b/rv+2/73/xP/H/87/0f/T/9b/2//h/+z/9//9/wMADgAXACcANgA8AD8AQQBHAE4AVABVAF0AYABiAGsAagB1AHsAewB/AIgAhwCMAI4AigCDAIQAfwB8AHwAfAB9AIMAjACQAJEAnACoAKwAtgC1ALUArgCfAJkAnACfAJAAkgCLAI8AkgCLAIwAigB5AGQAWABJADgANgA0AC4ANQA5AEgAUQBTAE0ATgBSAEwAQwAwAC8AOABAAD8APwBGAFcAXwBXAFkAWgBaAFcATwBLAEsASABEAEMAQgBFAE4AVwBqAH0AgwCXAKQAuQDNAOAA7AD5AAMBCAEPAQ4BDQEPARIBEgENAQoBCQEFAQ0BDwEVARsBGQEjAS0BNQE6AT8BRgFLAUIBRgFEAUQBRwFHAUEBQQE3AT8BPgE7AUgBPgFGATwBMQEsASUBKgEnASwBLAEuATEBLAErASMBHQEVAREBCgECAQIB6wDcAMkAxgDMAMwA0wDiAPQA+AD8AAABCAEJAQUBCAEGAfoA7QDcAM8AugCjAJgAjgCIAIYAiQCFAIYAfwB8AH0AcQBqAF8AVQBCADwANgAoACUAJAAfACMAHQAZACIAJAAkACIAJwAmACUAIwAkACQAKQA1ADgAPgA5ADUAOAA4ADYAMAAjABIACQAFAAgADAAJAAMA+//0//b/9P/u/+r/3v/a/9v/2//f/+X/5f/t//3/CwAaACgANAA6AEAASQBLAE0ATQBOAEwASgBLAEwATABMAE0AVABaAF4AaQB4AIcAmQCpALQAzADVAOQA6ADnAOgA5QDoAPIA+gD/APwA/gD4AOsA3wDQAM8AywDHAMkAywDOANEA1wDhAOYA7QDzAPoA/wABAQEB+wD2AO4A6ADgANgA2gDbANoA1QDRAM8AywDIAL0AsgCrAJwAiwB9AHAAZABWAFAATQBJAEgAPgA5ADQAKwAhAB4AEAAKAP//7//p/+H/2f/X/9j/0//V/8v/yP/O/7D/pP+b/4j/ev9z/2f/Yv9f/1X/SP87/yn/HP8F/wH/8/7j/uv+2v7b/tn+0v7E/rv+wv6//r3+vP60/q3+m/6V/oH+dv5v/mn+ev5w/nn+gP6F/pD+jv6G/oj+hP6C/n/+df5p/lL+V/5Z/mL+T/5Z/k3+T/5K/lL+UP5A/kH+If41/jT+Sv5R/l7+aP5x/oT+i/6J/pf+nP6c/pz+kv6Y/p3+l/6L/or+hf6F/oP+if6M/or+lv6a/qv+tv63/sz+0v7Y/uD+3v73/v/+B/8O/wz/D/8R/w3/DP8E/wL/Bf8G/wb/Af/+/v/+/f4A/wv/Cv8Q/xT/Ef8P/wn/BP8A//n+/f74/vD+9f7x/vT+8/79/gL/CP8S/xX/FP8U/xv/Dv8L/wX/9v76/vX+8f7t/vz+/f4I/wn/Ef8h/y//Of83/zX/Lf8w/yj/Ov87/zv/Ov82/0H/N/86/zX/Mf88/zn/N/89/0H/P/89/zv/N/88/zn/N/9B/z7/R/9L/1b/Xv9n/2v/b/9y/3b/fP98/4z/lP+j/6r/t//K/87/1f/i/+n/9P/6//P/9f/1//v/9//8/////P8GAAQAAgALAAkAFAAfACoANQA5AEMATgBaAF0AYwBgAFsAZABtAIEAjwCVAJYAmAChALoA0QDdAO8A/QAIASABHwEwATYBMgFIAVABXAFjAV4BZgFkAVkBTwFNAUsBSAFHAUsBUwFMATsBPQE5AT4BQQE+AUIBNgE3AS0BKQEzATMBPQFCAUcBSAFEAToBJAEYAQcB+wD0AOMA3QDTAMgAwgDEAMIAvQC7ALgAugC1ALAAqgCiAJwAlQCTAJQAnACfAJUAjgB/AHgAbwBeAD4AJAAVAA0A///8//v/AwAYABgAJwAjAB8AJQAqADMALgA9ADkARABcAF8AXQBjAHcAgQCDAIgAlwCaAJ4AngCjAKgAoACVAJwAmQCXAJoAnwCnAKgAsQC1ALgAwADJAMwA1wDjAOYA6QDnAOgA6ADpAOYA5wDhAOMA5wDnAPQA8wACARUBHAEhAScBMgE2AU8BXgFuAYEBhwGYAaQBqQGwAagBpAGbAYwBjQGRAYoBmgGgAZ8BsAGtAbYBxgHWAd4B5wH2Af0BEwIcAiMCIAIfAh0CIwIoAiACJQIeAhsCGwIeAh0CFwIUAhICGAIQAiECIwIpAjICMgInAhgCEAL8AeQB1QHMAbkBqAGlAZ0BlAGbAaABrgG6AcIBzQHTAdIByAG5AaYBjwF4AVsBPQEZAQIB5QDGAKwAkwB+AG0AYgBUAFAATQBKAEwARgBGAEQAQwBCAEEANwAnAA0A+f/i/7//qf+N/3v/bP9l/2D/Wf9M/zj/O/8q/yD/Gf8G//f+6P7e/tH+y/6//sD+vP66/rz+vP62/rL+sf6m/pr+hP5y/lD+M/4P/uz90v24/aX9kP1//W79Zf1R/Uz9P/1C/Uj9XP1t/Xf9gP12/Xr9a/1w/Wv9bv2B/YP9l/2i/af9tP29/cn91/3h/ev99P36/QX+Ff4l/jD+Nf44/j7+RP5T/lT+V/5O/kX+RP45/if+F/4A/u/95P3S/cz9w/3D/cn90P3V/d797f3r/ez9+v3+/fb9+/34/fT9+f32/fX98/3l/eb95v3m/f/9A/4H/hz+Iv5E/ln+Zv54/oP+g/6D/n/+f/6L/oz+kf6L/pj+q/65/sX+4v74/gX/Gf8R/yH/Iv8c/xf/EP8K/wf/Hf8s/zX/Rf9e/3L/c/97/4f/mf+o/8H/4P/0/wMAFQA0AE0AYABwAIgAlgCjALEAsgC5AMAAxwDgAPAA9wABAQcBEAESARgBGgEWARgBFQEUARMBGAEQAQEB9wDrAOwA5QDuAPAA9AD2APMA/QABAQYBDgESARoBIwEqATABMgE0AS0BLAE1AT0BTgFeAWkBawFvAXcBewGBAYkBjgGUAZ0BogG5Ac0B2QHxAf0BDAIYAhACHAIVAgoC+AHeAcsBrwGaAZkBfAFmAVkBSgFNAUIBPgExATEBKQEeARUBDQH2AOUA0wC8AKYAjQB9AGcAVgBJAEoARQA8AD4AOQAqACwAKgAnADMAOQA/ADwANgAzADAAJgAlABsAEAAHAPr/9f/n/97/1v/F/7r/uv+5/7D/pf+Y/4D/ff92/2n/Yv9i/23/dP92/3r/dv9z/3T/ZP9j/1n/Vv9S/0j/Q/80/y3/KP8a/xj/G/8X/xv/Fv8S/w//CP8J/wn/C/8O/xL/HP8j/yr/M/8w/zH/Mv80/zH/Mv89/z//Qf9J/1H/XP9g/2//cf91/4D/hv+D/4X/kP+E/43/i/+G/4v/h/+P/47/jv+a/6f/sf+5/8X/yv/Q/9v/2v/f/+L/4v/o/+r/7//v//P/9P/3//n//P/+/+//7//e/9//1P/H/8j/xv/X/9n/5v/m/97/6v/2/wUAEwAZABsAGgAeACMAJQAmACUAJwAeABcAEAAKAA8ADwAKAA0AFAAZACAAIgAhACUAJQAcABwAHgAZABoAGgAgABsAHgAiACcAJwAhACsAJQAmACAAHwAcAAgABADu/+v/5P/R/83/wf+5/7L/sf+p/6r/s/+p/7f/uf+//9f/4//y//P//v/7//X/8P/m/9n/0//c/97/7P/6/wgAEQAfADIATABjAG8AgwCRAJwAowCoAJ8AlwCSAIoAfQCBAIMAfwB4AHwAfgCDAIkAhQCNAI0AmACdAJ8ApwCoALAAuQDIANoA7QD5ABABHAEiAS0BLAE0AT0BRwFNAVYBYAFXAWMBcgFxAYMBkgGTAaMBpQGoAbYBvwHBAccBtwHFAbcBxAHKAckB9gHpAQUCJwI3AkkCUQJfAl8CZgJXAlICRwIvAiYCGQICAvEB7wHqAeIB8AHuAekB6wHxAfIB9gH6Af0BAgICAgECAAL9AfQB7gHjAdkBwQGpAYwBbAFRAT0BMgE0AUMBTAFeAWkBbQFyAW8BaQFaAUsBMgEnARwBEwH9AOYAzwCyAJoAiQB5AG8AbwBmAFgAUQA5AC4AJgAkABEA9v/j/9X/z/+3/6j/l/+A/23/YP9Z/07/Qf88/zL/Lv8t/y//Mf8w/zP/Mv8v/yj/Gf8D/+v+zP6u/pP+cf5a/kL+Kf4c/gP++P3p/dn90f26/bj9ov2c/Zv9kP2P/ZL9oP2t/b/9x/3U/dX93P3l/eL93/3W/c39v/24/bb9rv2z/bH9vf3L/dv9+P0M/h/+Kv5B/ln+cv6F/pP+oP6k/q/+q/6r/qD+jv59/mj+Uv5K/jn+L/4v/i7+NP4+/kT+Rf5K/kz+Tv5S/lL+Qf41/i3+Ff4N/gj++f31/ej96/3r/fn9Bf4i/jf+Q/5U/lj+bf5s/ob+lv6m/rv+z/7m/u3+8v7y/vD+9P70/u/+9P7z/vT+7/7a/tL+0/7P/sn+zv7I/sD+u/6p/p3+mv6R/o7+if6A/oP+gP6I/pL+nf6o/rD+vf7K/tn+6P7z/vv+9/77/v3+9v73/vT+8/75/gD/CP8V/yD/MP9D/1j/b/+C/5b/pP+x/8L/0P/d/+z/AQATACgANQA9AE0AWwBwAHoAhgCIAIwAkwCZAKIAoACuAL4A1QDoAPgACAESASIBMgFAAUkBUQFYAV8BZQFsAWsBcAF1AW4BcwF0AXABdQF4AYEBkwGdAaUBuQHGAdgB5AH1Af4BBwIPAhoCJQIkAiYCLAIvAjECNgI2Ai0CKAIlAiICHQIgAh8CHAIeAiECIgInAi0CMQI1AkECSgJMAksCTQJHAjwCLwIiAhUCCQIKAhICHAIkAigCNQJAAkoCUwJdAl4CYgJaAlUCRAIyAh0CDgL/AfAB5gHeAdYBzgHHAb0BsQGoAZ8BnAGZAZYBlgGVAY8BjAGEAXIBaQFfAVIBSwE9ATABJQEUAQkB/gDvAOYA4wDZANkA2wDYANUA0gDPAM0AzADNANQA3ADiAO8A+AD7AP4A+wD2APAA5wDcANIAwgCwAJoAhgB0AGMAWgBUAFUAVgBSAE4ASgBDADkALgAgABQAAgDz/+D/zP+5/6z/n/+V/43/hf+B/3j/a/9X/0b/O/80/zH/Kv8q/yX/If8b/xT/D/8B//7++f70/vb+8/71/vL+8f7w/vL+8/7y/u/+9P76/v7+Bv8H/wj/B/8H/wb/BP8C//3++f74/vj+/f4A/wn/Fv8j/zL/QP9Q/1b/Wv9c/1v/Wv9X/1T/Sv9H/0D/Ov85/zv/PP9A/0j/Tf9S/1n/Vv9c/2H/Xf9c/1r/QP82/zL/JP8b/w//BP/8/vv+6v7f/tj+0v7H/sD+t/64/r7+xP7S/uX++f4J/xn/Gf8j/yv/Lf83/zT/LP8q/y//Lv8n/yT/If8h/x3/JP8u/zf/NP8u/zL/Mv8x/zT/Ov88/0T/Uv9k/3H/f/+J/5f/nv+Z/5n/jv+F/3r/cP9g/1j/Uf9E/0H/Pf82/zL/L/8q/yb/If8d/xf/EP8N/wj/CP8G/wH/+/74/vX+9v71/vb+9v7//gX/Cv8U/xn/Hf8s/zf/Sv9a/2r/g/+P/57/q/+y/7v/v//E/7z/sf+t/6z/rf+p/6z/p/+o/7D/sf+u/6r/kP99/3j/aP9f/1n/Tv9P/0//Xv9t/3f/gf+S/6f/sv/A/9L/2v/o//P/+P8GAAsAEAAfACMAHgAcABYADwAGAP//AgACAAEAAwABAPb/5//i/+f/5//w//r/+P/4/wMAGQAuAEYATABQAEwARABHADcAPQBAAEkAagB0AJIApgCzAMoA0gDfAOoA9QD4AP8AAwELARQBFwEWARkBHgEhASoBNAE3AS8BJQEfARgBEQEKAQ4BGwElATEBPgFJAVgBYQFoAXQBewF/AYIBgAF+AXYBcAFpAWQBXAFQAUMBOgEuASUBGwEUAQ4BEAEXASEBLgE4AT0BQwE/ATYBLQEfAREBCQECAf4AAwEDAQQBCQEMAQ0BDAEMAQkBBwEBAfoA9ADtAOwA8wD8AAUBBwEIAQsBBwECAfgA5gDVAMcAuwC4AK8AqACiAJoAjgCFAIAAfAB7AHYAagBbAD8AJQAMAAMA///4//T/5//g/9T/0f/O/8b/xP+7/8D/uP+w/6j/mf+M/4X/gP+A/4v/kv+Z/6T/rP+s/67/rv+u/7b/vv/A/8H/vv++/7//uv+z/6z/o/+W/5H/h/99/3L/Yv9c/1T/UP9T/1T/WP9d/2X/bf9y/3X/dP90/3P/cv9x/27/a/9q/2r/av9o/2X/YP9f/2H/X/9i/2H/W/9c/1P/TP9G/z7/Pv89/z//RP9A/z3/Ov85/zb/MP8t/zD/Kf8f/xr/Ff8R/xH/F/8S/xn/G/8U/xr/Gv8X/xf/Ev8I/wH/+f7v/u7+5v7g/tz+zf7R/sX+y/7P/tD+4v7c/un++v4E/w7/FP8b/yD/Jf8m/yX/I/8l/yT/JP8j/x3/Gv8T/wr/Av/9/vP+6P7g/tj+2P7W/tH+z/7Q/sz+zf7S/sz+yf7A/rX+qP6f/pf+jv6O/pP+lf6h/rL+w/7Y/ub+8/4E/xL/If8q/zT/Qv9Y/2f/dv+F/5D/ov+x/7r/zP/U/9j/4//n/+b/7//o/+3/9//+//3//v8IAAgAEgATABUAHAAZAB4AKgAtADIAOQA9AEMASQBUAF0AZwBxAHkAgACBAIQAggCEAIcAhACDAI4AjgCXAKAAqQC5ALwAyQDQAN0A7wDyAAsBDwEXASUBJgExATEBNwFBAT0BPgE+ATgBOwE9ATYBMwEuASoBIAEdARIBCwEKAQcBCgEJAQYBEQESARgBIgEsATEBNgEvASsBIQEVAQwBAgEBAfoA7gDoAOYA3gDgAN8A3gDiAN4A4gDoAOYA4QDiAN0A0QDOAMoAxAC5ALAAoQCXAJEAgwB8AHAAZgBdAFYASwBEAEAAPQA/ADgAPgA4AEAAQwBEAEYAQwBDADYALQAnABoAFAAPAAgABwADAAAAAADm/9T/0//N/8b/wf+5/67/pP+X/4r/iP99/3//gf98/4P/gP93/3P/cv9l/1//Vf9N/03/S/9I/0L/Of80/zb/Mf8v/y7/Lf8t/zT/OP86/0L/Sf9T/2D/bP91/4D/hP+M/5T/lv+Z/5//of+t/7X/tv+8/77/yv/U/9//7v/0//7/BgASAB8ALAA7AEsAWABjAG0AdwB+AIIAigCJAIoAhQCFAIUAfgB3AHAAcABtAG0AbwB1AHwAfQB/AIgAkwCdAKgAuwDGAMoA0ADVANUA1QDQAM0AygDGAMYAyQDKAM4A1QDkAOwA+gABAQgBBgEMAQ4BDgEKAQQBAQH/AP8A+gD1AO8A7ADqAO8A8QDxAPAA7gDuAOwA8AD0APkAAQEHAQ0BFwEcASABIgEiAScBJwEcARMBBQH3AO0A5gDaANYA2gDRAM4AygC+ALwAuACsAKYAoACVAJEAjACIAIcAggCDAIkAjwCHAIwAiwCHAIYAewB3AHkAegB4AHoAcQBoAGYAYwBiAFwAUgBKAEkAQwBFAEoASQBMAFEAVQBYAFkAWwBhAGQAZgBmAGkAaABqAGYAXwBYAE8ARwA+ADsAMwArACcAJwAjACYAJwAsADEANABAAEcAUABVAFgAVQBUAFMATABKAEUARABCAEAAPQA7AEAARgBNAFQAWgBfAGEAYQBlAGUAZwBqAHAAeACBAIgAkACZAKMArQC3AMEAyADNANAA0ADPAM4AywDJAM4A0ADOANAA0wDVAOEA4QDlAOoA6gDwAO4A9gD9AP4ABAEJAQ0BEgETARMBGQETARIBEQEOAQ0BEQEJAQYBBwEAAQYBCgEIAQgBBgEEAQgB\" type=\"audio/x-wav\" />\n",
              "                    Your browser does not support the audio element.\n",
              "                </audio>\n",
              "              "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ghaUw-vz_qLa"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}