diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/.DS_Store differ diff --git a/genetic.ipynb b/genetic.ipynb index 9fe67ed..4c7bde5 100644 --- a/genetic.ipynb +++ b/genetic.ipynb @@ -1,615 +1,748 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sdi2UTA8vWEJ" - }, - "source": [ - "A proof of concept showing using a genetic algorithm with our environment.\n", - "It is similar to https://github.com/DEAP/deap/blob/a0b78956e28387785e3bb6e2b4b1f1b32c2b3883/examples/ga/onemax_short.py" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "accelerator": "GPU", "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 + "name": "genetic_newest+RL_tunning+print-whats-inside-tuning+add_all_imports.ipynb", + "provenance": [], + "collapsed_sections": [] }, - "colab_type": "code", - "id": "Qv1kWykfqRLl", - "outputId": "b9839811-2c7d-4132-f332-990247867c58", - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "fatal: destination path 'meta-transfer-learning' already exists and is not an empty directory.\n", - "Collecting ray==0.9.0.dev0\n", - " Using cached https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl\n", - "Requirement already satisfied, skipping upgrade: jsonschema in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.6.0)\n", - "Requirement already satisfied, skipping upgrade: colorama in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.4.3)\n", - "Requirement already satisfied, skipping upgrade: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (7.1.2)\n", - "Requirement already satisfied, skipping upgrade: msgpack<1.0.0,>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.6.2)\n", - "Requirement already satisfied, skipping upgrade: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.18.3)\n", - "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.0.12)\n", - "Requirement already satisfied, skipping upgrade: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.10.0)\n", - "Requirement already satisfied, skipping upgrade: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.13)\n", - "Requirement already satisfied, skipping upgrade: redis<3.5.0,>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.4.1)\n", - "Requirement already satisfied, skipping upgrade: grpcio in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.28.1)\n", - "Requirement already satisfied, skipping upgrade: google in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.0.3)\n", - "Requirement already satisfied, skipping upgrade: aiohttp in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.6.2)\n", - "Requirement already satisfied, skipping upgrade: py-spy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.3.3)\n", - "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray==0.9.0.dev0) (46.1.3)\n", - "Requirement already satisfied, skipping upgrade: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray==0.9.0.dev0) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from google->ray==0.9.0.dev0) (4.6.3)\n", - "Requirement already satisfied, skipping upgrade: chardet<4.0,>=2.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.0.4)\n", - "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.5; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.6.6)\n", - "Requirement already satisfied, skipping upgrade: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.0.1)\n", - "Requirement already satisfied, skipping upgrade: attrs>=17.3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (19.3.0)\n", - "Requirement already satisfied, skipping upgrade: yarl<2.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (1.4.2)\n", - "Requirement already satisfied, skipping upgrade: idna-ssl>=1.0; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (1.1.0)\n", - "Requirement already satisfied, skipping upgrade: multidict<5.0,>=4.5 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (4.7.5)\n", - "Requirement already satisfied, skipping upgrade: idna>=2.0 in /usr/local/lib/python3.6/dist-packages (from yarl<2.0,>=1.0->aiohttp->ray==0.9.0.dev0) (2.9)\n", - "Installing collected packages: ray\n", - " Found existing installation: ray 0.9.0.dev0\n", - " Uninstalling ray-0.9.0.dev0:\n", - " Successfully uninstalled ray-0.9.0.dev0\n", - "Successfully installed ray-0.9.0.dev0\n", - "Requirement already satisfied: cloudpickle==1.2.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 1)) (1.2.1)\n", - "Requirement already satisfied: future==0.17.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 2)) (0.17.1)\n", - "Requirement already satisfied: gym in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 3)) (0.17.1)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 4)) (1.18.3)\n", - "Requirement already satisfied: Pillow==6.2.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 5)) (6.2.0)\n", - "Requirement already satisfied: pyglet==1.3.2 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 6)) (1.3.2)\n", - "Requirement already satisfied: scipy==1.3.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 7)) (1.3.0)\n", - "Requirement already satisfied: six==1.12.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 8)) (1.12.0)\n", - "Requirement already satisfied: pytest_cases in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 9)) (1.13.1)\n", - "Requirement already satisfied: deap in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 10)) (1.3.1)\n", - "Requirement already satisfied: ray[rllib] in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 11)) (0.9.0.dev0)\n", - "Requirement already satisfied: tensorflow in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 12)) (2.2.0rc4)\n", - "Requirement already satisfied: decopatch in /usr/local/lib/python3.6/dist-packages (from pytest_cases->-r requirements.txt (line 9)) (1.4.8)\n", - "Requirement already satisfied: makefun>=1.7 in /usr/local/lib/python3.6/dist-packages (from pytest_cases->-r requirements.txt (line 9)) (1.9.2)\n", - "Requirement already satisfied: wrapt in /usr/local/lib/python3.6/dist-packages (from pytest_cases->-r requirements.txt (line 9)) (1.12.1)\n", - "Requirement already satisfied: google in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (2.0.3)\n", - "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.13)\n", - "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (2.6.0)\n", - "Requirement already satisfied: msgpack<1.0.0,>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.6.2)\n", - "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.10.0)\n", - "Requirement already satisfied: redis<3.5.0,>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.4.1)\n", - "Requirement already satisfied: colorama in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.4.3)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.0.12)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.6.2)\n", - "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (7.1.2)\n", - "Requirement already satisfied: grpcio in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (1.28.1)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.3.3)\n", - "Requirement already satisfied: pandas; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (1.0.3)\n", - "Requirement already satisfied: tabulate; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.8.7)\n", - "Requirement already satisfied: tensorboardX; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (2.0)\n", - "Requirement already satisfied: atari-py; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.2.6)\n", - "Requirement already satisfied: dm-tree; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.1.5)\n", - "Requirement already satisfied: opencv-python-headless; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (4.2.0.34)\n", - "Requirement already satisfied: lz4; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.0.2)\n", - "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.9.0)\n", - "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (1.1.0)\n", - "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (1.1.0)\n", - "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.2.0)\n", - "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (2.10.0)\n", - "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.34.2)\n", - "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (1.6.3)\n", - "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.3.3)\n", - "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (3.2.1)\n", - "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (2.2.1)\n", - "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (2.2.0)\n", - "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from google->ray[rllib]->-r requirements.txt (line 11)) (4.6.3)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray[rllib]->-r requirements.txt (line 11)) (46.1.3)\n", - "Requirement already satisfied: typing-extensions>=3.6.5; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (3.6.6)\n", - "Requirement already satisfied: chardet<4.0,>=2.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (3.0.4)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (1.4.2)\n", - "Requirement already satisfied: idna-ssl>=1.0; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (1.1.0)\n", - "Requirement already satisfied: multidict<5.0,>=4.5 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (4.7.5)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (19.3.0)\n", - "Requirement already satisfied: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (3.0.1)\n", - "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas; extra == \"rllib\"->ray[rllib]->-r requirements.txt (line 11)) (2018.9)\n", - "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas; extra == \"rllib\"->ray[rllib]->-r requirements.txt (line 11)) (2.8.1)\n", - "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (3.2.1)\n", - "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (2.23.0)\n", - "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.6.0.post3)\n", - "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.0.1)\n", - "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.7.2)\n", - "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (0.4.1)\n", - "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.6/dist-packages (from yarl<2.0,>=1.0->aiohttp->ray[rllib]->-r requirements.txt (line 11)) (2.9)\n", - "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.24.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (2020.4.5.1)\n", - "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (3.1.1)\n", - "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (4.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (0.2.8)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.3.0)\n", - "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (0.4.8)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (3.1.0)\n", - "\u001b[31mERROR: Could not find a version that satisfies the requirement ran (from versions: none)\u001b[0m\n", - "\u001b[31mERROR: No matching distribution found for ran\u001b[0m\n" - ] + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" } - ], - "source": [ - "# Run this cell if you're using colab. Otherwise, skip it.\n", - "\n", - "!git clone https://github.com/platers/meta-transfer-learning.git\n", - "\n", - "import os\n", - "os.chdir('meta-transfer-learning')\n", - "\n", - "!pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl\n", - "\n", - "!pip install -r requirements.txt\n", - "!pip install ran" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ojEPVGZmvTAe" - }, - "outputs": [], - "source": [ - "import array\n", - "import random\n", - "\n", - "import numpy as np\n", - "from typing import Dict\n", - "\n", - "from deap import algorithms\n", - "from deap import base\n", - "from deap import creator\n", - "from deap import tools\n", - "import gym\n", - "\n", - "from environments import SimpleEnv\n", - "import importlib\n", - "from environments import SimpleEnv\n", - "importlib.reload(SimpleEnv)\n", - "\n", - "from ray.rllib.agents import ppo\n", - "from ray import tune\n", - "from ray.rllib.policy.policy import Policy\n", - "from ray.rllib.env import BaseEnv\n", - "from ray.rllib.env.multi_agent_env import MultiAgentEnv\n", - "from ray.rllib.evaluation import MultiAgentEpisode, RolloutWorker\n", - "from ray.rllib.agents.callbacks import DefaultCallbacks\n", - "import ray\n", - "\n", - "import time\n", - "start = time.process_time()\n", - "\n", - "from ray import tune\n", - "\n", - "SimpleEnv = SimpleEnv.SimpleEnv" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wvaDdyNmEQ-o" - }, - "outputs": [], - "source": [ - "# some settings that we can tweak:\n", - "config_evolution={\n", - " \"MAX_STEP_COUNT\": 1, # number of steps in each round of DEAP evolution \n", - " \"POPULATION_SIZE\": 6, # population in each round of DEAP evolution \n", - " \"N_GEN\": 6 # number of rounds of DEAP evolution This has the most effect\n", - " }\n", - "N_RL_TRANING = 1 # number of rounds of RL training\n", - "TRAIN_BATCH_SIZE = 1 #Batch size for RL training\n", - "LEARNING_RATE = 0.001 #learning rate for RL training\n", - "EVAL_METHOD = \"OPTIMAL\" # RL or OPTIMAL\n", - "TUNING_EVOLUTION = False" - ] }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 71 + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Sdi2UTA8vWEJ" + }, + "source": [ + "A proof of concept showing using a genetic algorithm with our environment.\n", + "It is similar to https://github.com/DEAP/deap/blob/a0b78956e28387785e3bb6e2b4b1f1b32c2b3883/examples/ga/onemax_short.py" + ] }, - "colab_type": "code", - "id": "_cnG-FTpEQ-1", - "outputId": "b67b0b3c-5dc8-4d98-b6e4-b7c801acb818" - }, - "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "2020-05-08 17:21:04,728\tINFO resource_spec.py:212 -- Starting Ray with 29.69 GiB memory available for workers and up to 14.86 GiB for objects. You can adjust these settings with ray.init(memory=, object_store_memory=).\n", - "2020-05-08 17:21:05,174\tINFO services.py:1170 -- View the Ray dashboard at \u001b[1m\u001b[32mlocalhost:8265\u001b[39m\u001b[22m\n" - ] - } - ], - "source": [ - "ray.shutdown()\n", - "ray.init()\n", - "n_agents = 3\n", - "n_var = 2\n", - "training_envs = [\n", - "(SimpleEnv, {\n", - " 'n_agents': n_agents,\n", - " 'n_vars': n_var,\n", - " 'true_reward_weights': [1, 0],\n", - " 'max_step_count': config_evolution[\"MAX_STEP_COUNT\"],\n", - "}),\n", - "(SimpleEnv, {\n", - " 'n_agents': n_agents,\n", - " 'n_vars': n_var,\n", - " 'true_reward_weights': [0, 1],\n", - " 'max_step_count': config_evolution[\"MAX_STEP_COUNT\"],\n", - "})]\n", - "test_env = (SimpleEnv, {\n", - " 'n_agents': n_agents,\n", - " 'n_vars': n_var,\n", - " 'true_reward_weights': [1, 1],\n", - " 'max_step_count': config_evolution[\"MAX_STEP_COUNT\"],\n", - "})\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yn9RzymlEQ_G" - }, - "outputs": [], - "source": [ - "# Directly generate \"optimal action\" suggested by the reward function created by Evolution\n", - "# reward_weights is created from env(config=env_config)\n", - "def get_optimal_action(reward_weights, env_config):\n", - " n_vars = env_config[\"n_vars\"]\n", - " max_act = 5 \n", - " \n", - " reward_scale_factor = np.array([2]*n_vars + [1]*(len(reward_weights)-n_vars))\n", - " action = reward_weights * reward_scale_factor\n", - " action = action / np.max(action) * max_act\n", - " \n", - " action = np.reshape(action, [n_agents, n_vars])\n", - " return action\n", - "\n", - "# If we do not want to optimize RL but just want to optimize reward function using Evolution function\n", - "# then here we just do the \"optimal action\" suggested by the generated reward function\n", - "def evaluate_individual_env_optimal_act(individual, environment_fn, env_config):\n", - " env_config['reward_weights'] = np.array([individual for i in range(n_agents)])\n", - " env = environment_fn(config=env_config)\n", - " \n", - " ave_true_rewards = 0\n", - " obs = env.reset()\n", - " ave_reward = 0\n", - " for _ in range(env_config[\"max_step_count\"]):\n", - " actions = {i: get_optimal_action(env.reward_weights[i], env_config) for i in range(n_agents)}\n", - " obs, reward, _, _ = env.step(actions)\n", - " reward = np.array([reward[i] for i in range(len(reward))])\n", - " ave_reward += reward\n", - " ave_true_rewards += env.last_true_reward\n", - " \n", - " ave_true_rewards /= env_config[\"max_step_count\"]\n", - " ave_reward /= env_config[\"max_step_count\"]\n", - " \n", - " return np.mean(ave_true_rewards)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oP1G3iuuolJi" - }, - "outputs": [], - "source": [ - "# train RL agent with given setup\n", - "def evaluate_individual_env_rl(individual, environment, env_config):\n", - " \"\"\"Runs the environment. All agents have the same policy.\n", - " It returns the total true reward as the fitness.\n", - " \"\"\"\n", - " #Select random individuals from pop and create the reward weights\n", - " pop = np.array([individual for i in range(n_agents)])\n", - " reward_weights = pop\n", - " env_config['reward_weights'] = reward_weights\n", - " #env is only to get action space and observation space\n", - " env = environment(config=env_config)\n", - " class MyCallbacks(DefaultCallbacks):\n", - " #Callback functions to keep track of true reward while training\n", - " def on_episode_start(self, worker: RolloutWorker, base_env: BaseEnv,\n", - " policies: Dict[str, Policy],\n", - " episode: MultiAgentEpisode, **kwargs):\n", - " episode.user_data[\"true_rewards\"] = np.zeros(n_agents)\n", - "\n", - " def on_episode_step(self, worker: RolloutWorker, base_env: BaseEnv,\n", - " episode: MultiAgentEpisode, **kwargs):\n", - " env = base_env\n", - " true_reward = env.env_states[0].env.last_true_reward\n", - " episode.user_data[\"true_rewards\"] += true_reward\n", - "\n", - " def on_episode_end(self, worker: RolloutWorker, base_env: BaseEnv,\n", - " policies: Dict[str, Policy], episode: MultiAgentEpisode,\n", - " **kwargs):\n", - " true_reward = episode.user_data[\"true_rewards\"]\n", - " for i, r in enumerate(true_reward):\n", - " episode.custom_metrics[\"true_reward_agent_\" + str(i)] = r\n", - " \n", - " # settings for the RL agent trainer \n", - " config={\n", - " \"train_batch_size\": TRAIN_BATCH_SIZE,\n", - " \"lr\": LEARNING_RATE,\n", - " \"sgd_minibatch_size\": TRAIN_BATCH_SIZE,\n", - " \"multiagent\": {\n", - " \"policies\": {\n", - " },\n", - " \"policy_mapping_fn\": #all agents share a policy\n", - " lambda agent_id:\n", - " 'agent'\n", - " },\n", - " \"model\": {\"fcnet_hiddens\": []},\n", - " 'env_config': env_config,\n", - " \"callbacks\": MyCallbacks,\n", - " }\n", - " config['multiagent']['policies']['agent'] = (None, env.observation_space, env.action_space, {})\n", - " metrics = None\n", - " while True:\n", - " trainer = ppo.PPOTrainer(env=environment, config=config)\n", - " true_reward_mean = 0\n", - " for i in range(N_RL_TRANING):\n", - " true_reward_mean = 0\n", - " #Train the RL agent\n", - " metrics = trainer.train() # distributed training step\n", - " print(\"episode_reward_mean\", metrics[\"episode_reward_mean\"])\n", - " if metrics[\"episode_reward_mean\"] < 0 and i > 2:\n", - " break\n", - " #Train agent until it does well\n", - " if metrics[\"episode_reward_mean\"] > 0:\n", - " break\n", - " print(\"episode_reward_mean\", metrics[\"episode_reward_mean\"])\n", - " for i in range(n_agents):\n", - " true_reward_mean += metrics['custom_metrics']['true_reward_agent_' + str(i) + '_mean']\n", - " true_reward_mean /= n_agents\n", - " print('Evaluated', individual, 'Fitness', true_reward_mean)\n", - " return true_reward_mean" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dy4idwUWEQ_a" - }, - "outputs": [], - "source": [ - "if EVAL_METHOD == \"RL\":\n", - " evaluate_individual_env = evaluate_individual_env_rl\n", - "else:\n", - " evaluate_individual_env = evaluate_individual_env_optimal_act\n", - " \n", - "def evaluate_individual(individual):\n", - " \"\"\"Runs all environments. \n", - " returns the average true reward over all environments as the fitness.\n", - " \"\"\"\n", - " \n", - " all_fitness = []\n", - " for env, config in training_envs:\n", - " all_fitness.append(evaluate_individual_env(individual, env, config))\n", - " fitness_comb = np.min(all_fitness)\n", - " return (fitness_comb, )" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 85 + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "Qv1kWykfqRLl", + "outputId": "b88f0cdc-4599-46e4-c242-9b7268507a37", + "scrolled": true, + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + } + }, + "source": [ + "# Run this cell if you're using colab. Otherwise, skip it.\n", + "\n", + "!git clone https://github.com/platers/meta-transfer-learning.git\n", + "\n", + "import os\n", + "os.chdir('meta-transfer-learning')\n", + "\n", + "!pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl\n", + "\n", + "!pip install -r requirements.txt\n", + "!pip install ran\n", + "\n", + "\n", + "# Make Colab run faster. make sure that the current GPU memory utilization is 0\n", + "# memory footprint support libraries/code\n", + "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n", + "!pip install gputil\n", + "!pip install psutil\n", + "!pip install humanize\n", + "import psutil\n", + "import humanize\n", + "import os\n", + "import GPUtil as GPU\n", + "GPUs = GPU.getGPUs()\n", + "# XXX: only one GPU on Colab and isn’t guaranteed\n", + "gpu = GPUs[0]\n", + "def printm():\n", + " process = psutil.Process(os.getpid())\n", + " print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n", + " print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n", + "printm() " + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "text": [ + "fatal: destination path 'meta-transfer-learning' already exists and is not an empty directory.\n", + "Collecting ray==0.9.0.dev0\n", + " Using cached https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.18.4)\n", + "Requirement already satisfied, skipping upgrade: aiohttp in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.6.2)\n", + "Requirement already satisfied, skipping upgrade: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.13)\n", + "Requirement already satisfied, skipping upgrade: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (7.1.2)\n", + "Requirement already satisfied, skipping upgrade: redis<3.5.0,>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.4.1)\n", + "Requirement already satisfied, skipping upgrade: jsonschema in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.6.0)\n", + "Requirement already satisfied, skipping upgrade: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.10.0)\n", + "Requirement already satisfied, skipping upgrade: grpcio in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (1.28.1)\n", + "Requirement already satisfied, skipping upgrade: msgpack<1.0.0,>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.6.2)\n", + "Requirement already satisfied, skipping upgrade: google in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (2.0.3)\n", + "Requirement already satisfied, skipping upgrade: py-spy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.3.3)\n", + "Requirement already satisfied, skipping upgrade: colorama in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (0.4.3)\n", + "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from ray==0.9.0.dev0) (3.0.12)\n", + "Requirement already satisfied, skipping upgrade: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.0.1)\n", + "Requirement already satisfied, skipping upgrade: attrs>=17.3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (19.3.0)\n", + "Requirement already satisfied, skipping upgrade: yarl<2.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (1.4.2)\n", + "Requirement already satisfied, skipping upgrade: idna-ssl>=1.0; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (1.1.0)\n", + "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.5; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.6.6)\n", + "Requirement already satisfied, skipping upgrade: chardet<4.0,>=2.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (3.0.4)\n", + "Requirement already satisfied, skipping upgrade: multidict<5.0,>=4.5 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray==0.9.0.dev0) (4.7.5)\n", + "Requirement already satisfied, skipping upgrade: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray==0.9.0.dev0) (1.12.0)\n", + "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray==0.9.0.dev0) (46.1.3)\n", + "Requirement already satisfied, skipping upgrade: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from google->ray==0.9.0.dev0) (4.6.3)\n", + "Requirement already satisfied, skipping upgrade: idna>=2.0 in /usr/local/lib/python3.6/dist-packages (from yarl<2.0,>=1.0->aiohttp->ray==0.9.0.dev0) (2.9)\n", + "Installing collected packages: ray\n", + " Found existing installation: ray 0.9.0.dev0\n", + " Uninstalling ray-0.9.0.dev0:\n", + " Successfully uninstalled ray-0.9.0.dev0\n", + "Successfully installed ray-0.9.0.dev0\n", + "Requirement already satisfied: cloudpickle==1.2.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 1)) (1.2.1)\n", + "Requirement already satisfied: future==0.17.1 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 2)) (0.17.1)\n", + "Requirement already satisfied: gym in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 3)) (0.17.1)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 4)) (1.18.4)\n", + "Requirement already satisfied: Pillow==6.2.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 5)) (6.2.0)\n", + "Requirement already satisfied: pyglet==1.3.2 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 6)) (1.3.2)\n", + "Requirement already satisfied: scipy==1.3.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 7)) (1.3.0)\n", + "Requirement already satisfied: six==1.12.0 in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 8)) (1.12.0)\n", + "Requirement already satisfied: pytest_cases in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 9)) (1.13.1)\n", + "Requirement already satisfied: deap in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 10)) (1.3.1)\n", + "Requirement already satisfied: ray[rllib] in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 11)) (0.9.0.dev0)\n", + "Requirement already satisfied: tensorflow in /usr/local/lib/python3.6/dist-packages (from -r requirements.txt (line 12)) (2.2.0rc4)\n", + "Requirement already satisfied: decopatch in /usr/local/lib/python3.6/dist-packages (from pytest_cases->-r requirements.txt (line 9)) (1.4.8)\n", + "Requirement already satisfied: wrapt in /usr/local/lib/python3.6/dist-packages (from pytest_cases->-r requirements.txt (line 9)) (1.12.1)\n", + "Requirement already satisfied: makefun>=1.7 in /usr/local/lib/python3.6/dist-packages (from pytest_cases->-r requirements.txt (line 9)) (1.9.2)\n", + "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (2.6.0)\n", + "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (7.1.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.0.12)\n", + "Requirement already satisfied: msgpack<1.0.0,>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.6.2)\n", + "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.10.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.6.2)\n", + "Requirement already satisfied: google in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (2.0.3)\n", + "Requirement already satisfied: grpcio in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (1.28.1)\n", + "Requirement already satisfied: colorama in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.4.3)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.3.3)\n", + "Requirement already satisfied: redis<3.5.0,>=3.3.2 in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.4.1)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.13)\n", + "Requirement already satisfied: tensorboardX; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (2.0)\n", + "Requirement already satisfied: opencv-python-headless; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (4.2.0.34)\n", + "Requirement already satisfied: tabulate; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.8.7)\n", + "Requirement already satisfied: lz4; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (3.0.2)\n", + "Requirement already satisfied: atari-py; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.2.6)\n", + "Requirement already satisfied: pandas; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (1.0.3)\n", + "Requirement already satisfied: dm-tree; extra == \"rllib\" in /usr/local/lib/python3.6/dist-packages (from ray[rllib]->-r requirements.txt (line 11)) (0.1.5)\n", + "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.2.0)\n", + "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (2.2.1)\n", + "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (1.1.0)\n", + "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (1.6.3)\n", + "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (2.10.0)\n", + "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.9.0)\n", + "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (2.2.0)\n", + "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (1.1.0)\n", + "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.34.2)\n", + "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (0.3.3)\n", + "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow->-r requirements.txt (line 12)) (3.2.1)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->ray[rllib]->-r requirements.txt (line 11)) (46.1.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (1.4.2)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (19.3.0)\n", + "Requirement already satisfied: multidict<5.0,>=4.5 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (4.7.5)\n", + "Requirement already satisfied: chardet<4.0,>=2.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (3.0.4)\n", + "Requirement already satisfied: idna-ssl>=1.0; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (1.1.0)\n", + "Requirement already satisfied: typing-extensions>=3.6.5; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (3.6.6)\n", + "Requirement already satisfied: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.6/dist-packages (from aiohttp->ray[rllib]->-r requirements.txt (line 11)) (3.0.1)\n", + "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.6/dist-packages (from google->ray[rllib]->-r requirements.txt (line 11)) (4.6.3)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas; extra == \"rllib\"->ray[rllib]->-r requirements.txt (line 11)) (2.8.1)\n", + "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas; extra == \"rllib\"->ray[rllib]->-r requirements.txt (line 11)) (2018.9)\n", + "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (0.4.1)\n", + "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (2.23.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (3.2.1)\n", + "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.6.0.post3)\n", + "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.0.1)\n", + "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.7.2)\n", + "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.6/dist-packages (from yarl<2.0,>=1.0->aiohttp->ray[rllib]->-r requirements.txt (line 11)) (2.9)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (2020.4.5.1)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (1.24.3)\n", + "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (4.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (0.2.8)\n", + "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (3.1.1)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (3.1.0)\n", + "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow->-r requirements.txt (line 12)) (0.4.8)\n", + "\u001b[31mERROR: Could not find a version that satisfies the requirement ran (from versions: none)\u001b[0m\n", + "\u001b[31mERROR: No matching distribution found for ran\u001b[0m\n", + "Requirement already satisfied: gputil in /usr/local/lib/python3.6/dist-packages (1.4.0)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (5.4.8)\n", + "Requirement already satisfied: humanize in /usr/local/lib/python3.6/dist-packages (0.5.1)\n", + "Gen RAM Free: 12.7 GB | Proc size: 160.3 MB\n", + "GPU RAM Free: 16280MB | Used: 0MB | Util 0% | Total 16280MB\n" + ], + "name": "stdout" + } + ] }, - "colab_type": "code", - "id": "k1cp_IzAwg5n", - "outputId": "a1bfe319-afbf-462f-f027-cce5e18d1906" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "gen\tnevals\tavg \tstd \tmin \tmax \n", - "0 \t6 \t-3.05556\t2.42559\t-5.41667\t1.66667\n", - "1 \t3 \t-1.25 \t2.08333\t-3.33333\t1.66667\n", - "2 \t5 \t0.138889\t2.82256\t-4.16667\t3.33333\n", - "3 \t6 \t2.22222 \t0.785674\t1.66667 \t3.33333\n", - "4 \t2 \t3.05556 \t0.62113 \t1.66667 \t3.33333\n", - "5 \t4 \t3.33333 \t0 \t3.33333 \t3.33333\n", - "6 \t3 \t3.33333 \t0 \t3.33333 \t3.33333\n", - "Individual('d', [0.0, 0.0, 1.0, 1.0, 1.0, 1.0])\n", - "---test_reward: 6.666666666666665\n" - ] + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "ojEPVGZmvTAe", + "colab": {} + }, + "source": [ + "import array\n", + "import random\n", + "\n", + "import numpy as np\n", + "from typing import Dict\n", + "\n", + "from deap import algorithms\n", + "from deap import base\n", + "from deap import creator\n", + "from deap import tools\n", + "import gym\n", + "\n", + "from environments import SimpleEnv\n", + "import importlib\n", + "from environments import SimpleEnv\n", + "importlib.reload(SimpleEnv)\n", + "\n", + "from ray.rllib.agents import ppo\n", + "from ray import tune\n", + "from ray.rllib.policy.policy import Policy\n", + "from ray.rllib.env import BaseEnv\n", + "from ray.rllib.env.multi_agent_env import MultiAgentEnv\n", + "from ray.rllib.evaluation import MultiAgentEpisode, RolloutWorker\n", + "from ray.rllib.agents.callbacks import DefaultCallbacks\n", + "import ray\n", + "\n", + "import time\n", + "start = time.process_time()\n", + "\n", + "from ray import tune\n", + "\n", + "SimpleEnv = SimpleEnv.SimpleEnv" + ], + "execution_count": 0, + "outputs": [] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/wichersn/.local/lib/python3.7/site-packages/deap/creator.py:141: RuntimeWarning: A class named 'FitnessMax' has already been created and it will be overwritten. Consider deleting previous creation of that class or rename it.\n", - " RuntimeWarning)\n", - "/usr/local/google/home/wichersn/.local/lib/python3.7/site-packages/deap/creator.py:141: RuntimeWarning: A class named 'Individual' has already been created and it will be overwritten. Consider deleting previous creation of that class or rename it.\n", - " RuntimeWarning)\n" - ] - } - ], - "source": [ - "#function that tunes the reward function\n", - "def train_reward_function(config):\n", - " creator.create('FitnessMax', base.Fitness, weights=(1.0, ))\n", - " creator.create('Individual', array.array, typecode='d',\n", - " fitness=creator.FitnessMax)\n", - "\n", - " toolbox = base.Toolbox()\n", - "\n", - " toolbox.register('attr', random.randint, 0, 1)\n", - " toolbox.register('individual', tools.initRepeat, creator.Individual,\n", - " toolbox.attr, n_agents * n_var)\n", - " toolbox.register('population', tools.initRepeat, list,\n", - " toolbox.individual)\n", - " # some setup for Deap\n", - " toolbox.register('evaluate', evaluate_individual)\n", - " toolbox.register('mate', tools.cxTwoPoint)\n", - " toolbox.register('mutate', tools.mutFlipBit, indpb=0.05) # add more noise\n", - " toolbox.register('select', tools.selTournament, tournsize=3)\n", - "\n", - " # pop is a list individual reward function's weights \n", - " pop = toolbox.population(n=config[\"POPULATION_SIZE\"])\n", - " # the 10 best individual reward function (could even befround the first round)\n", - " hof = tools.HallOfFame(10)\n", - "\n", - " stats = tools.Statistics(lambda ind: ind.fitness.values)\n", - " stats.register('avg', np.mean)\n", - " stats.register('std', np.std)\n", - " stats.register('min', np.min)\n", - " stats.register('max', np.max)\n", - "\n", - " \n", - " pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=config[\"N_GEN\"], \n", - " stats=stats, halloffame=hof, verbose=True)\n", - " best_individual = hof[0]\n", - " print(best_individual)\n", - " test_reward = evaluate_individual_env(best_individual, test_env[0], test_env[1])\n", - " print(\"---test_reward:\", test_reward)\n", - " if TUNING_EVOLUTION:\n", - " tune.track.log(mean_accuracy=test_reward)\n", - "\n", - "if TUNING_EVOLUTION:\n", - " #configurations\n", - " analysis = tune.run(\n", - " train_reward_function, config={\n", - " \"N_GEN\": tune.grid_search(list(range(1, 20, 5))), \n", - " \"POPULATION_SIZE\": tune.grid_search(list(range(1, 20, 5)))\n", - " })\n", - " #print best config\n", - " print(\"Best config: \", analysis.get_best_config(metric=\"mean_accuracy\"))\n", - "else:\n", - " train_reward_function(config_evolution)\n", - "\n", - "\n", - "# print ('pop', pop)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "wvaDdyNmEQ-o", + "colab": {} + }, + "source": [ + "# some settings that we can tweak:\n", + "CONFIG_TRAINING={\n", + " \"MAX_STEP_COUNT\": 1, # number of steps in each round of DEAP evolution \n", + " \"POPULATION_SIZE\": 1, # population in each round of DEAP evolution \n", + " \"N_GEN\": 1, # number of rounds of DEAP evolution This has the most effect\n", + " \"N_RL_TRANING\" : 1, # number of rounds of RL training\n", + " \"TRAIN_BATCH_SIZE\" : 1, #Batch size for RL training\n", + " \"LEARNING_RATE\" : 0.001 #learning rate for RL training\n", + " }\n", + "\n", + "TUNNING = \"RL\" # \"RL\" / \"EV\" / \"OFF\". \n", + "\n", + "if TUNNING == \"RL\":\n", + " EVAL_METHOD = \"RL\" # RL (train reward function and agent) or OPTIMAL (only train reward function)\n", + " # EVAL_METHOD = \"OPTIMAL\" #-----------need to delete\n", + "elif TUNNING == \"EV\":\n", + " EVAL_METHOD = \"OPTIMAL\" # RL (train reward function and agent) or OPTIMAL (only train reward function)\n", + "elif TUNNING == \"OFF\":\n", + " EVAL_METHOD = \"RL\" # RL (train reward function and agent) or OPTIMAL (only train reward function)\n", + "\n", + "# List of indivual reward functions used for tunning parameters for evolution\n", + "INDIVIDUAL = [[0.9298461960519508, 0.7449587149229808, 0.4628576259710946, -0.282921307700329, -0.7019321146761455, 0.618488821337605]]\n" + ], + "execution_count": 0, + "outputs": [] }, - "colab_type": "code", - "id": "ztw5G6VWERA6", - "outputId": "0a24bf3d-97c5-4f03-ef19-f585b3b60ad5", - "scrolled": true - }, - "outputs": [], - "source": [ - "print(evaluate_individual([0.9298461960519508, 0.7449587149229808, 0.4628576259710946, -0.282921307700329, -0.7019321146761455, 0.618488821337605]))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 + { + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "_cnG-FTpEQ-1", + "outputId": "f1c0c835-fa19-4365-fb3a-a59751cd3654", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 72 + } + }, + "source": [ + "ray.shutdown()\n", + "ray.init(webui_host='127.0.0.1') #Use this line to fix the error indicated in: https://github.com/ray-project/ray/issues/7084#issuecomment-584505519\n", + "n_agents = 3\n", + "n_var = 2\n", + "training_envs = [\n", + "(SimpleEnv, {\n", + " 'n_agents': n_agents,\n", + " 'n_vars': n_var,\n", + " 'true_reward_weights': [1, 0],\n", + " 'max_step_count': CONFIG_TRAINING[\"MAX_STEP_COUNT\"],\n", + "}),\n", + "(SimpleEnv, {\n", + " 'n_agents': n_agents,\n", + " 'n_vars': n_var,\n", + " 'true_reward_weights': [0, 1],\n", + " 'max_step_count': CONFIG_TRAINING[\"MAX_STEP_COUNT\"],\n", + "})]\n", + "test_env = (SimpleEnv, {\n", + " 'n_agents': n_agents,\n", + " 'n_vars': n_var,\n", + " 'true_reward_weights': [1, 1],\n", + " 'max_step_count': CONFIG_TRAINING[\"MAX_STEP_COUNT\"],\n", + "})\n", + "\n" + ], + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "text": [ + "2020-05-09 23:16:06,166\tINFO resource_spec.py:212 -- Starting Ray with 7.18 GiB memory available for workers and up to 3.59 GiB for objects. You can adjust these settings with ray.init(memory=, object_store_memory=).\n", + "2020-05-09 23:16:06,585\tINFO services.py:1170 -- View the Ray dashboard at \u001b[1m\u001b[32m127.0.0.1:8265\u001b[39m\u001b[22m\n" + ], + "name": "stderr" + } + ] }, - "colab_type": "code", - "id": "GS2i0F_qERBb", - "outputId": "8962cd7e-aa9f-4d33-ce24-115e0d3e8a28" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "(-1.6666666666666667,)\n" - ] - } - ], - "source": [ - "print(evaluate_individual([1, 0, -1, 0, -1, 0])) #Worst reward, selfish agent" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "yn9RzymlEQ_G", + "colab": {} + }, + "source": [ + "# Directly generate \"optimal action\" suggested by the reward function created by Evolution\n", + "# reward_weights is created from env(config=env_config)\n", + "def get_optimal_action(reward_weights, env_config):\n", + " n_vars = env_config[\"n_vars\"]\n", + " max_act = 5 \n", + " \n", + " reward_scale_factor = np.array([2]*n_vars + [1]*(len(reward_weights)-n_vars))\n", + " action = reward_weights * reward_scale_factor\n", + " action = action / np.max(action) * max_act\n", + " \n", + " action = np.reshape(action, [n_agents, n_vars])\n", + " return action\n", + "\n", + "# If we do not want to optimize RL but just want to optimize reward function using Evolution function\n", + "# then here we just do the \"optimal action\" suggested by the generated reward function\n", + "def evaluate_individual_env_optimal_act(individual, environment_fn, env_config, config_trianing):\n", + " env_config['reward_weights'] = np.array([individual for i in range(n_agents)])\n", + " env = environment_fn(config=env_config)\n", + " \n", + " ave_true_rewards = 0\n", + " obs = env.reset()\n", + " ave_reward = 0\n", + " for _ in range(env_config[\"max_step_count\"]):\n", + " actions = {i: get_optimal_action(env.reward_weights[i], env_config) for i in range(n_agents)}\n", + " obs, reward, _, _ = env.step(actions)\n", + " reward = np.array([reward[i] for i in range(len(reward))])\n", + " ave_reward += reward\n", + " ave_true_rewards += env.last_true_reward\n", + " \n", + " ave_true_rewards /= env_config[\"max_step_count\"]\n", + " ave_reward /= env_config[\"max_step_count\"]\n", + " \n", + " return np.mean(ave_true_rewards)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "oP1G3iuuolJi", + "colab": {} + }, + "source": [ + "# train RL agent with given setup\n", + "def evaluate_individual_env_rl(individual, environment, env_config, config_trianing):\n", + " \"\"\"Runs the environment. All agents have the same policy.\n", + " It returns the total true reward as the fitness.\n", + " \"\"\"\n", + " # need to import many libaraies again, because otherwise tunning funciton will not find the libaraies. \n", + " import array\n", + " import random\n", + "\n", + " import numpy as np\n", + " from typing import Dict\n", + "\n", + " from deap import algorithms\n", + " from deap import base\n", + " from deap import creator\n", + " from deap import tools\n", + " import gym\n", + "\n", + " from environments import SimpleEnv\n", + " import importlib\n", + " from environments import SimpleEnv\n", + " importlib.reload(SimpleEnv)\n", + "\n", + " from ray.rllib.agents import ppo\n", + " from ray import tune\n", + " from ray.rllib.policy.policy import Policy\n", + " from ray.rllib.env import BaseEnv\n", + " from ray.rllib.env.multi_agent_env import MultiAgentEnv\n", + " from ray.rllib.evaluation import MultiAgentEpisode, RolloutWorker\n", + " from ray.rllib.agents.callbacks import DefaultCallbacks\n", + " import ray\n", + "\n", + " import time\n", + " start = time.process_time()\n", + "\n", + " from ray import tune\n", + "\n", + " SimpleEnv = SimpleEnv.SimpleEnv\n", + " \n", + " import time\n", + " start = time.process_time()\n", + " # your code here \n", + " print(\"Time Spent 1--- = \", (time.process_time() - start)/60, \" minutes\") \n", + " \n", + " #Select random individuals from pop and create the reward weights\n", + " pop = np.array([individual for i in range(n_agents)])\n", + " reward_weights = pop\n", + " env_config['reward_weights'] = reward_weights\n", + " #env is only to get action space and observation space\n", + " env = environment(config=env_config)\n", + " class MyCallbacks(DefaultCallbacks):\n", + " #Callback functions to keep track of true reward while training\n", + " def on_episode_start(self, worker: RolloutWorker, base_env: BaseEnv,\n", + " policies: Dict[str, Policy],\n", + " episode: MultiAgentEpisode, **kwargs):\n", + " episode.user_data[\"true_rewards\"] = np.zeros(n_agents)\n", + "\n", + " def on_episode_step(self, worker: RolloutWorker, base_env: BaseEnv,\n", + " episode: MultiAgentEpisode, **kwargs):\n", + " env = base_env\n", + " true_reward = env.env_states[0].env.last_true_reward\n", + " episode.user_data[\"true_rewards\"] += true_reward\n", + "\n", + " def on_episode_end(self, worker: RolloutWorker, base_env: BaseEnv,\n", + " policies: Dict[str, Policy], episode: MultiAgentEpisode,\n", + " **kwargs):\n", + " true_reward = episode.user_data[\"true_rewards\"]\n", + " for i, r in enumerate(true_reward):\n", + " episode.custom_metrics[\"true_reward_agent_\" + str(i)] = r\n", + " \n", + " print(\"Time Spent 2--- = \", (time.process_time() - start)/60, \" minutes\") \n", + " \n", + " \n", + " # settings for the RL agent trainer \n", + " config={\n", + " \"train_batch_size\": config_trianing[\"TRAIN_BATCH_SIZE\"],\n", + " \"lr\": config_trianing[\"LEARNING_RATE\"],\n", + " \"sgd_minibatch_size\": config_trianing[\"TRAIN_BATCH_SIZE\"],\n", + " \"multiagent\": {\n", + " \"policies\": {\n", + " },\n", + " \"policy_mapping_fn\": #all agents share a policy\n", + " lambda agent_id:\n", + " 'agent'\n", + " },\n", + " \"model\": {\"fcnet_hiddens\": []},\n", + " 'env_config': env_config,\n", + " \"callbacks\": MyCallbacks,\n", + " }\n", + " config['multiagent']['policies']['agent'] = (None, env.observation_space, env.action_space, {})\n", + " metrics = None\n", + "\n", + "\n", + " print(\"Time Spent 3--- = \", (time.process_time() - start)/60, \" minutes\") \n", + " \n", + " counter = 0 # ------ need to take this out later. use counter to break.\n", + " while True:\n", + " counter += 1 # ------ need to take this out later. use counter to break.\n", + " trainer = ppo.PPOTrainer(env=environment, config=config)\n", + " true_reward_mean = 0\n", + " print(\"Time Spent 4--- = \", (time.process_time() - start)/60, \" minutes, counter = \", counter) \n", + " \n", + " for i in range(config_trianing[\"N_RL_TRANING\"]):\n", + " true_reward_mean = 0\n", + " #Train the RL agent\n", + " print(\"Time Spent 5--(right before trainig)-- = \", (time.process_time() - start)/60, \" minutes, counter = \", counter)\n", + " # import pdb; pdb.set_trace()\n", + " print(\"config_trianing = \", config_trianing, \"individual=\", individual)\n", + "\n", + " metrics = trainer.train() # distributed training step\n", + " print(\"episode_reward_mean\", metrics[\"episode_reward_mean\"])\n", + " print(\"Time Spent 6--- = \", (time.process_time() - start)/60, \" minutes, counter = \", counter) \n", + " # ------ need to take this out later. use counter to break.\n", + " if counter >= 1:\n", + " break\n", + "\n", + " if metrics[\"episode_reward_mean\"] < 0 and i > 2:\n", + " break\n", + " #Train agent until it does well\n", + " if metrics[\"episode_reward_mean\"] > 0:\n", + " break\n", + " # ------ need to take this out later. use counter to break.\n", + " if counter >= 1:\n", + " break\n", + " \n", + " print(\"Time Spent 7--- = \", (time.process_time() - start)/60, \" minutes\") \n", + " \n", + " print(\"episode_reward_mean\", metrics[\"episode_reward_mean\"])\n", + " for i in range(n_agents):\n", + " true_reward_mean += metrics['custom_metrics']['true_reward_agent_' + str(i) + '_mean']\n", + " true_reward_mean /= n_agents\n", + " print('Evaluated', individual, 'Fitness', true_reward_mean)\n", + "\n", + " print(\"Time Spent 8--- = \", (time.process_time() - start)/60, \" minutes\") \n", + " \n", + " return true_reward_mean" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "dy4idwUWEQ_a", + "colab": {} + }, + "source": [ + "if EVAL_METHOD == \"RL\":\n", + " evaluate_individual_env = evaluate_individual_env_rl\n", + "else:\n", + " evaluate_individual_env = evaluate_individual_env_optimal_act\n", + " \n", + "def evaluate_individual(individual, config_trianing = CONFIG_TRAINING):\n", + " \"\"\"Runs all environments. \n", + " returns the average true reward over all environments as the fitness.\n", + " \"\"\"\n", + " \n", + " all_fitness = []\n", + " for env, config in training_envs:\n", + " all_fitness.append(evaluate_individual_env(individual, env, config, config_trianing))\n", + " fitness_comb = np.min(all_fitness)\n", + " return (fitness_comb, )" + ], + "execution_count": 0, + "outputs": [] }, - "colab_type": "code", - "id": "PiWsv5EGERCQ", - "outputId": "f86e17d0-f612-4e3a-fd87-b9bb60d646b4" - }, - "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Time Spent = 0.006995936300000002 minutes\n" - ] + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "k1cp_IzAwg5n", + "outputId": "4c92e574-4234-41f7-ad50-221fdb3f1624", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 509 + } + }, + "source": [ + "#function that tunes the reward function\n", + "def train_reward_function(config_trianing):\n", + "\n", + " creator.create('FitnessMax', base.Fitness, weights=(1.0, ))\n", + " creator.create('Individual', array.array, typecode='d',\n", + " fitness=creator.FitnessMax)\n", + "\n", + " toolbox = base.Toolbox()\n", + "\n", + " toolbox.register('attr', random.randint, 0, 1)\n", + " toolbox.register('individual', tools.initRepeat, creator.Individual,\n", + " toolbox.attr, n_agents * n_var)\n", + " toolbox.register('population', tools.initRepeat, list,\n", + " toolbox.individual)\n", + " # some setup for Deap\n", + " toolbox.register('evaluate', evaluate_individual)\n", + " toolbox.register('mate', tools.cxTwoPoint)\n", + " toolbox.register('mutate', tools.mutFlipBit, indpb=0.05) # add more noise\n", + " toolbox.register('select', tools.selTournament, tournsize=3)\n", + "\n", + " # pop is a list individual reward function's weights \n", + " pop = toolbox.population(n=config_trianing[\"POPULATION_SIZE\"])\n", + " # the 10 best individual reward function (could even befround the first round)\n", + " hof = tools.HallOfFame(10)\n", + "\n", + " stats = tools.Statistics(lambda ind: ind.fitness.values)\n", + " stats.register('avg', np.mean)\n", + " stats.register('std', np.std)\n", + " stats.register('min', np.min)\n", + " stats.register('max', np.max)\n", + "\n", + "\n", + "\n", + " \n", + " pop, log = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=config_trianing[\"N_GEN\"], \n", + " stats=stats, halloffame=hof, verbose=True)\n", + " best_individual = hof[0]\n", + " print(best_individual)\n", + " test_reward = evaluate_individual_env(best_individual, test_env[0], test_env[1], config_trianing)\n", + " print(\"---test_reward:\", test_reward)\n", + " if TUNNING == \"EV\":\n", + " tune.track.log(mean_accuracy=test_reward) \n", + "\n", + "def tune_rl_training(config_trianing):\n", + " print(\"---rl tunning starts\")\n", + " all_score = []\n", + " for individual in INDIVIDUAL:\n", + " all_score.append(evaluate_individual(individual, config_trianing)[0])\n", + " tune.track.log(mean_accuracy=sum(all_score)/len(all_score))\n", + " \n", + "\n", + "if TUNNING == \"EV\":\n", + " analysis = tune.run(\n", + " train_reward_function, config={\n", + " \"N_GEN\": tune.grid_search(list(range(1, 20, 5))), \n", + " \"POPULATION_SIZE\": tune.grid_search(list(range(1, 20, 5)))\n", + " # \"N_GEN\": tune.grid_search([1]), \n", + " # \"POPULATION_SIZE\": tune.grid_search([1])\n", + " })\n", + " #print best config\n", + " print(\"Best config: \", analysis.get_best_config(metric=\"mean_accuracy\"))\n", + "elif TUNNING == \"RL\":\n", + " analysis = tune.run(\n", + " tune_rl_training, config={\n", + " \"N_RL_TRANING\": tune.grid_search([1]),\n", + " \"TRAIN_BATCH_SIZE\": tune.grid_search([1]),\n", + " \"LEARNING_RATE\": tune.grid_search([0.001])\n", + " })\n", + " print(\"Best config: \", analysis.get_best_config(metric=\"mean_accuracy\"))\n", + "elif TUNNING == \"OFF\":\n", + " train_reward_function(CONFIG_TRAINING)\n", + "\n", + "\n", + "\n", + "# print ('pop', pop)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "2020-05-09 23:16:07,999\tWARNING tune.py:316 -- Tune detects GPUs, but no trials are using GPUs. To enable trials to use GPUs, set tune.run(resources_per_trial={'gpu': 1}...) which allows Tune to expose 1 GPU to each trial. You can also override `Trainable.default_resource_request` if using the Trainable API.\n" + ], + "name": "stderr" + }, + { + "output_type": "display_data", + "data": { + "text/html": [ + "== Status ==
Memory usage on this node: 1.2/12.7 GiB
Using FIFO scheduling algorithm.
Resources requested: 1/2 CPUs, 0/1 GPUs, 0.0/7.18 GiB heap, 0.0/2.44 GiB objects
Result logdir: /root/ray_results/tune_rl_training
Number of trials: 1 (1 RUNNING)
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Trial name status loc LEARNING_RATE N_RL_TRANING TRAIN_BATCH_SIZE
tune_rl_training_102e1_00000RUNNING 0.001 1 1


" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m 2020-05-09 23:16:12,069\tINFO trainable.py:217 -- Getting current IP.\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m 2020-05-09 23:16:12,081\tINFO trainer.py:425 -- Tip: set 'eager': true or the --eager flag to enable TensorFlow eager execution\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m 2020-05-09 23:16:12,104\tINFO trainer.py:584 -- Current log_level is WARN. For more information, set 'log_level': 'INFO' / 'DEBUG' or use the -v and -vv flags.\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m ---rl tunning starts\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m Time Spent 1--- = 1.8499999998589088e-08 minutes\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m Time Spent 2--- = 6.469303333333324e-05 minutes\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m Time Spent 3--- = 6.517558333333599e-05 minutes\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m 2020-05-09 23:16:17,134\tINFO trainable.py:217 -- Getting current IP.\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m Time Spent 4--- = 0.08304348726666667 minutes, counter = 1\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m Time Spent 5--(right before trainig)-- = 0.08304423146666666 minutes, counter = 1\n", + "\u001b[2m\u001b[36m(pid=12446)\u001b[0m config_trianing = {'N_RL_TRANING': 1, 'TRAIN_BATCH_SIZE': 1, 'LEARNING_RATE': 0.001} individual= [0.9298461960519508, 0.7449587149229808, 0.4628576259710946, -0.282921307700329, -0.7019321146761455, 0.618488821337605]\n", + "\u001b[2m\u001b[36m(pid=12447)\u001b[0m /usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n", + "\u001b[2m\u001b[36m(pid=12447)\u001b[0m warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "2020-05-09 23:16:26,752\tWARNING worker.py:1094 -- The actor or task with ID ffffffffffffffffdb1638d70100 is pending and cannot currently be scheduled. It requires {CPU: 1.000000} for execution and {CPU: 1.000000} for placement, but this node only has remaining {node:172.28.0.2: 1.000000}, {memory: 7.177734 GiB}, {GPU: 1.000000}, {object_store_memory: 2.441406 GiB}. In total there are 0 pending tasks and 1 pending actors on this node. This is likely due to all cluster resources being claimed by actors. To resolve the issue, consider creating fewer actors or increase the resources available to this Ray cluster. You can ignore this message if this Ray cluster is expected to auto-scale.\n", + "2020-05-09 23:16:27,139\tINFO (unknown file):0 -- gc.collect() freed 36 refs in 0.29417959200009136 seconds\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\u001b[2m\u001b[36m(pid=12447)\u001b[0m 2020-05-09 23:16:27,044\tINFO (unknown file):0 -- gc.collect() freed 16 refs in 0.19930188500029544 seconds\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "ztw5G6VWERA6", + "scrolled": true, + "colab": {} + }, + "source": [ + "print(evaluate_individual([0.9298461960519508, 0.7449587149229808, 0.4628576259710946, -0.282921307700329, -0.7019321146761455, 0.618488821337605]))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "GS2i0F_qERBb", + "colab": {} + }, + "source": [ + "print(evaluate_individual([1, 0, -1, 0, -1, 0])) #Worst reward, selfish agent" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab_type": "code", + "id": "PiWsv5EGERCQ", + "colab": {} + }, + "source": [ + "print(\"Time Spent = \", (time.process_time() - start)/60, \" minutes\")" + ], + "execution_count": 0, + "outputs": [] } - ], - "source": [ - "print(\"Time Spent = \", (time.process_time() - start)/60, \" minutes\")" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "genetic_with_comments (1).ipynb", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} + ] +} \ No newline at end of file