Dev dagger

.apache-license-checker.yaml

@@ -1,8 +1,6 @@
 ignore:
-  - "?eggs"
-  - "**/__pycache__"
   - "**/.venv"
-  - "**/cogment_*.yaml"
+  - "**/_old"
   - "**/*_pb2*.py"
   - "**/*.pb.go"
   - "**/*_pb*.js"
@@ -12,12 +10,8 @@ ignore:
   - "**/CogSettings.d.ts"
   - "**/CogSettings.js"
   - "**/CogTypes.d.ts"
-  - "**/third_party"
-  - "*/cogment/api"
-  - "**/htmlcov"
-  - "web_client/node_modules"
-  - "web_client/build"
-  - "**/pybullet_driving_env/*"
+  - "**/node_modules"
+  - "**/build"
 license:
-  CopyrightYear: 2021
+  CopyrightYear: 2022
   Author: "AI Redefined Inc. <[email protected]>"

.gitignore

@@ -1,10 +1,10 @@
 # Generated code
-/*/*.proto
-/*/cogment.yaml
-cog_settings.py
-*_pb2.py
-*_pb2_grpc.py
-base_python/cogment_verse/api
+CogSettings.*
+CogTypes.d.ts
+*_pb.d.ts
+*_pb.js
+cogment_verse/web/cogment.yaml
+cogment_verse/web/*.proto
 
 # Python stuffs
 __pycache__/
@@ -24,6 +24,11 @@ node_modules/
 
 # Runtime data
 /data
+/debug
 
 # Cogment
-/.cogment
+/.cogment_verse
+
+# Run outputs generated by Hydra
+outputs
+multirun

.gitlab-ci.yml

@@ -1,44 +1,50 @@
+stages:
+  - lint
+  - test
+
 .base:
   image: python:3.9
   variables:
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-    NPM_CACHE_DIR: "$CI_PROJECT_DIR/.cache/npm"
   before_script:
     - mkdir -p ${PIP_CACHE_DIR}
-    - mkdir -p ${NPM_CACHE_DIR}
-    # Installation instructions from https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions
-    - curl -fsSL https://deb.nodesource.com/setup_14.x | bash -
-    - apt-get update && apt-get install -y software-properties-common && apt-add-repository non-free && apt-get update
-    - apt-get install -y nodejs swig unrar python3-opencv
-    - pip install virtualenv
-    - npm config set cache ${NPM_CACHE_DIR} --global
+    - apt-get update
+    - apt-get install -y swig python3-opencv
+    - python -m venv .venv
+    - source .venv/bin/activate
+    - pip install -r requirements.txt
   cache:
     # pip's cache
     - paths:
         - .cache/pip
         - "**/.venv"
-    # npm's cache
+    # .venv
     - key:
         files:
-          - web_client/package-lock.json
+          - requirements.txt
       paths:
-        - .cache/npm/
-    # atari roms cache
-    - key:
-        files:
-          - run.sh
-          - environment/requirements.txr
-      paths:
-        - environment/.atari_roms
+        - .venv
+
+black:
+  stage: lint
+  extends: .base
+  script:
+    - black --check --diff .
 
-build_and_test:
+pylint:
+  stage: lint
   extends: .base
   script:
-    - ./run.sh build
-    - ./run.sh lint
-    - ./run.sh test
+    - pylint --recursive=y .
 
 apache_licenses_check:
+  stage: lint
   image: registry.gitlab.com/ai-r/apache-license-checker:latest
   script:
     - apache-license-checker
+
+pytest:
+  stage: test
+  extends: .base
+  script:
+    - python -m pytest

README.md

@@ -41,75 +41,88 @@ Cogment verse includes environments from:
 
 ## Getting started
 
-### Setup, Build and Run
-
 1. Clone this repository
-2. Install the following dependencies:
-   - [Python 3.9](https://www.python.org/) or above,
-   - [Node.JS v14](https://nodejs.org/) or above,
-   - `parallel`, on ubuntu it is installable using `apt-get install parallel`, on mac it is available through `brew install parallel`,
-   - `unrar`, on ubuntu it is installable using `apt-get install unrar`, on mac it is available through `brew install unrar`.
-3. `./run.sh build`
-4. `./run.sh services_start`
-5. In a different terminal, start the trials with `./run.sh client start <run-name>`.
-   Different run names can be found in `run_params.yaml`
-6. (Optional) To launch webclient, run `./run.sh web_client_start` in a different
-   terminal. Open http://localhost:8000 to join or visualize trials
+2. Install [Python 3.9](https://www.python.org/)
+3. Depending on your specific machine, you might also need to following dependencies:
+
+   - `swig`, which is required for the Box2d gym environments, it can be installed using `apt-get install swig` on ubuntu or `brew install swig` on macOS
+   - `python3-opencv`, which is required on ubuntu systems, it can be installed using `apt-get install python3-opencv`
+
+4. Create and activate a virtual environment by runnning
+   ```console
+   $ python -m venv .venv
+   $ source .venv/bin/activate
+   ```
+5. Install the python dependencies by running
+   ```console
+   $ pip install -r requirements.txt
+   ```
+6. In another terminal, launch a mlflow server on port 3000 by running
+   ```console
+   $ source .venv/bin/activate
+   $ python -m simple_mlflow
+   ```
+7. Start the default Cogment Verse run using `python -m main`
+8. Open Chrome (other web browser might work but haven't tested) and navigate to http://localhost:8080/
+9. Play the game!
 
-#### Run monitoring
+That's the basic setup for Cogment Verse, you are now ready to train AI agents.
 
-You can monitor ongoing run using [mlflow](https://mlflow.org). By default a local instance of mlflow is started by cogment-verse and is accessible at <http://localhost:3000>.
+### Configuration
 
-#### Human player
+Cogment Verse relies on [hydra](https://hydra.cc) for configuration. This enables easy configuration and composition of configuration directly from yaml files and the command line.
 
-Some of the availabe run involve a human player,
-for example `benchmark_lander_hill` enables a human player
-to momentarily take control of the lunar lander to help the
-AI agents during the training process.
+The configuration files are located in the `config` directory, with defaults defined in `config/config.yaml`.
 
-Then start the run
+Here are a few examples:
 
-```console
-./run.sh client start benchmark_lander_hill
-```
+- Launch a Simple Behavior Cloning run with the [Mountain Car Gym environment](https://www.gymlibrary.ml/environments/classic_control/mountain_car/) (which is the default environment)
+  ```console
+  $ python -m main +experiment=simple_bc/mountain_car
+  ```
+- Launch a Simple Behavior Cloning run with the [Lunar Lander Gym environment](https://www.gymlibrary.ml/environments/box2d/lunar_lander/)
+  ```console
+  $ python -m main +experiment=simple_bc/mountain_car services/environment=lunar_lander
+  ```
+- Launch and play a single trial of the Lunar Lander Gym environment with continuous controls
+  ```console
+  $ python -m main services/environment=lunar_lander_continuous
+  ```
+- Launch an A2C training run with the [Cartpole Gym environment](https://www.gymlibrary.ml/environments/classic_control/cartpole/)
 
-Access the playing interface by launching a webclient with
-`./run.sh web_client_start` and navigating to <http://localhost:8000>
+  ```console
+  $ python -m main +experiment=simple_a2c/cartpole
+  ```
 
-#### **Play**
+  This one is completely _headless_ (training doens't involve interaction with a human player). It will take a little while to run, you can monitor the progress using mlflow at <http://localhost:3000>
 
-The `play` run implementation can be used to have any actor play in any environment. 3 example run parameters are provided:
+- Launch an DQN self training run with the [Connect Four Petting Zoo environment](https://www.pettingzoo.ml/classic/connect_four)
 
-**`headless_play`** instanciates any agents and start a number of trials.
+  ```console
+  $ python -m main +experiment=simple_dqn/connect_four
+  ```
 
-```console
-./run.sh client start headless_play
-```
+  The same experiment can be launched with a ratio of human-in-the-loop training trials (that are playable on in the web client)
 
-**`observe`** instanciates any agents and start a number of trials with a human observer through the webclient.
+  ```console
+  $ python -m main +experiment=simple_dqn/connect_four +run.hill_training_trials_ratio=0.05
+  ```
 
-```console
-./run.sh client start observe
-```
+- Launch a [DAGGER](https://arxiv.org/abs/1011.0686) imitation learning algorithm by first training the expert using the simple_a2c method
 
-**`play`** instanciates let a human player play in a supported environment.
+  ```console
+  $ python -m main +experiment=simple_a2c/cartpole
+  ```
 
-```console
-./run.sh client start play
-```
+  Then modify the teacher_model_id field of the config/experiment/cartpole.yaml file with the model_id of the trained simple_a2c method. Next, run the DAGGER algorithm using
 
-They can be inspected and adapted to your needs in `run_params.yaml`:
+  ```console
+  $ python -m main +experiment=dagger/cartpole
+  ```
 
 ## List of publications and submissions using Cogment and/or Cogment Verse
 
 - Analyzing and Overcoming Degradation in Warm-Start Off-Policy Reinforcement Learning [code](https://github.com/benwex93/cogment-verse)
 - Multi-Teacher Curriculum Design for Sparse Reward Environments [code](https://github.com/kharyal/cogment-verse/)
 
 (please open a pull request to add missing entries)
-
-## Acknowledgements
-
-The subdirectories `/tf_agents/cogment_verse_tf_agents/third_party` and `/torch_agents/cogment_verse_torch_agents/third_party` contains code from third party sources
-
-- `hive`: Taken from the [Hive library](https://github.com/chandar-lab/RLHive)
-- `td3`: Taken form the [authors' implementation](https://github.com/sfujim/TD3)

.env → _old/.env

@@ -7,6 +7,7 @@ COGMENT_VERSE_MODEL_REGISTRY_PORT=9002
 COGMENT_VERSE_TORCH_AGENTS_PORT=9003
 COGMENT_VERSE_TF_AGENTS_PORT=9004
 COGMENT_VERSE_ENVIRONMENT_PORT=9005
+COGMENT_VERSE_PRETRIAL_HOOK_PORT=9006
 
 ## Prometheus metrics server ports
 COGMENT_VERSE_TORCH_AGENTS_PROMETHEUS_PORT=9500

environment/tests/test_atari.py → _old/environment/tests/test_atari.py

@@ -14,7 +14,7 @@
 
 import pytest
 from cogment_verse_environment.utils.serialization_helpers import deserialize_img, deserialize_np_array
-from data_pb2 import AgentAction, EnvironmentConfig
+from data_pb2 import PlayerAction, EnvironmentConfig
 from mock_environment_session import ActorInfo
 
 # pylint doesn't like test fixtures

environment/tests/test_pettingzoo.py → _old/environment/tests/test_pettingzoo.py

@@ -15,7 +15,7 @@
 import numpy as np
 import pytest
 from cogment_verse_environment.utils.serialization_helpers import deserialize_img, deserialize_np_array
-from data_pb2 import AgentAction, EnvironmentConfig
+from data_pb2 import PlayerAction, EnvironmentConfig
 from mock_environment_session import ActorInfo
 
 # pylint doesn't like test fixtures

environment/tests/test_procgen.py → _old/environment/tests/test_procgen.py

@@ -16,7 +16,7 @@
 import pytest
 from cogment_verse_environment.procgen_env import ENV_NAMES, ProcGenEnv
 from cogment_verse_environment.utils.serialization_helpers import deserialize_img, deserialize_np_array
-from data_pb2 import AgentAction, EnvironmentConfig
+from data_pb2 import PlayerAction, EnvironmentConfig
 from mock_environment_session import ActorInfo
 
 # pylint doesn't like test fixtures

run.sh → _old/run.sh

@@ -236,7 +236,7 @@ function mlflow_start() {
 function web_client_build() {
   _load_dot_env
   export PORT="${COGMENT_VERSE_WEBCLIENT_PORT}"
-  export REACT_APP_ORCHESTRATOR_HTTP_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}"
+  export REACT_APP_ORCHESTRATOR_WEB_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}"
   cp "${ROOT_DIR}/data.proto" "${ROOT_DIR}/cogment.yaml" "${ROOT_DIR}/web_client"
   cd "${ROOT_DIR}/web_client"
   npm install --no-audit
@@ -253,7 +253,7 @@ function web_client_start() {
 function web_client_start_dev() {
   _load_dot_env
   export PORT="${COGMENT_VERSE_WEBCLIENT_PORT}"
-  export REACT_APP_ORCHESTRATOR_HTTP_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}"
+  export REACT_APP_ORCHESTRATOR_WEB_ENDPOINT="${COGMENT_VERSE_ORCHESTRATOR_HTTP_ENDPOINT}"
   cd "${ROOT_DIR}/web_client"
   npm run dev
 }

tf_agents/cogment_verse_tf_agents/wrapper.py → _old/tf_agents/cogment_verse_tf_agents/wrapper.py

@@ -14,7 +14,7 @@
 
 import cv2
 import numpy as np
-from data_pb2 import AgentAction, ContinuousAction
+from data_pb2 import PlayerAction, ContinuousAction
 
 # TODO directly use tf tensors
 
@@ -53,11 +53,11 @@ def cog_action_from_tf_action(action):
     if dtype in (int, np.int32, np.int64):
         field = "discrete_action"
         kwargs = {field: action}
-        return AgentAction(**kwargs)
+        return PlayerAction(**kwargs)
 
     # else
     # pylint: disable=no-member
-    agent_action = AgentAction(continuous_action=ContinuousAction())
+    agent_action = PlayerAction(continuous_action=ContinuousAction())
     action = np.squeeze(action)
     if action.shape == ():
         agent_action.continuous_action.data.append(action)

torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py → _old/torch_agents/cogment_verse_torch_agents/hf_sb3/sb3_adapter.py

@@ -21,7 +21,7 @@
 from cogment_verse import AgentAdapter
 
 from cogment_verse_torch_agents.utils.tensors import tensor_from_cog_obs
-from data_pb2 import AgentAction
+from data_pb2 import PlayerAction
 
 from huggingface_sb3 import load_from_hub
 from stable_baselines3 import PPO

torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py → _old/torch_agents/cogment_verse_torch_agents/hive_adapter/hive_agent_adapter.py

@@ -24,7 +24,11 @@
 from cogment_verse_torch_agents.third_party.hive.dqn import DQNAgent
 from cogment_verse_torch_agents.third_party.hive.rainbow import RainbowDQNAgent
 from cogment_verse_torch_agents.third_party.td3.td3 import TD3Agent
-from cogment_verse_torch_agents.wrapper import cog_action_from_torch_action, format_legal_moves, torch_obs_from_cog_obs
+from cogment_verse_torch_agents.wrapper import (
+    cog_action_from_torch_action,
+    format_legal_moves,
+    torch_obs_from_cog_obs,
+)
 from data_pb2 import RunConfig
 from prometheus_client import Summary
 
@@ -76,7 +80,15 @@ def _create(self, model_id, impl_name, environment_specs, **kwargs):
 
         return model, model_user_data
 
-    def _load(self, model_id, version_number, model_user_data, version_user_data, model_data_f, **kwargs):
+    def _load(
+        self,
+        model_id,
+        version_number,
+        model_user_data,
+        version_user_data,
+        model_data_f,
+        **kwargs,
+    ):
         model = self.agent_class_from_impl_name(model_user_data["agent_implementation"])(
             id=model_id,
             obs_dim=int(model_user_data["num_input"]),
@@ -143,4 +155,10 @@ async def impl(actor_session):
         return {impl_name: (create_actor_impl(impl_name), ["agent"]) for impl_name in self._agent_classes}
 
     def _create_run_implementations(self):
-        return {"cogment_verse_run_impl": (sample_producer, create_training_run(self), RunConfig())}
+        return {
+            "cogment_verse_run_impl": (
+                sample_producer,
+                create_training_run(self),
+                RunConfig(),
+            )
+        }