WIP: commit pending changes

.github/workflows/python-publish.yml

@@ -18,11 +18,11 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
-        python-version: '3.6'
+        python-version: '3.7'
     - name: Install dependencies and test
       run: |
-        python -m pip install --upgrade pip
-        pip install poetry
+        python -m pip install pip==22.2.2 setuptools==65.3.0 wheel==0.37.1
+        pip install poetry==1.1.15
         poetry install
         poetry run coverage run -m pytest
     - name: Generate Documentation

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "ml_fairness_gym"]
-	path = mlfairnessgym
-	url = https://github.com/luksfarris/ml-fairness-gym.git
-	branch = master

.pre-commit-config.yaml

@@ -1,39 +1,48 @@
 repos:
--   repo: https://github.com/ambv/black
+  - repo: https://github.com/ambv/black
     rev: 21.9b0
     hooks:
-    - id: black
-      language_version: python3.6
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+      - id: black
+        language_version: python3.6
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
     hooks:
-    - id: flake8
-      additional_dependencies: ['flake8-bugbear', mccabe, 'pep8-naming']
-      args: ['--max-complexity=5']
--   repo: https://github.com/pycqa/isort
+      - id: flake8
+        additional_dependencies: ["flake8-bugbear", mccabe, "pep8-naming"]
+        args: ["--max-complexity=5"]
+  - repo: https://github.com/pycqa/isort
     rev: 5.8.0
     hooks:
       - id: isort
         name: isort (python)
-        args: ['--profile=black']
--   repo: https://github.com/PyCQA/pydocstyle
+        args: ["--profile=black"]
+  - repo: https://github.com/PyCQA/pydocstyle
     rev: 6.1.1
     hooks:
       - id: pydocstyle
-        args: ['--ignore=D100,D104,D203,D205,D209,D213,D400,D415',
-            '--match=deeprecsys/rl/.*']
+        args:
+          [
+            "--ignore=D100,D104,D203,D205,D209,D213,D400,D415",
+            "--match=deeprecsys/rl/.*",
+          ]
         additional_dependencies: [toml]
--   repo: https://github.com/pre-commit/mirrors-mypy
+  - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.930
     hooks:
       - id: mypy
-        args: [--no-strict-optional, --ignore-missing-imports, --disallow-untyped-calls, --disallow-untyped-defs, --disallow-incomplete-defs]
+        args:
+          [
+            --no-strict-optional,
+            --ignore-missing-imports,
+            --disallow-untyped-calls,
+            --disallow-untyped-defs,
+            --disallow-incomplete-defs,
+          ]
         additional_dependencies: [types-attrs]
--   repo: https://github.com/PyCQA/bandit
-    rev: '1.6.2'
+  - repo: https://github.com/PyCQA/bandit
+    rev: "1.6.2"
     hooks:
-    - id: bandit
-      language_version: python3
-      exclude: ^deeprecsys/tests/
-      args: ['-q', '--ignore-nosec', '-s', 'B608,B303']
-
+      - id: bandit
+        language_version: python3
+        exclude: ^deeprecsys/tests/
+        args: ["-q", "--ignore-nosec", "-s", "B608,B303,B101"]

README.md

@@ -7,3 +7,14 @@
 For instructions on setting up, check out the [Getting Started Wiki Page](https://github.com/luksfarris/pydeeprecsys/wiki/Getting-Started).
 
 This project is a WIP. It is provided "as is" under the MIT license.
+
+
+### Dev Setup
+
+Similar to what's in the CI workflow:
+```
+python3 -m venv venv
+python -m pip install pip==22.2.2 setuptools==65.3.0 wheel==0.37.1
+pip install poetry==1.1.15
+poetry install
+```

deeprecsys/movielens_fairness_env.py

@@ -8,13 +8,12 @@
 from gym import Env, Space
 from gym.envs.registration import register
 from gym.spaces import Box, Discrete
+from mlfairnessgym.environments.recommenders import movie_lens_dynamic as movie_lens
+from mlfairnessgym.environments.recommenders import movie_lens_utils, recsim_samplers
 from numpy.core.multiarray import ndarray
 from recsim.simulator import recsim_gym
 from recsim.simulator.recsim_gym import RecSimGymEnv
 
-from mlfairnessgym.environments.recommenders import movie_lens_dynamic as movie_lens
-from mlfairnessgym.environments.recommenders import movie_lens_utils, recsim_samplers
-
 _env_specs = {
     "id": "MovieLensFairness-v0",
     "entry_point": "deeprecsys.movielens_fairness_env:MovieLensFairness",

deeprecsys/neural_networks/binary_classifier.py

@@ -0,0 +1,31 @@
+from torch import Tensor
+from torch.nn import BCELoss, Linear, ReLU, Sequential, Sigmoid
+from torch.optim import Adam
+
+from deeprecsys.neural_networks.base_network import BaseNetwork
+
+
+class BinaryClassifier(BaseNetwork):
+    def __init__(self, input_shape: int, learning_rate: float = 0.0025):
+        super().__init__()
+        layers = [
+            Linear(in_features=input_shape, out_features=256, bias=True),
+            ReLU(),
+            Linear(in_features=256, out_features=64, bias=True),
+            ReLU(),
+            Linear(in_features=64, out_features=16, bias=True),
+            ReLU(),
+            Linear(in_features=16, out_features=1, bias=True),
+            Sigmoid(),
+        ]
+        self.model = Sequential(*layers)
+        self.optimizer = Adam(self.parameters(), lr=learning_rate)
+        self.loss_function = BCELoss()
+        if self.device == "cuda":
+            self.model.cuda()
+
+    def update(self, features_batch: Tensor, targets_batch: Tensor) -> float:
+        predicted_selection = self.forward(features_batch)
+        loss = self.loss_function(predicted_selection, targets_batch)  # .reshape(-1,1))
+        self.run_backpropagation(loss)
+        return loss.item()

deeprecsys/rl/neural_networks/deep_q_network.py → deeprecsys/neural_networks/deep_q_network.py

@@ -5,8 +5,8 @@
 from torch.nn import Linear, Module, MSELoss, ReLU, Sequential
 from torch.optim import Adam
 
+from deeprecsys.neural_networks.base_network import BaseNetwork
 from deeprecsys.rl.learning_statistics import LearningStatistics
-from deeprecsys.rl.neural_networks.base_network import BaseNetwork
 
 
 def sequential_architecture(layers: List[int], bias: bool = True) -> Module:

deeprecsys/rl/neural_networks/dueling.py → deeprecsys/neural_networks/dueling.py

@@ -5,12 +5,12 @@
 from torch.nn import Linear, Module, ReLU, Sequential, functional
 from torch.optim import Adam
 
+from deeprecsys.neural_networks.base_network import BaseNetwork
+from deeprecsys.neural_networks.noisy_layer import NoisyLayer
 from deeprecsys.rl.experience_replay.priority_replay_buffer import (
     PrioritizedExperienceReplayBuffer,
 )
 from deeprecsys.rl.learning_statistics import LearningStatistics
-from deeprecsys.rl.neural_networks.base_network import BaseNetwork
-from deeprecsys.rl.neural_networks.noisy_layer import NoisyLayer
 
 
 class DuelingDDQN(BaseNetwork):

deeprecsys/rl/neural_networks/gaussian_actor.py → deeprecsys/neural_networks/gaussian_actor.py

@@ -6,9 +6,9 @@
 from torch.distributions import Normal
 from torch.optim import Adam
 
-from deeprecsys.rl.neural_networks.base_network import BaseNetwork
-from deeprecsys.rl.neural_networks.deep_q_network import sequential_architecture
-from deeprecsys.rl.neural_networks.q_value_estimator import TwinnedQValueEstimator
+from deeprecsys.neural_networks.base_network import BaseNetwork
+from deeprecsys.neural_networks.deep_q_network import sequential_architecture
+from deeprecsys.neural_networks.q_value_estimator import TwinnedQValueEstimator
 
 LOG_STD_MAX = 2
 LOG_STD_MIN = -20

deeprecsys/rl/neural_networks/policy_estimator.py → deeprecsys/neural_networks/policy_estimator.py

@@ -7,7 +7,7 @@
 from torch.nn import Linear, Sequential, Softmax, Tanh
 from torch.optim import Adam
 
-from deeprecsys.rl.neural_networks.base_network import BaseNetwork
+from deeprecsys.neural_networks.base_network import BaseNetwork
 
 
 class PolicyEstimator(BaseNetwork):

deeprecsys/rl/neural_networks/q_value_estimator.py → deeprecsys/neural_networks/q_value_estimator.py

@@ -5,8 +5,8 @@
 from torch.nn import Module
 from torch.optim import Adam
 
-from deeprecsys.rl.neural_networks.base_network import BaseNetwork
-from deeprecsys.rl.neural_networks.deep_q_network import sequential_architecture
+from deeprecsys.neural_networks.base_network import BaseNetwork
+from deeprecsys.neural_networks.deep_q_network import sequential_architecture
 
 
 class QValueEstimator(BaseNetwork):

deeprecsys/rl/neural_networks/value_estimator.py → deeprecsys/neural_networks/value_estimator.py

@@ -5,8 +5,8 @@
 from torch.nn import MSELoss
 from torch.optim import Adam
 
-from deeprecsys.rl.neural_networks.base_network import BaseNetwork
-from deeprecsys.rl.neural_networks.deep_q_network import sequential_architecture
+from deeprecsys.neural_networks.base_network import BaseNetwork
+from deeprecsys.neural_networks.deep_q_network import sequential_architecture
 
 
 class ValueEstimator(BaseNetwork):

deeprecsys/rl/agents/actor_critic.py

@@ -1,12 +1,12 @@
 from typing import Any, List, Optional
 
+from deeprecsys.neural_networks.policy_estimator import PolicyEstimator
+from deeprecsys.neural_networks.value_estimator import ValueEstimator
 from deeprecsys.rl.agents.agent import ReinforcementLearning
 from deeprecsys.rl.experience_replay.buffer_parameters import (
     ExperienceReplayBufferParameters,
 )
 from deeprecsys.rl.experience_replay.experience_buffer import ExperienceReplayBuffer
-from deeprecsys.rl.neural_networks.policy_estimator import PolicyEstimator
-from deeprecsys.rl.neural_networks.value_estimator import ValueEstimator
 
 
 class ActorCriticAgent(ReinforcementLearning):

deeprecsys/rl/agents/dqn.py

@@ -3,15 +3,15 @@
 from numpy import arange
 from numpy.random import RandomState
 
+from deeprecsys.neural_networks.deep_q_network import (
+    DeepQNetwork,
+    sequential_architecture,
+)
 from deeprecsys.rl.agents.epsilon_greedy import DecayingEpsilonGreedy
 from deeprecsys.rl.experience_replay.buffer_parameters import (
     ExperienceReplayBufferParameters,
 )
 from deeprecsys.rl.experience_replay.experience_buffer import ExperienceReplayBuffer
-from deeprecsys.rl.neural_networks.deep_q_network import (
-    DeepQNetwork,
-    sequential_architecture,
-)
 
 
 class DQNAgent(DecayingEpsilonGreedy):

deeprecsys/rl/agents/epsilon_greedy.py

@@ -3,7 +3,7 @@
 
 from numpy.random import RandomState
 
-from .agent import ReinforcementLearning
+from deeprecsys.rl.agents.agent import ReinforcementLearning
 
 
 class DecayingEpsilonGreedy(ReinforcementLearning, ABC):

deeprecsys/rl/agents/rainbow.py

@@ -4,6 +4,7 @@
 from numpy import arange
 from numpy.random import RandomState
 
+from deeprecsys.neural_networks.dueling import DuelingDDQN
 from deeprecsys.rl.agents.agent import ReinforcementLearning
 from deeprecsys.rl.experience_replay.buffer_parameters import (
     ExperienceReplayBufferParameters,
@@ -13,7 +14,6 @@
     PrioritizedExperienceReplayBuffer,
 )
 from deeprecsys.rl.learning_statistics import LearningStatistics
-from deeprecsys.rl.neural_networks.dueling import DuelingDDQN
 
 
 class RainbowDQNAgent(ReinforcementLearning):

deeprecsys/rl/agents/reinforce.py

@@ -3,12 +3,12 @@
 import numpy as np
 from torch import FloatTensor
 
+from deeprecsys.neural_networks.policy_estimator import PolicyEstimator
 from deeprecsys.rl.agents.agent import ReinforcementLearning
 from deeprecsys.rl.experience_replay.buffer_parameters import (
     ExperienceReplayBufferParameters,
 )
 from deeprecsys.rl.experience_replay.experience_buffer import ExperienceReplayBuffer
-from deeprecsys.rl.neural_networks.policy_estimator import PolicyEstimator
 
 
 class ReinforceAgent(ReinforcementLearning):

deeprecsys/rl/agents/soft_actor_critic.py

@@ -5,6 +5,8 @@
 from gym.spaces import Discrete
 from torch import BoolTensor, FloatTensor
 
+from deeprecsys.neural_networks.gaussian_actor import GaussianActor
+from deeprecsys.neural_networks.q_value_estimator import TwinnedQValueEstimator
 from deeprecsys.rl.agents.agent import ReinforcementLearning
 from deeprecsys.rl.experience_replay.buffer_parameters import (
     ExperienceReplayBufferParameters,
@@ -13,8 +15,6 @@
 from deeprecsys.rl.experience_replay.priority_replay_buffer import (
     PrioritizedExperienceReplayBuffer,
 )
-from deeprecsys.rl.neural_networks.gaussian_actor import GaussianActor
-from deeprecsys.rl.neural_networks.q_value_estimator import TwinnedQValueEstimator
 
 
 class SoftActorCritic(ReinforcementLearning):