Metrics class and eye gaze example

.github/workflows/python-package-conda.yml

@@ -21,6 +21,7 @@ jobs:
     - name: Install dependencies
       run: |
         conda env update --file environment.yml --name base
+        conda env update --file examples/examples_env.yml --name base
 #     - name: Lint with flake8
 #       run: |
 #         conda install flake8

.github/workflows/python-package.yml

@@ -29,6 +29,7 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f examples/examples_req.txt ]; then pip install -r examples/examples_req.txt; fi
 #     - name: Lint with flake8
 #       run: |
 #         python -m pip install flake8

.gitignore

@@ -11,9 +11,14 @@ __pycache__/
 # C extensions
 *.so
 
-### VisualStudioCode ###
+# VSCode
 .vscode/     
 
+# Data
+data/*
+*.h5
+*.hdf5
+
 
 # Distribution / packaging
 .Python

environment.yml

@@ -1,4 +1,5 @@
 name: neuralib
 dependencies:
-  - numpy
-  - matplotlib
+  - pip
+  - pip:
+    - -r file:requirements.txt

examples/README.md

@@ -0,0 +1,10 @@
+## Eye-gaze estimation
+Eye-gaze information has many applications in Human-Computer Interaction such as improving user experience in everyday tasks, such as [reading](http://gbuscher.com/publications/BuscherBiedert10_readingRegions.pdf), or facilitate [gaze-based interaction](https://perceptual.mpi-inf.mpg.de/files/2014/07/majaranta14_apc.pdf). Eye-gaze also plays a crucial role in assisting users with motor-disabilities and can even be used to infer cognitive state such as cognitive load. Recently, deep-learning based gaze estimation was used for [unsupervised eye contact detection](https://perceptual.mpi-inf.mpg.de/files/2017/05/zhang17_uist.pdf) in everyday scenarios.
+
+The eye gaze data used is based on [UnityEyes](https://www.cl.cam.ac.uk/research/rainbow/projects/unityeyes/), a synthetic eyes dataset. Note that this is a clean and simple dataset as far as eye gaze estimation goes. The real-world task (real images, uncontrolled environmental conditions) is much more challenging and comprehensively solving this is an active area of research that would go beyond the scope of this notebook. The dataset consists of grayscale images (20x30) and pitch and yaw angles in radians as labels. 
+
+Since pitch and yaw angles are difficult to interpret, a more intuitive *angular* error metric based on [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity#Angular_distance_and_similarity) has been defined. Angular distance in this case, is a single scalar value which would describe how many degrees the eyeball would need to turn to face match the estimated gaze direction.
+
+Helper functions to compute these metrics and a method to visualize our results are contained in `helpers/eye_gaze_helpers.py`.
+
+When you are ready, run `python eye_gaze_estimation.py`. This will download the data required automatically.

examples/examples_env.yml

@@ -0,0 +1,7 @@
+name: neuralib_examples
+dependencies:
+  - h5py
+  - pip
+  - pip:
+    - -r examples_req.txt
+

examples/examples_req.txt

@@ -0,0 +1,3 @@
+-r ../requirements.txt
+h5py
+requests==2.27.1

examples/eye_gaze_estimation.py

@@ -0,0 +1,78 @@
+import h5py
+import os
+from helpers.eye_gaze_helpers import AngularError
+from neuralib.architectures import MLP
+from neuralib.layers.activations import ReLU
+from neuralib.layers.layers import Identity
+from neuralib.optimizers import VGD
+
+from helpers.download_data import download
+
+if __name__ == '__main__':
+
+    abspath = os.path.abspath(__file__)
+    dname = os.path.dirname(abspath)
+    os.chdir(dname)
+    file_path = os.path.join(dname, '../data/eye_gaze/eye_data.h5')
+
+    # Download data
+    download(url='https://github.com/jtj21/ComputationalInteraction18/blob/master/Otmar/data/eye_data.h5?raw=true',
+             file_path=file_path)
+
+    # Load in our data
+    with h5py.File(file_path, 'r') as h5f:
+        train_x = h5f['train/x_small'][:]
+        train_y = h5f['train/y'][:]
+
+        validation_x = h5f['validation/x_small'][:]
+        validation_y = h5f['validation/y'][:]
+
+        test_x = h5f['test/x_small'][:]
+        test_y = h5f['test/y'][:]
+
+    # A neural network should be trained until the training and test
+    # errors plateau, that is, they do not improve any more.
+    epochs = 201
+
+    # Having more neurons in a network allows for more complex 
+    # mappings to be learned between input data and expected outputs.
+    # However, defining the function to be too complex can lead to 
+    # overfitting, that is, any function can be learned to memorize
+    # training data.
+    n_hidden_units = 64
+
+    # Lower batch sizes can cause noisy training error progression,
+    # but sometimes lead to better generalization (less overfitting
+    # to training data)
+    batch_size = 16
+
+    # A higher learning rate makes training faster, but can cause
+    # overfitting
+    learning_rate = 0.0005
+
+    # TODO: implement L2 regularization
+    # Increase to reduce over-fitting effects
+    # l2_regularization_coefficient = 0.0001
+
+    # train_x_flat = train_x.reshape(train_x.shape[0], -1)
+    # test_x_flat = test_x.reshape(test_x.shape[0], -1)
+
+    n_features = train_x.reshape(train_x.shape[0], -1).shape[1] # flattened grayscale image of eye
+    n_outputs = train_y.shape[1] # Pitch and yaw in radians
+
+    mlp = MLP(output_size=n_outputs,
+              input_size=n_features,
+              hidden_size=n_hidden_units,
+              activations=[ReLU(), Identity()], 
+              metrics = [AngularError(img_visualize=True)], random_seed=42)
+
+    mlp.train(train_x, train_y, epochs=epochs, batch_size=batch_size, optimizer=VGD(lr=learning_rate), X_test=test_x, y_test=test_y)
+
+    metric = mlp.metrics[0]
+    print('Cosine Similarity Error Train [deg]: ', metric.metric_history_train[-1][1])
+    if len(metric.metric_history_test) > 0:
+        print('Cosine Similarity Error Test [deg]: ', metric.metric_history_test[-1][1])
+
+    # Plot metrics
+    metric.plot_progress('train')
+    metric.plot_progress('test')

examples/helpers/download_data.py

@@ -0,0 +1,23 @@
+import os
+import requests
+
+
+def download(url: str, file_path: str):
+    """Download file from url to file_path."""
+    if os.path.exists(file_path):
+        print('File already exists: %s' % file_path)
+        return
+    # Create directory if it does not exist
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+    r = requests.get(url, stream=True)
+    if r.ok:
+        print("saving to", os.path.abspath(file_path))
+        with open(file_path, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=1024 * 8):
+                if chunk:
+                    f.write(chunk)
+                    f.flush()
+                    os.fsync(f.fileno())
+    else:  # HTTP status code 4XX/5XX
+        print("Download failed: status code {}\n{}".format(r.status_code, r.text))

examples/helpers/eye_gaze_helpers.py

@@ -0,0 +1,61 @@
+from xmlrpc.client import Boolean
+from matplotlib import pyplot as plt
+import numpy as np
+
+from neuralib.metrics import ScalarMetric
+
+class AngularError(ScalarMetric):
+    """Calculate angular error (via cosine similarity)."""
+
+    def __init__(self, every_n_epochs: int = 1, img_visualize: Boolean = False) -> None:
+        super().__init__(every_n_epochs)
+        self.img_visualize = img_visualize
+
+    def calculate_from_predictions(self, y_pred: np.array, y: np.array):
+        """Calculate angular error (via cosine similarity)."""
+        return np.mean(self._angular_error(y_pred, y))
+
+    def visualize(self, X: np.array, y: np.array, y_pred: np.array) -> None:
+        """Visualize errors of neural network on given data."""
+        if self.img_visualize:
+            nr, nc = 1, 12
+            n = nr * nc
+            fig = plt.figure(figsize=(12, 2.))
+            for i, (image, label, prediction) in enumerate(zip(X[:n], y[:n], y_pred)):
+                plt.subplot(nr, nc, i + 1)
+                plt.imshow(image, cmap='gray')
+                error = self._angular_error(prediction.reshape(1, 2), label.reshape(1, 2))
+                plt.title('%.1f' % error, color='g' if error < 7.0 else 'r')
+                plt.gca().get_xaxis().set_visible(False)
+                plt.gca().get_yaxis().set_visible(False)
+            plt.tight_layout(pad=0.0)
+            plt.show()
+
+    def _angular_error(self, X, y):
+        """Calculate angular error (via cosine similarity)."""
+
+        def pitchyaw_to_vector(pitchyaws):
+            """Convert given pitch and yaw angles to unit gaze vectors."""
+            n = pitchyaws.shape[0]
+            sin = np.sin(pitchyaws)
+            cos = np.cos(pitchyaws)
+            out = np.empty((n, 3))
+            out[:, 0] = np.multiply(cos[:, 0], sin[:, 1])
+            out[:, 1] = sin[:, 0]
+            out[:, 2] = np.multiply(cos[:, 0], cos[:, 1])
+            return out
+
+        a = pitchyaw_to_vector(y) 
+        b = pitchyaw_to_vector(X)
+
+        ab = np.sum(np.multiply(a, b), axis=1)
+        a_norm = np.linalg.norm(a, axis=1)
+        b_norm = np.linalg.norm(b, axis=1)
+
+        # Avoid zero-values (to avoid NaNs)
+        a_norm = np.clip(a_norm, a_min=1e-7, a_max=None)
+        b_norm = np.clip(b_norm, a_min=1e-7, a_max=None)
+
+        similarity = np.divide(ab, np.multiply(a_norm, b_norm))
+
+        return np.arccos(similarity) * (180.0 / np.pi)

examples/x_or.py

@@ -1,6 +1,6 @@
 import numpy as np
 import matplotlib.pyplot as plt
-from neuralib import Model
+from neuralib import SequentialModel
 from neuralib.layers import Linear, Sigmoid, MSE
 from neuralib.optimizers import VGD
 
@@ -48,7 +48,7 @@ def plot_grid(model):
     hidden_dim = 4   # number of neurons in the hidden layer
     target_dim = 1    # label dimensionality 
 
-    model = Model([Linear(input_size=input_dim, output_size=hidden_dim), 
+    model = SequentialModel([Linear(input_size=input_dim, output_size=hidden_dim), 
                    Sigmoid(), 
                    Linear(input_size=hidden_dim, output_size=target_dim), 
                    MSE()])

neuralib/__init__.py

@@ -1,3 +1,4 @@
 from .utils import *
 from .architectures import *
 from .optimizers import *
+from .metrics import *