initializations.py

import numpy as np

"""
References:
http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf

"""


def normal(shape, scale=0.5):
    return np.random.normal(size=shape, scale=scale)


def uniform(shape, scale=0.5):
    return np.random.uniform(size=shape, low=-scale, high=scale)


def zero(shape, **kwargs):
    return np.zeros(shape)


def one(shape, **kwargs):
    return np.ones(shape)


def orthogonal(shape, scale=0.5):
    flat_shape = (shape[0], np.prod(shape[1:]))
    array = np.random.normal(size=flat_shape)
    u, _, v = np.linalg.svd(array, full_matrices=False)
    array = u if u.shape == flat_shape else v
    return np.reshape(array * scale, shape)


def _glorot_fan(shape):
    assert len(shape) >= 2

    if len(shape) == 4:
        receptive_field_size = np.prod(shape[2:])
        fan_in = shape[1] * receptive_field_size
        fan_out = shape[0] * receptive_field_size
    else:
        fan_in, fan_out = shape[:2]
    return float(fan_in), float(fan_out)


def glorot_normal(shape, **kwargs):
    fan_in, fan_out = _glorot_fan(shape)
    s = np.sqrt(2.0 / (fan_in + fan_out))
    return normal(shape, s)


def glorot_uniform(shape, **kwargs):
    fan_in, fan_out = _glorot_fan(shape)
    s = np.sqrt(6.0 / (fan_in + fan_out))
    return uniform(shape, s)


def he_normal(shape, **kwargs):
    fan_in, fan_out = _glorot_fan(shape)
    s = np.sqrt(2.0 / fan_in)
    return normal(shape, s)


def he_uniform(shape, **kwargs):
    fan_in, fan_out = _glorot_fan(shape)
    s = np.sqrt(6.0 / fan_in)
    return uniform(shape, s)


def get_initializer(name):
    """Returns initialization function by the name."""
    try:
        return globals()[name]
    except Exception:
        raise ValueError("Invalid initialization function.")