FairNeuron.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import logging\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split,KFold\n",
    "from ray import tune\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable,Function\n",
    "from torch.utils.data import Dataset, TensorDataset, DataLoader\n",
    "from torch.utils.data.dataset import random_split\n",
    "from tqdm import trange\n",
    "from pycm import ConfusionMatrix\n",
    "from secml.array.c_array import CArray\n",
    "\n",
    "myfont = {'family': 'Times New Roman',\n",
    "         'weight': 'normal',\n",
    "         'size': 20,\n",
    "         }\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def attack_keras_model(X, Y, S, nb_attack=25, dmax=0.1):\n",
    "    \"\"\"\n",
    "    Generates an adversarial attack on a general model.\n",
    "    :param X: Original inputs on which the model is trained\n",
    "    :param Y: Original outputs on which the model is trained\n",
    "    :param S: Original protected attributes on which the model is trained\n",
    "    :return: Adversarial dataset (i.e. new data points + original input)\n",
    "    \"\"\"\n",
    "\n",
    "    from secml.data import CDataset\n",
    "    from secml.array import CArray\n",
    "\n",
    "    data_set_encoded_secML = CArray(X, dtype=float, copy=True)\n",
    "    data_set_encoded_secML = CDataset(data_set_encoded_secML, Y)\n",
    "\n",
    "    n_tr = round(0.66 * X.shape[0])\n",
    "    n_ts = X.shape[0] - n_tr\n",
    "\n",
    "    logger.debug(X.shape)\n",
    "    logger.debug(n_tr)\n",
    "    logger.debug(n_ts)\n",
    "\n",
    "    from secml.data.splitter import CTrainTestSplit\n",
    "    splitter = CTrainTestSplit(train_size=n_tr, test_size=n_ts)\n",
    "\n",
    "    tr_set_secML, ts_set_secML = splitter.split(data_set_encoded_secML)\n",
    "\n",
    "    from secml.ml.classifiers import CClassifierSVM\n",
    "    from secml.ml.classifiers.multiclass import CClassifierMulticlassOVA\n",
    "    from secml.ml.kernel import CKernelRBF\n",
    "    clf = CClassifierMulticlassOVA(CClassifierSVM, kernel=CKernelRBF())\n",
    "\n",
    "    xval_params = {'C': [1e-4, 1e-3, 1e-2, 0.1, 1], 'kernel.gamma': [0.01, 0.1, 1, 10, 100, 1e3]}\n",
    "\n",
    "    random_state = 999\n",
    "\n",
    "    from secml.data.splitter import CDataSplitterKFold\n",
    "    xval_splitter = CDataSplitterKFold(num_folds=3, random_state=random_state)\n",
    "\n",
    "    logger.debug(\"Estimating the best training parameters...\")\n",
    "    best_params = clf.estimate_parameters(\n",
    "        dataset=tr_set_secML,\n",
    "        parameters=xval_params,\n",
    "        splitter=xval_splitter,\n",
    "        metric='accuracy',\n",
    "        perf_evaluator='xval'\n",
    "    )\n",
    "    logger.debug(\"The best training parameters are: \", best_params)\n",
    "\n",
    "    logger.debug(clf.get_params())\n",
    "    logger.debug(clf.num_classifiers)\n",
    "\n",
    "    from secml.ml.peval.metrics import CMetricAccuracy\n",
    "    metric = CMetricAccuracy()\n",
    "\n",
    "    # Train the classifier\n",
    "    clf.fit(tr_set_secML)\n",
    "    logger.debug(clf.num_classifiers)\n",
    "\n",
    "    # Compute predictions on a test set\n",
    "    y_pred = clf.predict(ts_set_secML.X)\n",
    "\n",
    "    # Evaluate the accuracy of the classifier\n",
    "    acc = metric.performance_score(y_true=ts_set_secML.Y, y_pred=y_pred)\n",
    "\n",
    "    logger.debug(\"Accuracy on test set: {:.2%}\".format(acc))\n",
    "\n",
    "    # Prepare attack configuration\n",
    "\n",
    "    noise_type = 'l2'   # Type of perturbation 'l1' or 'l2'\n",
    "    lb, ub = 0, 1       # Bounds of the attack space. Can be set to `None` for unbounded\n",
    "    y_target = None     # None if `error-generic` or a class label for `error-specific`\n",
    "\n",
    "    solver_params = {\n",
    "        'eta': 0.1,         # grid search resolution\n",
    "        'eta_min': 0.1,\n",
    "        'eta_max': None,    # None should be ok\n",
    "        'max_iter': 1000,\n",
    "        'eps': 1e-2         # Tolerance on the stopping crit.\n",
    "    }\n",
    "\n",
    "    # Run attack\n",
    "\n",
    "    from secml.adv.attacks.evasion import CAttackEvasionPGDLS\n",
    "    pgd_ls_attack = CAttackEvasionPGDLS(\n",
    "        classifier=clf,\n",
    "        surrogate_classifier=clf,\n",
    "        surrogate_data=tr_set_secML,\n",
    "        distance=noise_type,\n",
    "        dmax=dmax,\n",
    "        lb=lb, ub=ub,\n",
    "        solver_params=solver_params,\n",
    "        y_target=y_target)\n",
    "\n",
    "    nb_feat = X.shape[1]\n",
    "\n",
    "    result_pts = np.empty([nb_attack, nb_feat])\n",
    "    result_class = np.empty([nb_attack, 1])\n",
    "\n",
    "    # take a point at random being the starting point of the attack and run the attack\n",
    "    import random\n",
    "    for nb_iter in range(0, nb_attack):\n",
    "        rn = random.randint(0, ts_set_secML.num_samples - 1)\n",
    "        x0, y0 = ts_set_secML[rn, :].X, ts_set_secML[rn, :].Y,\n",
    "\n",
    "        try:\n",
    "            y_pred_pgdls, _, adv_ds_pgdls, _ = pgd_ls_attack.run(x0, y0)\n",
    "            adv_pt = adv_ds_pgdls.X.get_data()\n",
    "            # np.asarray([np.asarray(row, dtype=float) for row in y_tr], dtype=float)\n",
    "            result_pts[nb_iter] = adv_pt\n",
    "            result_class[nb_iter] = y_pred_pgdls.get_data()[0]\n",
    "        except ValueError:\n",
    "            logger.warning(\"value error on {}\".format(nb_iter))\n",
    "\n",
    "    return result_pts, result_class, ts_set_secML[:nb_attack, :].Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class bm:\n",
    "    def __init__(self, df):\n",
    "        self._df = df\n",
    "\n",
    "    def P(self, **kwargs):\n",
    "        \"\"\"\n",
    "        Declares the random variables from the set `kwargs`.\n",
    "        \"\"\"\n",
    "        self._variables = kwargs\n",
    "        return self\n",
    "\n",
    "    def given(self, **kwargs):\n",
    "        \"\"\"\n",
    "        Calculates the probability on a finite set of samples with `kwargs` in the\n",
    "        conditioning set. \n",
    "        \"\"\"\n",
    "        self._given = kwargs\n",
    "        \n",
    "        # Here's where the magic happens\n",
    "        prior = True\n",
    "        posterior = True\n",
    "        \n",
    "        for k in self._variables:\n",
    "            if type(self._variables[k]) == type(lambda x:x):\n",
    "                posterior = posterior & (self._df[k].apply(self._variables[k]))\n",
    "            else:\n",
    "                posterior = posterior & (self._df[k] == self._variables[k])\n",
    "\n",
    "        \n",
    "        for k in self._given:\n",
    "            if type(self._given[k]) == type(lambda x:x):\n",
    "                prior = prior & (self._df[k].apply(self._given[k]))\n",
    "                posterior = posterior & (self._df[k].apply(self._given[k]))\n",
    "            else:\n",
    "                prior = prior & (self._df[k] == self._given[k])\n",
    "                posterior = posterior & (self._df[k] == self._given[k])\n",
    "        return posterior.sum()/prior.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_dataset(df):\n",
    "    \"\"\"\n",
    "\n",
    "    :param df:\n",
    "    :return: Tuple of the transformed dataset and the labels Y and S\n",
    "    \"\"\"\n",
    "\n",
    "    df_binary = df[(df[\"race\"] == \"Caucasian\") | (df[\"race\"] == \"African-American\")]\n",
    "\n",
    "    del df_binary['c_jail_in']\n",
    "    del df_binary['c_jail_out']\n",
    "\n",
    "    ##separated class from the rests of the features\n",
    "    # remove unnecessary dimensions from Y -> only the decile_score remains\n",
    "    Y = df_binary['decile_score']\n",
    "    del df_binary['decile_score']\n",
    "    Y_true = df_binary['two_year_recid']\n",
    "    del df_binary['two_year_recid']\n",
    "    del df_binary['score_text']\n",
    "\n",
    "    S = df_binary['race']\n",
    "    #del df_binary['race']\n",
    "    #del df_binary['is_recid']\n",
    "\n",
    "    print(df_binary.shape)\n",
    "\n",
    "    # set sparse to False to return dense matrix after transformation and keep all dimensions homogeneous\n",
    "    encod = preprocessing.OneHotEncoder(sparse=False)\n",
    "\n",
    "    data_to_encode = df_binary.to_numpy()\n",
    "    feat_to_encode = data_to_encode[:, 0]\n",
    "    # print(feat_to_encode)\n",
    "    # transposition\n",
    "    feat_to_encode = feat_to_encode.reshape(-1, 1)\n",
    "    # print(feat_to_encode)\n",
    "    encoded_feature = encod.fit_transform(feat_to_encode)\n",
    "\n",
    "    df_binary_encoded = pd.DataFrame(encoded_feature)\n",
    "\n",
    "    feat_to_encode = data_to_encode[:, 1]\n",
    "    feat_to_encode = feat_to_encode.reshape(-1, 1)\n",
    "    encoded_feature = encod.fit_transform(feat_to_encode)\n",
    "\n",
    "\n",
    "    df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    feat_to_encode = data_to_encode[:, 2] == \"Caucasian\"\n",
    "    feat_to_encode = feat_to_encode.reshape(-1, 1)\n",
    "    encoded_feature = encod.fit_transform(feat_to_encode)\n",
    "\n",
    "    df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    # feature [2] [3] [4] [5] [6] [7] [8] has to be put between 0 and 1\n",
    "\n",
    "    for i in range(3, 10):\n",
    "        encoded_feature = data_to_encode[:, i]\n",
    "        ma = np.amax(encoded_feature)\n",
    "        mi = np.amin(encoded_feature)\n",
    "        encoded_feature = (encoded_feature - mi) / (ma - mi)\n",
    "        df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    feat_to_encode = data_to_encode[:, 10]\n",
    "    feat_to_encode = feat_to_encode.reshape(-1, 1)\n",
    "    encoded_feature = encod.fit_transform(feat_to_encode)\n",
    "\n",
    "    df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    feat_to_encode = data_to_encode[:, 11]\n",
    "    feat_to_encode = feat_to_encode.reshape(-1, 1)\n",
    "    encoded_feature = encod.fit_transform(feat_to_encode)\n",
    "\n",
    "    df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    return df_binary_encoded, Y, S, Y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_dataset_census(df):\n",
    "    \"\"\"\n",
    "    :param df: the dataset \"census income\" from a csv file with reduced features, heterogeneous types and missing values, no header\n",
    "    :return: Tuple of the transformed dataset and the labels Y and S\n",
    "    \"\"\"\n",
    "    df_replace = df.replace(to_replace=\"?\",value=np.nan)\n",
    "    df_replace.dropna(inplace=True, axis=0)\n",
    "\n",
    "    label_encoder = preprocessing.LabelEncoder()\n",
    "    oh_encoder = preprocessing.OneHotEncoder(sparse=False)\n",
    "\n",
    "    df_label = df_replace.iloc[:,-1]\n",
    "\n",
    "    ##Y_true is the vector containing labels, at this point, labels (initially strings) have been transformed into integer (0 and 1) -> -5000 is now '0' and 5000+ is now '+1'\n",
    "    Y = label_encoder.fit_transform(df_label)\n",
    "    #remove last column from df\n",
    "    del df_replace[df_replace.columns[-1]]\n",
    "\n",
    "    # Y_true is the true outcome, in this case we're not using a future predictor (vs. compas)\n",
    "    Y_true=[]\n",
    "\n",
    "    #S is the protected attribute\n",
    "    # could also be feature 7 (sex) or feature 13 (citizenship)\n",
    "    S=df_replace[\"sex\"]\n",
    "    del df_replace[\"sex\"]\n",
    "\n",
    "    #remove feature fnlwgt\n",
    "    del df_replace[\"fnlwgt\"]\n",
    "\n",
    "    #remove examples with missing values\n",
    "              ## change 1 to 0 \n",
    "\n",
    "    #     if df_replace.shape == df.shape:\n",
    "    #         raise AssertionError(\"The removal of na values failed\")\n",
    "\n",
    "    print(df_replace.shape)\n",
    "\n",
    "    #transform other features\n",
    "    #feature age to normalize\n",
    "    encoded_feature = df_replace.to_numpy()[:, 0]\n",
    "    mi = np.amin(encoded_feature)\n",
    "    ma = np.amax(encoded_feature)\n",
    "    encoded_feature = (encoded_feature - mi) / (ma - mi)\n",
    "\n",
    "    #df_binary_encoded is the data frame containing encoded features\n",
    "    df_binary_encoded = pd.DataFrame(encoded_feature)\n",
    "    print(df_binary_encoded.shape)\n",
    "\n",
    "\n",
    "    encod_feature = df_replace.iloc[:,1]\n",
    "    encoded_feature = pd.get_dummies(encod_feature)\n",
    "    # df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    #feature 1 to 7 (after removal) are categorical\n",
    "    for i in range(1,8):\n",
    "        encod_feature = df_replace.iloc[:,i]\n",
    "    #     print(encod_feature.shape)\n",
    "        encoded_feature = pd.get_dummies(encod_feature)\n",
    "    #     print(encoded_feature)\n",
    "    #     print(df_binary_encoded)\n",
    "        df_binary_encoded = pd.concat([df_binary_encoded.reset_index(drop=True), pd.DataFrame(encoded_feature).reset_index(drop=True)], axis=1)\n",
    "#         print(df_binary_encoded)\n",
    "    #     print('')\n",
    "\n",
    "    #feature 8 and 9 are numerical\n",
    "    for i in range(8,10):\n",
    "        encod_feature = df_replace.iloc[:,i]\n",
    "        mi = np.amin(encod_feature)\n",
    "        ma = np.amax(encod_feature)\n",
    "        encoded_feature = (encod_feature - mi) / (ma - mi)\n",
    "        df_binary_encoded = pd.concat([df_binary_encoded.reset_index(drop=True), pd.DataFrame(encoded_feature).reset_index(drop=True)], axis=1)\n",
    "    #     print(df_binary_encoded.shape)\n",
    "    #feature 10 and 11 are categorical\n",
    "    for i in range(10,12):\n",
    "        encod_feature = df_replace.iloc[:,i]\n",
    "        encoded_feature = pd.get_dummies(encod_feature)\n",
    "        df_binary_encoded = pd.concat([df_binary_encoded.reset_index(drop=True), pd.DataFrame(encoded_feature).reset_index(drop=True)], axis=1)\n",
    "    #     print(df_binary_encoded.shape)\n",
    "\n",
    "    return df_binary_encoded, Y, S, Y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transform_dataset_credit(df):\n",
    "\n",
    "    label_encoder = preprocessing.LabelEncoder()\n",
    "    oh_encoder = preprocessing.OneHotEncoder(sparse=False)\n",
    "\n",
    "    Y = np.array(df.iloc[:,-1] == 2)\n",
    "\n",
    "    del df[df.columns[-1]]\n",
    "\n",
    "    # Y_true is the true outcome, in this case we're not using a future predictor (vs. compas)\n",
    "    Y_true=[]\n",
    "\n",
    "    #S is the protected attribute\n",
    "    S=df.iloc[:,12] > 25\n",
    "    #del df[\"Age\"]\n",
    "\n",
    "    #remove examples with missing values\n",
    "    df_replace = df.replace(to_replace=\"?\",value=np.nan)\n",
    "    df_replace.dropna(inplace=True, axis=1)\n",
    "\n",
    "    print(df_replace.shape)\n",
    "\n",
    "    #transform other features\n",
    "    #feature age to normalize\n",
    "    encoded_feature = df_replace.to_numpy()[:, 1]\n",
    "    mi = np.amin(encoded_feature)\n",
    "    ma = np.amax(encoded_feature)\n",
    "    encoded_feature = (encoded_feature - mi) / (ma - mi)\n",
    "\n",
    "    #df_binary_encoded is the data frame containing encoded features\n",
    "    df_binary_encoded = pd.DataFrame(encoded_feature)\n",
    "\n",
    "    # categorical attributes\n",
    "    for i in [0, 2, 3, 5, 6, 8, 9, 11, 13, 14, 16, 18,19]:\n",
    "        encod_feature = df_replace.iloc[:,i]\n",
    "        encoded_feature = pd.get_dummies(encod_feature)\n",
    "        df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    # Numerical attributes\n",
    "    for i in [1, 7, 10, 15, 17]:\n",
    "        encod_feature = df_replace.iloc[:,i]\n",
    "        mi = np.amin(encod_feature)\n",
    "        ma = np.amax(encod_feature)\n",
    "        encoded_feature = (encod_feature - mi) / (ma - mi)\n",
    "        df_binary_encoded = pd.concat([df_binary_encoded, pd.DataFrame(encoded_feature)], axis=1)\n",
    "\n",
    "    print(S)\n",
    "\n",
    "    return df_binary_encoded, Y, S, Y_true\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GradientReversalFunction(Function):\n",
    "\n",
    "    @staticmethod\n",
    "    def forward(ctx, x, lambda_):\n",
    "        ctx.lambda_ = lambda_\n",
    "        return x.clone()\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grads):\n",
    "        lambda_ = ctx.lambda_\n",
    "        lambda_ = grads.new_tensor(lambda_)\n",
    "        dx = -lambda_ * grads\n",
    "        return dx, None\n",
    "\n",
    "class GradientReversal(torch.nn.Module):\n",
    "    def __init__(self, lambda_=1):\n",
    "        super(GradientReversal, self).__init__()\n",
    "        self.lambda_ = lambda_\n",
    "\n",
    "    def forward(self, x):\n",
    "        return GradientReversalFunction.apply(x, self.lambda_)\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self, input_shape, grl_lambda=100):\n",
    "        super(Net, self).__init__()\n",
    "        # an affine operation: y = Wx + b\n",
    "        self._grl_lambda = grl_lambda\n",
    "        self.fc1 = nn.Linear(input_shape, 32)\n",
    "        self.fc2 = nn.Linear(32, 32)\n",
    "        self.fc3 = nn.Linear(32, 32)\n",
    "        self.fc4 = nn.Linear(32, 1)\n",
    "        if self._grl_lambda != 0:\n",
    "            self.grl = GradientReversal(grl_lambda)\n",
    "            self.fc5 = nn.Linear(32, 2)\n",
    "        # self.grl = GradientReversal(100)\n",
    "\n",
    "    def forward(self, x):\n",
    "        hidden = self.fc1(x)\n",
    "        hidden = F.relu(hidden)\n",
    "        hidden = F.dropout(hidden, 0.1)\n",
    "        hidden = self.fc2(hidden)\n",
    "        hidden = F.relu(hidden)\n",
    "        hidden = self.fc3(hidden)\n",
    "        hidden = F.relu(hidden)\n",
    "\n",
    "        y = self.fc4(hidden)\n",
    "        # y = F.dropout(y, 0.1)\n",
    "\n",
    "        if self._grl_lambda != 0:\n",
    "            s = self.grl(hidden)\n",
    "            s = self.fc5(s)\n",
    "            # s = F.sigmoid(s)\n",
    "            # s = F.dropout(s, 0.1)\n",
    "            return y, s\n",
    "        else:\n",
    "            return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net_nodrop(nn.Module):\n",
    "\n",
    "    def __init__(self, input_shape, grl_lambda=100):\n",
    "        super(Net_nodrop, self).__init__()\n",
    "        # an affine operation: y = Wx + b\n",
    "        self._grl_lambda = grl_lambda\n",
    "        self.fc1 = nn.Linear(input_shape, 32)\n",
    "        self.fc2 = nn.Linear(32, 32)\n",
    "        self.fc3 = nn.Linear(32, 32)\n",
    "        self.fc4 = nn.Linear(32, 1)\n",
    "        if self._grl_lambda != 0:\n",
    "            self.grl = GradientReversal(grl_lambda)\n",
    "            self.fc5 = nn.Linear(32, 2)\n",
    "        # self.grl = GradientReversal(100)\n",
    "\n",
    "    def forward(self, x):\n",
    "        hidden = self.fc1(x)\n",
    "        hidden = F.relu(hidden)\n",
    "#         hidden = F.dropout(hidden, 0.1)\n",
    "        hidden = self.fc2(hidden)\n",
    "        hidden = F.relu(hidden)\n",
    "        hidden = self.fc3(hidden)\n",
    "        hidden = F.relu(hidden)\n",
    "\n",
    "        y = self.fc4(hidden)\n",
    "        # y = F.dropout(y, 0.1)\n",
    "\n",
    "        if self._grl_lambda != 0:\n",
    "            s = self.grl(hidden)\n",
    "            s = self.fc5(s)\n",
    "            # s = F.sigmoid(s)\n",
    "            # s = F.dropout(s, 0.1)\n",
    "            return y, s\n",
    "        else:\n",
    "            return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net_CENSUS(nn.Module):\n",
    "\n",
    "    def __init__(self, input_shape, grl_lambda=100):\n",
    "        super(Net_CENSUS, self).__init__()\n",
    "        # an affine operation: y = Wx + b\n",
    "        self._grl_lambda = grl_lambda\n",
    "        self.fc1 = nn.Linear(input_shape, 128)\n",
    "        self.fc2 = nn.Linear(128, 128)\n",
    "        self.fc3 = nn.Linear(128, 128)\n",
    "        self.fc4 = nn.Linear(128, 1)\n",
    "        if self._grl_lambda != 0:\n",
    "            self.grl = GradientReversal(grl_lambda)\n",
    "            self.fc5 = nn.Linear(128, 2)\n",
    "        # self.grl = GradientReversal(100)\n",
    "\n",
    "    def forward(self, x):\n",
    "        hidden = self.fc1(x)\n",
    "        hidden = F.relu(hidden)\n",
    "        hidden = F.dropout(hidden, 0.1, training=self.training)\n",
    "        hidden = self.fc2(hidden)\n",
    "        hidden = F.relu(hidden)\n",
    "        hidden = self.fc3(hidden)\n",
    "        hidden = F.relu(hidden)\n",
    "\n",
    "        y = self.fc4(hidden)\n",
    "        # y = F.dropout(y, 0.1)\n",
    "\n",
    "        if self._grl_lambda != 0:\n",
    "            s = self.grl(hidden)\n",
    "            s = self.fc5(s)\n",
    "            # s = F.sigmoid(s)\n",
    "            # s = F.dropout(s, 0.1)\n",
    "            return y, s\n",
    "        else:\n",
    "            return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_paras(net):\n",
    "    paras=[]\n",
    "    for name,parameters in net.named_parameters():\n",
    "        paras.append(parameters)\n",
    "    return paras\n",
    "\n",
    "def get_active_neurons4(net,sample):\n",
    "    neurons=[]\n",
    "    def hook(module,input,output):\n",
    "        neurons.append(output.data)\n",
    "    handle1=net.fc1.register_forward_hook(hook)\n",
    "    handle2=net.fc2.register_forward_hook(hook)\n",
    "    handle3=net.fc3.register_forward_hook(hook)\n",
    "    handle4=net.fc4.register_forward_hook(hook)\n",
    "    net(x=torch.tensor(sample,dtype=torch.float32))\n",
    "    handle1.remove()\n",
    "    handle2.remove()\n",
    "    handle3.remove()\n",
    "    handle4.remove()\n",
    "    return neurons\n",
    "\n",
    "def get_contrib4(paras,neurons):\n",
    "    contrib_list=[]\n",
    "    for i in range(3):\n",
    "        i=i\n",
    "        contrib=neurons[i]*paras[2*i+2]\n",
    "        contrib_list.append(contrib)\n",
    "    return contrib_list\n",
    "\n",
    "def get_path_set4(net,sample,GAMMA=0.9):\n",
    "    active_neuron_indice=[[],[],[],[]]\n",
    "    path_set=set()\n",
    "    neurons=get_active_neurons4(net,sample)\n",
    "    paras=get_paras(net)\n",
    "    contrib_list=get_contrib4(paras,neurons)\n",
    "    active_neuron_indice[3].append(torch.argmax(neurons[3]).item())\n",
    "    for i in range(3):\n",
    "        L=3-i\n",
    "        for j in active_neuron_indice[L]:\n",
    "            s=torch.sort(contrib_list[L-1][j],descending=True)\n",
    "            sum=0\n",
    "            for k in range(len(contrib_list[L-1][j])):\n",
    "                sum+=s.values[k].item()\n",
    "                active_neuron_indice[L-1].append(s.indices[k].item())\n",
    "                path_set.add((L,s.indices[k].item(),j))\n",
    "                if(sum>=GAMMA*neurons[L][j].item()):\n",
    "                    break\n",
    "    return path_set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(results, threshold, fraction,dataset='compas'):\n",
    "    \"Create the metrics from an output df.\"\n",
    "\n",
    "    # Calculate biases after training\n",
    "    dem_parity = abs(\n",
    "        bm(results).P(pred=lambda x: x > threshold).given(race=0)\n",
    "        - bm(results).P(pred=lambda x: x > threshold).given(\n",
    "            race=1))\n",
    "\n",
    "    eq_op = abs(\n",
    "        bm(results).P(pred=lambda x: x > threshold).given(race=0, compas=True)\n",
    "        - bm(results).P(pred=lambda x: x > threshold).given(race=1, compas=True))\n",
    "\n",
    "    dem_parity_ratio = abs(\n",
    "        bm(results).P(pred=lambda x: x > threshold).given(race=0)\n",
    "        / bm(results).P(pred=lambda x: x > threshold).given(\n",
    "            race=1))\n",
    "\n",
    "    cm = ConfusionMatrix(actual_vector=(results['true'] == True).values,\n",
    "                         predict_vector=(results['pred'] > threshold).values)\n",
    "    if dataset=='compas':\n",
    "        cm_high_risk = ConfusionMatrix(actual_vector=(results['compas'] > 8).values,\n",
    "                             predict_vector=(results['pred'] > 8).values)\n",
    "\n",
    "        result = {\"DP\": dem_parity,\n",
    "                  \"EO\": eq_op,\n",
    "                  \"DP ratio\": dem_parity_ratio,\n",
    "                  \"acc\": cm.Overall_ACC,\n",
    "                  \"acc_ci_min\": cm.CI95[0],\n",
    "                  \"acc_ci_max\": cm.CI95[1],\n",
    "                  \"f1\": cm.F1_Macro,\n",
    "                  \"acc_high_risk\": cm_high_risk.Overall_ACC,\n",
    "                  \"acc_ci_min_high_risk\": cm_high_risk.CI95[0],\n",
    "                  \"acc_ci_max_high_risk\": cm_high_risk.CI95[1],\n",
    "                  \"f1_high_risk\": cm_high_risk.F1_Macro,\n",
    "                  \"adversarial_fraction\": fraction\n",
    "                  }\n",
    "    else:\n",
    "        result = {\"DP\": dem_parity,\n",
    "                  \"EO\": eq_op,\n",
    "                  \"DP ratio\": dem_parity_ratio,\n",
    "                  \"acc\": cm.Overall_ACC,\n",
    "                  \"acc_ci_min\": cm.CI95[0],\n",
    "                  \"acc_ci_max\": cm.CI95[1],\n",
    "                  \"f1\": cm.F1_Macro,\n",
    "                  \"adversarial_fraction\": fraction\n",
    "                  }\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_evaluate(train_loader: DataLoader,\n",
    "                       val_loader: DataLoader,\n",
    "                       test_loader: DataLoader,\n",
    "                       device,\n",
    "                       input_shape,\n",
    "                       grl_lambda=None,\n",
    "                       model=None,\n",
    "                       dataset='compas'):\n",
    "    \"\"\"\n",
    "\n",
    "    :param train_loader: Pytorch-like DataLoader with training data.\n",
    "    :param val_loader: Pytorch-like DataLoader with validation data.\n",
    "    :param test_loader: Pytorch-like DataLoader with testing data.\n",
    "    :param device: The target device for the training.\n",
    "    :return: A tuple: (trained Pytorch-like model, dataframe with results on test set)\n",
    "    \"\"\"\n",
    "\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    grl_lambda = 50\n",
    "    epochs = 50\n",
    "\n",
    "    if model is None:\n",
    "        # Redefine the model\n",
    "        if dataset=='census':\n",
    "            model = Net_CENSUS(input_shape=input_shape, grl_lambda=grl_lambda).to(device)\n",
    "        elif dataset=='compas_nodrop':\n",
    "            model = Net_nodrop(input_shape=input_shape, grl_lambda=grl_lambda).to(device)\n",
    "        else:\n",
    "            model = Net(input_shape=input_shape, grl_lambda=grl_lambda).to(device)\n",
    "        \n",
    "    model = model.to(device)\n",
    "    criterion = nn.MSELoss().to(device)\n",
    "    criterion_bias = nn.CrossEntropyLoss().to(device)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=1e-2)\n",
    "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, threshold=0.3, cooldown=5)\n",
    "\n",
    "    training_losses = []\n",
    "    validation_losses = []\n",
    "\n",
    "    t_prog = trange(epochs, desc='Training neural network', leave=False, position=1, mininterval=5)\n",
    "    # t_prog = trange(50)\n",
    "\n",
    "    for epoch in t_prog:\n",
    "        model.train()\n",
    "\n",
    "        batch_losses = []\n",
    "        for x_batch, y_batch, _, s_batch in train_loader:\n",
    "            x_batch = x_batch.to(device)\n",
    "            y_batch = y_batch.to(device)\n",
    "            s_batch = s_batch.to(device)\n",
    "\n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # forward + backward + optimize\n",
    "            if grl_lambda is not None and grl_lambda != 0:\n",
    "                outputs, outputs_protected = model(x_batch)\n",
    "                loss = criterion(outputs, y_batch) + criterion_bias(outputs_protected, s_batch.argmax(dim=1))\n",
    "            else:\n",
    "                outputs = model(x_batch)\n",
    "                loss = criterion(outputs, y_batch)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "        training_loss = np.mean(batch_losses)\n",
    "        training_losses.append(training_loss)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            val_losses = []\n",
    "            for x_val, y_val, _, s_val in val_loader:\n",
    "                x_val = x_val.to(device)\n",
    "                y_val = y_val.to(device)\n",
    "                s_val = s_val.to(device)\n",
    "                model.eval()\n",
    "                if grl_lambda is not None and grl_lambda != 0:\n",
    "                    yhat, s_hat = model(x_val)\n",
    "                    val_loss = (criterion(y_val, yhat) + criterion_bias(s_val, s_hat.argmax(dim=1))).item()\n",
    "                else:\n",
    "                    yhat = model(x_val)\n",
    "                    val_loss = criterion(y_val, yhat).item()\n",
    "                val_losses.append(val_loss)\n",
    "            validation_loss = np.mean(val_losses)\n",
    "            validation_losses.append(validation_loss)\n",
    "\n",
    "            scheduler.step(val_loss)\n",
    "\n",
    "        t_prog.set_postfix({\"epoch\": epoch, \"training_loss\": training_loss,\n",
    "                            \"validation_loss\": validation_loss}, refresh=False)  # print last metrics\n",
    "\n",
    "#     if args.show_graphs:\n",
    "#         plt.plot(range(len(training_losses)), training_losses)\n",
    "#         plt.plot(range(len(validation_losses)), validation_losses)\n",
    "#         # plt.scatter(x_tensor, y_out.detach().numpy())\n",
    "#         plt.ylabel('some numbers')\n",
    "#         plt.show()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        test_losses = []\n",
    "        test_results = []\n",
    "        for x_test, y_test, ytrue, s_true in test_loader:\n",
    "            x_test = x_test.to(device)\n",
    "            y_test = y_test.to(device)\n",
    "            s_true = s_true.to(device)\n",
    "            model.eval()\n",
    "            if grl_lambda is not None and grl_lambda != 0:\n",
    "                yhat, s_hat = model(x_test)\n",
    "                test_loss = (criterion(y_test, yhat) + criterion_bias(s_true, s_hat.argmax(dim=1))).item()\n",
    "                test_losses.append(val_loss)\n",
    "                test_results.append({\"y_hat\": yhat, \"y_true\": ytrue, \"y_compas\": y_test, \"s\": s_true, \"s_hat\": s_hat})\n",
    "            else:\n",
    "                yhat = model(x_test)\n",
    "                test_loss = (criterion(y_test, yhat)).item()\n",
    "                test_losses.append(val_loss)\n",
    "                test_results.append({\"y_hat\": yhat, \"y_true\": ytrue, \"y_compas\": y_test, \"s\": s_true})\n",
    "\n",
    "        # print({\"Test loss\": np.mean(test_losses)})\n",
    "\n",
    "    results = test_results[0]['y_hat']\n",
    "    outcome = test_results[0]['y_true']\n",
    "    compas = test_results[0]['y_compas']\n",
    "    protected_results = test_results[0]['s']\n",
    "    if grl_lambda is not None and grl_lambda != 0:\n",
    "        protected = test_results[0]['s_hat']\n",
    "    for r in test_results[1:]:\n",
    "        results = torch.cat((results, r['y_hat']))\n",
    "        outcome = torch.cat((outcome, r['y_true']))\n",
    "        compas = torch.cat((compas, r['y_compas']))\n",
    "        protected_results = torch.cat((protected_results, r['s']))\n",
    "        if grl_lambda is not None and grl_lambda != 0:\n",
    "            protected = torch.cat((protected, r['s_hat']))\n",
    "\n",
    "    df = pd.DataFrame(data=results.cpu().numpy(), columns=['pred'])\n",
    "\n",
    "    df['true'] = outcome.cpu().numpy()\n",
    "    df['compas'] = compas.cpu().numpy()\n",
    "    df['race'] = protected_results.cpu().numpy()[:, 0]\n",
    "    if grl_lambda is not None and grl_lambda != 0:\n",
    "        df['race_hat'] = protected.cpu().numpy()[:, 0]\n",
    "\n",
    "    return model, df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_evaluate_drop(adv_loader: DataLoader,\n",
    "                            benign_loader: DataLoader,\n",
    "                            val_loader: DataLoader,\n",
    "                            test_loader: DataLoader,\n",
    "                            device,\n",
    "                            input_shape,\n",
    "                            grl_lambda=None,\n",
    "                            model=None,\n",
    "                            dataset='compas'):\n",
    "    \"\"\"\n",
    "\n",
    "    :param train_loader: Pytorch-like DataLoader with training data.\n",
    "    :param val_loader: Pytorch-like DataLoader with validation data.\n",
    "    :param test_loader: Pytorch-like DataLoader with testing data.\n",
    "    :param device: The target device for the training.\n",
    "    :return: A tuple: (trained Pytorch-like model, dataframe with results on test set)\n",
    "    \"\"\"\n",
    "\n",
    "#     torch.manual_seed(0)\n",
    "\n",
    "#     grl_lambda = 50\n",
    "    epochs = 50\n",
    "\n",
    "    if model is None:\n",
    "        if dataset=='CENSUS':\n",
    "            model = Net_CENSUS(input_shape=input_shape, grl_lambda=grl_lambda).to(device)\n",
    "        else:\n",
    "            model = Net(input_shape=input_shape, grl_lambda=grl_lambda).to(device)\n",
    "    model = model.to(device)\n",
    "    criterion = nn.MSELoss().to(device)\n",
    "    criterion_bias = nn.CrossEntropyLoss().to(device)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=1e-2)\n",
    "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, threshold=0.3, cooldown=5)\n",
    "\n",
    "    training_losses = []\n",
    "    validation_losses = []\n",
    "\n",
    "    t_prog = trange(epochs, desc='Training neural network', leave=False, position=1, mininterval=5)\n",
    "    # t_prog = trange(50)\n",
    "\n",
    "    for epoch in t_prog:\n",
    "        batch_losses = []\n",
    "        \n",
    "        model.train()\n",
    "        for x_batch, y_batch, _, s_batch in adv_loader:\n",
    "            x_batch = x_batch.to(device)\n",
    "            y_batch = y_batch.to(device)\n",
    "            s_batch = s_batch.to(device)\n",
    "\n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # forward + backward + optimize\n",
    "            if grl_lambda is not None and grl_lambda != 0:\n",
    "                outputs, outputs_protected = model(x_batch)\n",
    "                loss = criterion(outputs, y_batch) + criterion_bias(outputs_protected, s_batch.argmax(dim=1))\n",
    "            else:\n",
    "                outputs = model(x_batch)\n",
    "                loss = criterion(outputs, y_batch)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            batch_losses.append(loss.item())\n",
    "            \n",
    "        model.eval()\n",
    "        for x_batch, y_batch, _, s_batch in benign_loader:\n",
    "            x_batch = x_batch.to(device)\n",
    "            y_batch = y_batch.to(device)\n",
    "            s_batch = s_batch.to(device)\n",
    "\n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # forward + backward + optimize\n",
    "            if grl_lambda is not None and grl_lambda != 0:\n",
    "                outputs, outputs_protected = model(x_batch)\n",
    "                loss = criterion(outputs, y_batch) + criterion_bias(outputs_protected, s_batch.argmax(dim=1))\n",
    "            else:\n",
    "                outputs = model(x_batch)\n",
    "                loss = criterion(outputs, y_batch)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "        training_loss = np.mean(batch_losses)\n",
    "        training_losses.append(training_loss)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            val_losses = []\n",
    "            for x_val, y_val, _, s_val in val_loader:\n",
    "                x_val = x_val.to(device)\n",
    "                y_val = y_val.to(device)\n",
    "                s_val = s_val.to(device)\n",
    "                model.eval()\n",
    "                if grl_lambda is not None and grl_lambda != 0:\n",
    "                    yhat, s_hat = model(x_val)\n",
    "                    val_loss = (criterion(y_val, yhat) + criterion_bias(s_val, s_hat.argmax(dim=1))).item()\n",
    "                else:\n",
    "                    yhat = model(x_val)\n",
    "                    val_loss = criterion(y_val, yhat).item()\n",
    "                val_losses.append(val_loss)\n",
    "            validation_loss = np.mean(val_losses)\n",
    "            validation_losses.append(validation_loss)\n",
    "\n",
    "            scheduler.step(val_loss)\n",
    "\n",
    "        t_prog.set_postfix({\"epoch\": epoch, \"training_loss\": training_loss,\n",
    "                            \"validation_loss\": validation_loss}, refresh=False)  # print last metrics\n",
    "\n",
    "#     if args.show_graphs:\n",
    "#         plt.plot(range(len(training_losses)), training_losses)\n",
    "#         plt.plot(range(len(validation_losses)), validation_losses)\n",
    "#         # plt.scatter(x_tensor, y_out.detach().numpy())\n",
    "#         plt.ylabel('some numbers')\n",
    "#         plt.show()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        test_losses = []\n",
    "        test_results = []\n",
    "        for x_test, y_test, ytrue, s_true in test_loader:\n",
    "            x_test = x_test.to(device)\n",
    "            y_test = y_test.to(device)\n",
    "            s_true = s_true.to(device)\n",
    "            model.eval()\n",
    "            if grl_lambda is not None and grl_lambda != 0:\n",
    "                yhat, s_hat = model(x_test)\n",
    "                test_loss = (criterion(y_test, yhat) + criterion_bias(s_true, s_hat.argmax(dim=1))).item()\n",
    "                test_losses.append(val_loss)\n",
    "                test_results.append({\"y_hat\": yhat, \"y_true\": ytrue, \"y_compas\": y_test, \"s\": s_true, \"s_hat\": s_hat})\n",
    "            else:\n",
    "                yhat = model(x_test)\n",
    "                test_loss = (criterion(y_test, yhat)).item()\n",
    "                test_losses.append(val_loss)\n",
    "                test_results.append({\"y_hat\": yhat, \"y_true\": ytrue, \"y_compas\": y_test, \"s\": s_true})\n",
    "\n",
    "        # print({\"Test loss\": np.mean(test_losses)})\n",
    "\n",
    "    results = test_results[0]['y_hat']\n",
    "    outcome = test_results[0]['y_true']\n",
    "    compas = test_results[0]['y_compas']\n",
    "    protected_results = test_results[0]['s']\n",
    "    if grl_lambda is not None and grl_lambda != 0:\n",
    "        protected = test_results[0]['s_hat']\n",
    "    for r in test_results[1:]:\n",
    "        results = torch.cat((results, r['y_hat']))\n",
    "        outcome = torch.cat((outcome, r['y_true']))\n",
    "        compas = torch.cat((compas, r['y_compas']))\n",
    "        protected_results = torch.cat((protected_results, r['s']))\n",
    "        if grl_lambda is not None and grl_lambda != 0:\n",
    "            protected = torch.cat((protected, r['s_hat']))\n",
    "\n",
    "    df = pd.DataFrame(data=results.cpu().numpy(), columns=['pred'])\n",
    "\n",
    "    df['true'] = outcome.cpu().numpy()\n",
    "    df['compas'] = compas.cpu().numpy()\n",
    "    df['race'] = protected_results.cpu().numpy()[:, 0]\n",
    "    if grl_lambda is not None and grl_lambda != 0:\n",
    "        df['race_hat'] = protected.cpu().numpy()[:, 0]\n",
    "\n",
    "    return model, df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_sort(net, train_dataset, THETA=1e-3, GAMMA=0.9):\n",
    "    net=net.cpu()\n",
    "    # THETA = 1e-3\n",
    "    path_set_list=[]\n",
    "    for i in (range(len(train_dataset))):\n",
    "        path_set=get_path_set4(net,train_dataset[i][0],GAMMA=GAMMA)\n",
    "        path_set_list.append(path_set)\n",
    "    v=pd.value_counts(path_set_list).rename_axis('pathset').reset_index(name='counts')\n",
    "#     v_list.append(v)\n",
    "    t=tuple(v[v.counts<=max(v.counts[0]*THETA,1)].pathset)\n",
    "    adv_data_idx=[]\n",
    "    for i in range(len(path_set_list)):\n",
    "        if path_set_list[i] in t:\n",
    "            adv_data_idx.append(i)\n",
    "    print(\"frac:{}\".format(len(adv_data_idx)/len(train_dataset)))\n",
    "    return adv_data_idx\n",
    "\n",
    "v_list=[]\n",
    "def sample_sort_test(net, train_dataset, THETA=1e-3, GAMMA=0.9):\n",
    "    net=net.cpu()\n",
    "    # THETA = 1e-3\n",
    "    path_set_list=[]\n",
    "    for i in (range(len(train_dataset))):\n",
    "        path_set=get_path_set4(net,train_dataset[i][0],GAMMA=GAMMA)\n",
    "        path_set_list.append(path_set)\n",
    "    v=pd.value_counts(path_set_list).rename_axis('pathset').reset_index(name='counts')\n",
    "    v_list.append(v)\n",
    "    t=tuple(v[v.counts<=max(v.counts[0]*THETA,1)].pathset)\n",
    "    adv_data_idx=[]\n",
    "    for i in range(len(path_set_list)):\n",
    "        if path_set_list[i] in t:\n",
    "            adv_data_idx.append(i)\n",
    "    print(\"frac:{}\".format(len(adv_data_idx)/len(train_dataset)))\n",
    "    return adv_data_idx\n",
    "\n",
    "def get_adv(train_dataset,adv_data_idx):\n",
    "    x_t_adv, y_t_adv, l_t_adv, s_t_adv = (None,None,None,None)\n",
    "    for i in range(len(train_dataset)):\n",
    "        if i in adv_data_idx:\n",
    "            a,b,c,d=train_dataset[i]\n",
    "            x_t_adv = a.unsqueeze(0) if x_t_adv is None else torch.cat((x_t_adv,a.unsqueeze(0)),0)\n",
    "            y_t_adv = b.unsqueeze(0) if y_t_adv is None else torch.cat((y_t_adv,b.unsqueeze(0)),0)\n",
    "            l_t_adv = c.unsqueeze(0) if l_t_adv is None else torch.cat((l_t_adv,c.unsqueeze(0)),0)\n",
    "            s_t_adv = d.unsqueeze(0) if s_t_adv is None else torch.cat((s_t_adv,d.unsqueeze(0)),0)\n",
    "    x_t_benign, y_t_benign, l_t_benign, s_t_benign = (None,None,None,None)\n",
    "    for i in range(len(train_dataset)):\n",
    "        if i not in adv_data_idx:\n",
    "            a,b,c,d=train_dataset[i]\n",
    "            x_t_benign = a.unsqueeze(0) if x_t_benign is None else torch.cat((x_t_benign,a.unsqueeze(0)),0)\n",
    "            y_t_benign = b.unsqueeze(0) if y_t_benign is None else torch.cat((y_t_benign,b.unsqueeze(0)),0)\n",
    "            l_t_benign = c.unsqueeze(0) if l_t_benign is None else torch.cat((l_t_benign,c.unsqueeze(0)),0)\n",
    "            s_t_benign = d.unsqueeze(0) if s_t_benign is None else torch.cat((s_t_benign,d.unsqueeze(0)),0)\n",
    "\n",
    "    adv_dataset = TensorDataset(x_t_adv, y_t_adv, l_t_adv, s_t_adv)\n",
    "    benign_dataset = TensorDataset(x_t_benign, y_t_benign, l_t_benign, s_t_benign)\n",
    "\n",
    "    adv_loader = DataLoader(dataset=adv_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    benign_loader = DataLoader(dataset=benign_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    return adv_loader,benign_loader\n",
    "\n",
    "def get_adv_rand(train_dataset,adv_data_idx):\n",
    "    adv_data_idx=random.choices(range(0,len(train_dataset)),k=len(adv_data_idx))\n",
    "    x_t_adv, y_t_adv, l_t_adv, s_t_adv = (None,None,None,None)\n",
    "    for i in range(len(train_dataset)):\n",
    "        if i in adv_data_idx:\n",
    "            a,b,c,d=train_dataset[i]\n",
    "            x_t_adv = a.unsqueeze(0) if x_t_adv is None else torch.cat((x_t_adv,a.unsqueeze(0)),0)\n",
    "            y_t_adv = b.unsqueeze(0) if y_t_adv is None else torch.cat((y_t_adv,b.unsqueeze(0)),0)\n",
    "            l_t_adv = c.unsqueeze(0) if l_t_adv is None else torch.cat((l_t_adv,c.unsqueeze(0)),0)\n",
    "            s_t_adv = d.unsqueeze(0) if s_t_adv is None else torch.cat((s_t_adv,d.unsqueeze(0)),0)\n",
    "    x_t_benign, y_t_benign, l_t_benign, s_t_benign = (None,None,None,None)\n",
    "    for i in range(len(train_dataset)):\n",
    "        if i not in adv_data_idx:\n",
    "            a,b,c,d=train_dataset[i]\n",
    "            x_t_benign = a.unsqueeze(0) if x_t_benign is None else torch.cat((x_t_benign,a.unsqueeze(0)),0)\n",
    "            y_t_benign = b.unsqueeze(0) if y_t_benign is None else torch.cat((y_t_benign,b.unsqueeze(0)),0)\n",
    "            l_t_benign = c.unsqueeze(0) if l_t_benign is None else torch.cat((l_t_benign,c.unsqueeze(0)),0)\n",
    "            s_t_benign = d.unsqueeze(0) if s_t_benign is None else torch.cat((s_t_benign,d.unsqueeze(0)),0)\n",
    "\n",
    "    adv_dataset = TensorDataset(x_t_adv, y_t_adv, l_t_adv, s_t_adv)\n",
    "    benign_dataset = TensorDataset(x_t_benign, y_t_benign, l_t_benign, s_t_benign)\n",
    "\n",
    "    adv_loader = DataLoader(dataset=adv_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    benign_loader = DataLoader(dataset=benign_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    return adv_loader,benign_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def EA(net, attack_size, iter_num, dataset='compas'):\n",
    "    model=net\n",
    "    EA_start=time.time()\n",
    "    t_main=trange(10,desc=\"Attack\", leave=False, position=0)\n",
    "    global train_loader, x_train_tensor, y_train_tensor, l_train_tensor, s_train_tensor\n",
    "    for i in range(iter_num):\n",
    "\n",
    "        model, results = train_and_evaluate(train_loader, val_loader, test_loader, device,\n",
    "                                            input_shape=x_tensor.shape[1], model=model)\n",
    "        print(i)\n",
    "        result = get_metrics(results, threshold, fraction=(attack_size)/(base_size * 7), dataset=dataset)\n",
    "        t_main.set_postfix(result)\n",
    "        global_results.append(result)    \n",
    "        result_pts, result_class, labels = attack_keras_model(\n",
    "                CArray(x_train_tensor),\n",
    "                Y=CArray((y_train_tensor[:, 0] > threshold).int()),\n",
    "                S=s_train_tensor,\n",
    "                nb_attack=10)\n",
    "        print('attack_done!')\n",
    "        result_pts = torch.tensor(np.around(result_pts.astype(np.float32), decimals=3)).clamp(0.0, 1.0)\n",
    "        result_pts[result_pts != result_pts] = 0.0\n",
    "        result_class[result_class != result_class] = 0.0\n",
    "\n",
    "        x_train_tensor = torch.cat((x_train_tensor, result_pts))\n",
    "        y_train_tensor = torch.cat(\n",
    "            (y_train_tensor, torch.tensor(result_class.reshape(-1, 1).astype(np.float32)).clamp(0, 10)))\n",
    "        l_train_tensor = torch.cat((l_train_tensor, torch.tensor(labels.tondarray().reshape(-1, 1).astype(np.float32))))\n",
    "        s = np.random.randint(2, size=len(result_class))\n",
    "        s_train_tensor = torch.cat((s_train_tensor, torch.tensor(np.array([s, 1 - s]).T.astype(np.float64))))\n",
    "\n",
    "        train_dataset = TensorDataset(x_train_tensor, y_train_tensor, l_train_tensor, s_train_tensor)\n",
    "        train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "        logging.debug(\"New training dataset has size {} (original {}).\".format(len(train_loader), base_size * 7))\n",
    "        EA_mid=time.time()\n",
    "        cost_time=EA_mid-EA_start\n",
    "        print('time costs:{} s'.format(cost_time))\n",
    "\n",
    "    EA_end=time.time()\n",
    "    cost_time=EA_end-EA_start\n",
    "    print('time costs:{} s'.format(cost_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Fixate(THETA=1e-3,GAMMA=0.9,epoch=10,dataset='compas'):\n",
    "    our_start=time.time()\n",
    "    for i in range(epoch):\n",
    "        adv_data_idx = sample_sort(net,train_dataset,THETA,GAMMA)\n",
    "        adv_loader, benign_loader = get_adv(train_dataset,adv_data_idx)\n",
    "        net_drop, results = train_and_evaluate_drop(adv_loader, benign_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                                grl_lambda=0)\n",
    "\n",
    "        result = get_metrics(results, threshold, 0, dataset=dataset)\n",
    "        global_results.append(result)\n",
    "\n",
    "    our_end=time.time()\n",
    "    cost_time=our_end-our_start\n",
    "    print('time costs:{} s'.format(cost_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_function(config):\n",
    "    \n",
    "    THETA, GAMMA = config['THETA'], config['GAMMA']\n",
    "    train_dataset_s=config['train_dataset_s']\n",
    "    val_loader_s=config['val']\n",
    "    test_loader_s=config['test']\n",
    "    x_train_tensor_s=config['x_tensor']\n",
    "\n",
    "    adv_data_idx = sample_sort(net,train_dataset_s,THETA,GAMMA)\n",
    "    adv_loader, benign_loader = get_adv(train_dataset_s,adv_data_idx)\n",
    "    net_drop, results = train_and_evaluate_drop(adv_loader, benign_loader, val_loader_s, test_loader_s, device, input_shape=x_train_tensor_s.shape[1],\n",
    "                                            grl_lambda=0,dataset=config['dataset'])\n",
    "    result = get_metrics(results, threshold, 0,dataset=config['dataset'])\n",
    "    complex_score = result['DP']+result['EO']+(1-result['DP ratio'])-0.01*result['acc']\n",
    "    tune.report(mean_loss=complex_score)\n",
    "    \n",
    "    global_results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_function_rand(config):\n",
    "    \n",
    "    THETA, GAMMA = config['THETA'], config['GAMMA']\n",
    "    train_dataset_s=config['train_dataset_s']\n",
    "    val_loader_s=config['val']\n",
    "    test_loader_s=config['test']\n",
    "    x_train_tensor_s=config['x_tensor']\n",
    "\n",
    "    adv_data_idx = sample_sort(net,train_dataset_s,THETA,GAMMA)\n",
    "    adv_loader, benign_loader = get_adv_rand(train_dataset_s,adv_data_idx)\n",
    "    net_drop, results = train_and_evaluate_drop(adv_loader, benign_loader, val_loader_s, test_loader_s, device, input_shape=x_train_tensor_s.shape[1],\n",
    "                                            grl_lambda=0,dataset=config['dataset'])\n",
    "    result = get_metrics(results, threshold, 0,dataset=config['dataset'])\n",
    "    complex_score = result['DP']+result['EO']+(1-result['DP ratio'])-0.01*result['acc']\n",
    "    tune.report(mean_loss=complex_score)\n",
    "    \n",
    "    global_results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Fixate_with_val(epoch=10,dataset='compas'):\n",
    "    our_start=time.time()\n",
    "    base_size = len(val_dataset) // 10\n",
    "    split = [8 * base_size, 1 * base_size, len(val_dataset) - 9 * base_size]  # Train, validation, test\n",
    "    train_dataset_s, val_dataset_s, test_dataset_s = random_split(val_dataset, split)\n",
    "#     print(train_dataset_s)\n",
    "    \n",
    "#     train_loader_s = DataLoader(dataset=train_dataset_s, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    val_loader_s = DataLoader(dataset=val_dataset_s, batch_size=BATCH_SIZE)\n",
    "    test_loader_s = DataLoader(dataset=test_dataset_s, batch_size=BATCH_SIZE)\n",
    "\n",
    "    x_train_tensor_s = val_dataset[:][0]\n",
    "\n",
    "\n",
    "    analysis = tune.run(\n",
    "        training_function,\n",
    "        config={\n",
    "            'THETA': tune.grid_search([0.1, 0.01, 3e-3, 1e-3, 3e-4, 1e-4]),\n",
    "            'GAMMA': tune.grid_search([0.95, 0.9, 0.85, 0.8, 0.7, 0.6]),\n",
    "            'dataset':dataset,\n",
    "            'train_dataset_s':train_dataset_s,\n",
    "            'val':val_loader_s,\n",
    "            'test':test_loader_s,\n",
    "            'x_tensor':x_train_tensor_s\n",
    "        },\n",
    "        resources_per_trial={\n",
    "            \"cpu\": 16,\n",
    "            \"gpu\": 2,\n",
    "        }\n",
    "    )\n",
    "    best_config=analysis.get_best_config(metric=\"mean_loss\", mode=\"min\")\n",
    "    print(\"Best config: \",best_config)\n",
    "    THETA = best_config['THETA']\n",
    "    GAMMA = best_config['GAMMA']\n",
    "    val_end=time.time()\n",
    "    for i in range(epoch):\n",
    "        adv_data_idx = sample_sort(net,train_dataset,THETA,GAMMA)\n",
    "        PA_end=time.time()\n",
    "        adv_loader, benign_loader = get_adv(train_dataset,adv_data_idx)\n",
    "        SS_end=time.time()\n",
    "        net_drop, results = train_and_evaluate_drop(adv_loader, benign_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                                grl_lambda=0,dataset=dataset)\n",
    "        Dropout_end=time.time()\n",
    "        result = get_metrics(results, threshold, 0, dataset=dataset)\n",
    "        global_results.append(result)\n",
    "\n",
    "    our_end=time.time()\n",
    "    cost_time=our_end-our_start\n",
    "    val_time=val_end-our_start\n",
    "    PA_time=PA_end-val_end\n",
    "    SS_time=SS_end-PA_end\n",
    "    Dropout_time=Dropout_end-SS_end\n",
    "    print('param selection costs:{} s'.format(val_time))\n",
    "    print('path analysis costs:{} s'.format(PA_time))\n",
    "    print('sample separation costs:{} s'.format(SS_time))\n",
    "    print('partial dropout training costs:{} s'.format(Dropout_time))\n",
    "    print('total time costs:{} s'.format(cost_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Fixate_with_val_rand(epoch=10,dataset='compas'):\n",
    "    our_start=time.time()\n",
    "    base_size = len(val_dataset) // 10\n",
    "    split = [8 * base_size, 1 * base_size, len(val_dataset) - 9 * base_size]  # Train, validation, test\n",
    "    train_dataset_s, val_dataset_s, test_dataset_s = random_split(val_dataset, split)\n",
    "#     print(train_dataset_s)\n",
    "    \n",
    "#     train_loader_s = DataLoader(dataset=train_dataset_s, batch_size=BATCH_SIZE, shuffle=True)\n",
    "    val_loader_s = DataLoader(dataset=val_dataset_s, batch_size=BATCH_SIZE)\n",
    "    test_loader_s = DataLoader(dataset=test_dataset_s, batch_size=BATCH_SIZE)\n",
    "\n",
    "    x_train_tensor_s = val_dataset[:][0]\n",
    "\n",
    "\n",
    "    analysis = tune.run(\n",
    "        training_function_rand,\n",
    "        config={\n",
    "            'THETA': tune.grid_search([0.1, 0.01, 3e-3, 1e-3, 3e-4, 1e-4]),\n",
    "            'GAMMA': tune.grid_search([0.95, 0.9, 0.85, 0.8, 0.7, 0.6]),\n",
    "            'dataset':dataset,\n",
    "            'train_dataset_s':train_dataset_s,\n",
    "            'val':val_loader_s,\n",
    "            'test':test_loader_s,\n",
    "            'x_tensor':x_train_tensor_s\n",
    "        },\n",
    "        resources_per_trial={\n",
    "            \"cpu\": 8,\n",
    "            \"gpu\": 2,\n",
    "        }\n",
    "    )\n",
    "\n",
    "    best_config=analysis.get_best_config(metric=\"mean_loss\", mode=\"min\")\n",
    "    print(\"Best config: \",best_config)\n",
    "    THETA = best_config['THETA']\n",
    "    GAMMA = best_config['GAMMA']\n",
    "    for i in range(epoch):\n",
    "        adv_data_idx = sample_sort(net,train_dataset,THETA,GAMMA)\n",
    "        adv_loader, benign_loader = get_adv_rand(train_dataset,adv_data_idx)\n",
    "        net_drop, results = train_and_evaluate_drop(adv_loader, benign_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                                grl_lambda=0,dataset=dataset)\n",
    "\n",
    "        result = get_metrics(results, threshold, 0, dataset=dataset)\n",
    "        global_results.append(result)\n",
    "\n",
    "    our_end=time.time()\n",
    "    cost_time=our_end-our_start\n",
    "    print('time costs:{} s'.format(cost_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "BATCH_SIZE=128\n",
    "\n",
    "df=pd.read_csv('data/COMPAS/compas_recidive_two_years_sanitize_age_category_jail_time_decile_score.csv')\n",
    "df_binary, Y, S, Y_true = transform_dataset(df)\n",
    "Y = Y.to_numpy()\n",
    "print(np.mean(Y))\n",
    "\n",
    "l_tensor = torch.tensor(Y_true.to_numpy().reshape(-1, 1).astype(np.float32))\n",
    "x_tensor = torch.tensor(df_binary.to_numpy().astype(np.float32))\n",
    "y_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "s_tensor = torch.tensor(preprocessing.OneHotEncoder().fit_transform(np.array(S).reshape(-1, 1)).toarray())\n",
    "\n",
    "dataset = TensorDataset(x_tensor, y_tensor, l_tensor, s_tensor)  # dataset = CustomDataset(x_tensor, y_tensor)\n",
    "\n",
    "base_size = len(dataset) // 10\n",
    "split = [7 * base_size, 1 * base_size, len(dataset) - 8 * base_size]  # Train, validation, test\n",
    "\n",
    "train_dataset, val_dataset, test_dataset = random_split(dataset, split)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "# train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE)\n",
    "val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE)\n",
    "\n",
    "x_train_tensor = train_dataset[:][0]\n",
    "y_train_tensor = train_dataset[:][1]\n",
    "l_train_tensor = train_dataset[:][2]\n",
    "s_train_tensor = train_dataset[:][3]\n",
    "\n",
    "global_results = []\n",
    "\n",
    "# get the classification threshold, we use the same scale for compas so 4 instead of 0.5\n",
    "ori_start=time.time()\n",
    "threshold = 4\n",
    "\n",
    "net, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0)\n",
    "ori_end=time.time()\n",
    "ori_cost_time=ori_end-ori_start\n",
    "print('time costs:{} s'.format(ori_cost_time))\n",
    "\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "\n",
    "# EA\n",
    "# EA(net,attack_size=10, iter_num=50)\n",
    "\n",
    "for THETA in list(np.logspace(-0.01,-5,50)):\n",
    "    Fixate(THETA=THETA,GAMMA=0.95,epoch=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(global_results)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from itertools import product\n",
    "\n",
    "# param = dict(\n",
    "#     THETA = [0.1, 0.01, 3e-3, 1e-3, 3e-4, 1e-4],\n",
    "#     GAMMA = [0.95, 0.9, 0.85, 0.8, 0.75, 0.7]\n",
    "#     )\n",
    "\n",
    "# param_values = [v for v in param.values()]\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "BATCH_SIZE=128\n",
    "\n",
    "df=pd.read_csv('data/COMPAS/compas_recidive_two_years_sanitize_age_category_jail_time_decile_score.csv')\n",
    "df_binary, Y, S, Y_true = transform_dataset(df)\n",
    "Y = Y.to_numpy()\n",
    "print(np.mean(Y))\n",
    "\n",
    "l_tensor = torch.tensor(Y_true.to_numpy().reshape(-1, 1).astype(np.float32))\n",
    "x_tensor = torch.tensor(df_binary.to_numpy().astype(np.float32))\n",
    "y_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "s_tensor = torch.tensor(preprocessing.OneHotEncoder().fit_transform(np.array(S).reshape(-1, 1)).toarray())\n",
    "\n",
    "dataset = TensorDataset(x_tensor, y_tensor, l_tensor, s_tensor)  # dataset = CustomDataset(x_tensor, y_tensor)\n",
    "\n",
    "base_size = len(dataset) // 10\n",
    "split = [7 * base_size, 1 * base_size, len(dataset) - 8 * base_size]  # Train, validation, test\n",
    "\n",
    "train_dataset, val_dataset, test_dataset = random_split(dataset, split)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "# train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE)\n",
    "val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE)\n",
    "\n",
    "x_train_tensor = train_dataset[:][0]\n",
    "y_train_tensor = train_dataset[:][1]\n",
    "l_train_tensor = train_dataset[:][2]\n",
    "s_train_tensor = train_dataset[:][3]\n",
    "\n",
    "global_results = []\n",
    "\n",
    "# get the classification threshold, we use the same scale for compas so 4 instead of 0.5\n",
    "ori_start=time.time()\n",
    "threshold = 4\n",
    "\n",
    "net, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0)\n",
    "ori_end=time.time()\n",
    "ori_cost_time=ori_end-ori_start\n",
    "print('time costs:{} s'.format(ori_cost_time))\n",
    "\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "\n",
    "# EA\n",
    "# EA(net,attack_size=10, iter_num=50)\n",
    "\n",
    "for THETA in list(np.logspace(-0.01,-5,50)):\n",
    "    Fixate(THETA=THETA,GAMMA=0.95,epoch=3)\n",
    "for GAMMA in list(np.linspace(1,0.5,50)):\n",
    "    Fixate(THETA=0.01,GAMMA=GAMMA,epoch=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(global_results)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EA_ablation={'acc':0.759,\n",
    "            'DP':0.095,\n",
    "            'EO':0.095,\n",
    "            'DP ratio':1.203}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_df=pd.DataFrame(columns=['Theta','Gamma','acc','DP','EO','DP ratio'])\n",
    "epoch=3\n",
    "i=0\n",
    "for GAMMA in list(np.linspace(1,0.5,50)):\n",
    "    start=1+i*epoch\n",
    "    end=start+epoch\n",
    "    dic={'Theta':0.0003,\n",
    "         'Gamma':GAMMA,\n",
    "         'acc':(df['acc'].iloc[start:end]).mean(),\n",
    "         'DP':(df['DP'].iloc[start:end]).mean(),\n",
    "         'EO':(df['EO'].iloc[start:end]).mean(),\n",
    "         'DP ratio':(df['DP ratio'].iloc[start:end]).mean()}\n",
    "    ablation_df=ablation_df.append(dic, ignore_index=True)\n",
    "    i+=1\n",
    "ablation_df.to_csv('data/results/ablation_df_gamma')\n",
    "ablation_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_df=pd.read_csv('data/results/ablation_df_gamma')\n",
    "metric='EO'\n",
    "plt.figure(figsize=(7, 5))\n",
    "plt.tick_params(labelsize=14)\n",
    "# for GAMMA in list(np.linspace(1,0.5,50)):\n",
    "plt.plot((np.linspace(1,0.5,50)),(df.iloc[0][metric])*np.ones(50),'r-.',label='Naive baseline')\n",
    "plt.plot((np.linspace(1,0.5,50)),(EA_ablation[metric])*np.ones(50),'g-+',label='Ethical Adversaries')\n",
    "plt.plot((np.linspace(1,0.5,50)),ablation_df[metric],'bo-',label='FairNeuron')\n",
    "plt.plot((np.linspace(1,0.5,50)),ablation_df[metric][27]*np.ones(50),'b-.',label='best param')\n",
    "\n",
    "plt.xlabel('gamma',myfont)\n",
    "plt.ylabel(metric,myfont)\n",
    "# plt.ylim((0,1))\n",
    "# legend=plt.legend()\n",
    "plt.grid(linestyle='-.')\n",
    "plt.savefig('data/results/Figures/ablation_gamma_{}.pdf'.format(metric))\n",
    "print(legend.figure)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_legend(legend, filename=\"data/results/Figures/legend.pdf\", expand=[-5,-5,5,5]):\n",
    "    fig  = legend.figure\n",
    "    fig.canvas.draw()\n",
    "    bbox  = legend.get_window_extent()\n",
    "    bbox = bbox.from_extents(*(bbox.extents + np.array(expand)))\n",
    "    bbox = bbox.transformed(fig.dpi_scale_trans.inverted())\n",
    "    fig.savefig(filename, dpi=\"figure\", bbox_inches=bbox)\n",
    "export_legend(legend)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ablation_df=pd.DataFrame(columns=['Theta','Gamma','acc','DP','EO','DP ratio'])\n",
    "epoch=3\n",
    "i=0\n",
    "for THETA in list(np.logspace(-0.01,-5,50)):\n",
    "    start=151+i*epoch\n",
    "    end=start+epoch\n",
    "    dic={'Theta':THETA,\n",
    "         'Gamma':0.95,\n",
    "         'acc':(df['acc'].iloc[start:end]).mean(),\n",
    "         'DP':(df['DP'].iloc[start:end]).mean(),\n",
    "         'EO':(df['EO'].iloc[start:end]).mean(),\n",
    "         'DP ratio':(df['DP ratio'].iloc[start:end]).mean()}\n",
    "    ablation_df=ablation_df.append(dic, ignore_index=True)\n",
    "    i+=1\n",
    "ablation_df.to_csv('data/results/ablation_df_theta')\n",
    "ablation_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ablation_df=pd.read_csv('data/results/ablation_df_theta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric='acc'\n",
    "plt.figure(figsize=(7, 5))\n",
    "plt.tick_params(labelsize=14)\n",
    "# for GAMMA in list(np.linspace(1,0.5,50)):\n",
    "plt.plot(np.log10(np.logspace(-0.01,-5,50)),df.iloc[0][metric]*np.ones(50),'r-.',label='Naive baseline')\n",
    "plt.plot(np.log10(np.logspace(-0.01,-5,50)),EA_ablation[metric]*np.ones(50),'g-+',label='Ethical Adversaries')\n",
    "plt.plot(np.log10(np.logspace(-0.01,-5,50)),ablation_df[metric],'bo-',label='FairNeuron')\n",
    "plt.plot(np.log10(np.logspace(-0.01,-5,50)),ablation_df[metric][38]*np.ones(50),'b-.',label='best param')\n",
    "plt.xlabel('lg theta',myfont)\n",
    "plt.ylabel(metric,myfont)\n",
    "# plt.ylim((0,1))\n",
    "# plt.legend()\n",
    "plt.grid(linestyle='-.')\n",
    "plt.savefig('data/results/Figures/ablation_theta_{}.pdf'.format(metric))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "BATCH_SIZE=128\n",
    "\n",
    "df=pd.read_csv('data/COMPAS/compas_recidive_two_years_sanitize_age_category_jail_time_decile_score.csv')\n",
    "df_binary, Y, S, Y_true = transform_dataset(df)\n",
    "Y = Y.to_numpy()\n",
    "print(np.mean(Y))\n",
    "\n",
    "l_tensor = torch.tensor(Y_true.to_numpy().reshape(-1, 1).astype(np.float32))\n",
    "x_tensor = torch.tensor(df_binary.to_numpy().astype(np.float32))\n",
    "y_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "s_tensor = torch.tensor(preprocessing.OneHotEncoder().fit_transform(np.array(S).reshape(-1, 1)).toarray())\n",
    "\n",
    "dataset = TensorDataset(x_tensor, y_tensor, l_tensor, s_tensor)  # dataset = CustomDataset(x_tensor, y_tensor)\n",
    "\n",
    "base_size = len(dataset) // 10\n",
    "split = [7 * base_size, 1 * base_size, len(dataset) - 8 * base_size]  # Train, validation, test\n",
    "\n",
    "train_dataset, val_dataset, test_dataset = random_split(dataset, split)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "# train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE)\n",
    "val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE)\n",
    "\n",
    "x_train_tensor = train_dataset[:][0]\n",
    "y_train_tensor = train_dataset[:][1]\n",
    "l_train_tensor = train_dataset[:][2]\n",
    "s_train_tensor = train_dataset[:][3]\n",
    "\n",
    "global_results = []\n",
    "\n",
    "# get the classification threshold, we use the same scale for compas so 4 instead of 0.5\n",
    "ori_start=time.time()\n",
    "threshold = 4\n",
    "\n",
    "net, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0)\n",
    "\n",
    "\n",
    "ori_end=time.time()\n",
    "ori_cost_time=ori_end-ori_start\n",
    "print('time costs:{} s'.format(ori_cost_time))\n",
    "\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "net_nodrop, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0,dataset='compas_nodrop')\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "\n",
    "# EA\n",
    "# EA(net,attack_size=10, iter_num=50)\n",
    "\n",
    "Fixate_with_val(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v_list=[]\n",
    "\n",
    "sample_sort_test(net,train_dataset,0.01,0.6)\n",
    "v_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total=path_stat.sum()\n",
    "cum=[]\n",
    "s=0\n",
    "for i in range(1,path_stat[0]+1):\n",
    "    s=s+i*(path_stat==i).sum()\n",
    "    cum.append(s.copy())\n",
    "cum=np.array(cum)/s\n",
    "cum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_stat=np.array(v_list[0].counts)\n",
    "plt.figure(figsize=(7, 5))\n",
    "plt.tick_params(labelsize=14)\n",
    "# for GAMMA in list(np.linspace(1,0.5,50)):\n",
    "# plt.plot(range(0,len(path_stat)),path_stat,'r-.')\n",
    "# plt.plot(range(0,len(path_stat)),(path_stat[0]*0.01)*np.ones(len(path_stat)),'g-.')\n",
    "plt.hist(path_stat,bins=40,density=1, histtype='step',label='Path activation statistics PDF')\n",
    "# plt.hist(path_stat,bins=40,density=1,cumulative=1, histtype='step',label='Cumulative Distribution Function')\n",
    "plt.plot(range(0,len(cum)),cum,'ro-',label='Sample cumulative ratio')\n",
    "plt.vlines(47*0.03, 0, 2, colors='g', linestyles='dashed')\n",
    "plt.xlabel('path activation statistics',myfont)\n",
    "plt.ylabel('ratio',myfont)\n",
    "plt.ylim((0,1.01))\n",
    "plt.legend(loc='lower right')\n",
    "plt.grid(linestyle='-.')\n",
    "plt.savefig('data/results/Figures/detection.pdf')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df=pd.DataFrame(global_results)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# census\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "BATCH_SIZE=128\n",
    "\n",
    "df=pd.read_csv('data/Census/adult')\n",
    "df_binary, Y, S, Y_true = transform_dataset_census(df)\n",
    "print(np.mean(Y))\n",
    "\n",
    "l_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "x_tensor = torch.tensor(df_binary.to_numpy().astype(np.float32))\n",
    "y_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "s_tensor = torch.tensor(preprocessing.OneHotEncoder().fit_transform(np.array(S).reshape(-1, 1)).toarray())\n",
    "\n",
    "dataset = TensorDataset(x_tensor, y_tensor, l_tensor, s_tensor)  # dataset = CustomDataset(x_tensor, y_tensor)\n",
    "\n",
    "base_size = len(dataset) // 10\n",
    "split = [7 * base_size, 1 * base_size, len(dataset) - 8 * base_size]  # Train, validation, test\n",
    "\n",
    "train_dataset, val_dataset, test_dataset = random_split(dataset, split)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "# train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE)\n",
    "val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE)\n",
    "\n",
    "x_train_tensor = train_dataset[:][0]\n",
    "y_train_tensor = train_dataset[:][1]\n",
    "l_train_tensor = train_dataset[:][2]\n",
    "s_train_tensor = train_dataset[:][3]\n",
    "\n",
    "global_results = []\n",
    "\n",
    "# get the classification threshold, we use the same scale for compas so 4 instead of 0.5\n",
    "ori_start=time.time()\n",
    "threshold = 0.5\n",
    "\n",
    "net, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0, dataset='census')\n",
    "ori_end=time.time()\n",
    "ori_cost_time=ori_end-ori_start\n",
    "print('time costs:{} s'.format(ori_cost_time))\n",
    "\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "\n",
    "\n",
    "EA(net,attack_size=10, iter_num=50)\n",
    "\n",
    "Fixate_with_val(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# credit\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "BATCH_SIZE=128\n",
    "\n",
    "df=pd.read_csv('data/Census/adult')\n",
    "df_binary, Y, S, Y_true = transform_dataset_census(df)\n",
    "print(np.mean(Y))\n",
    "\n",
    "l_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "x_tensor = torch.tensor(df_binary.to_numpy().astype(np.float32))\n",
    "y_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "s_tensor = torch.tensor(preprocessing.OneHotEncoder().fit_transform(np.array(S).reshape(-1, 1)).toarray())\n",
    "\n",
    "dataset = TensorDataset(x_tensor, y_tensor, l_tensor, s_tensor)  # dataset = CustomDataset(x_tensor, y_tensor)\n",
    "\n",
    "base_size = len(dataset) // 10\n",
    "split = [7 * base_size, 1 * base_size, len(dataset) - 8 * base_size]  # Train, validation, test\n",
    "\n",
    "train_dataset, val_dataset, test_dataset = random_split(dataset, split)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "# train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE)\n",
    "val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE)\n",
    "\n",
    "x_train_tensor = train_dataset[:][0]\n",
    "y_train_tensor = train_dataset[:][1]\n",
    "l_train_tensor = train_dataset[:][2]\n",
    "s_train_tensor = train_dataset[:][3]\n",
    "\n",
    "global_results = []\n",
    "\n",
    "# get the classification threshold, we use the same scale for compas so 4 instead of 0.5\n",
    "ori_start=time.time()\n",
    "threshold = 0.5\n",
    "\n",
    "net, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0, dataset='census')\n",
    "ori_end=time.time()\n",
    "ori_cost_time=ori_end-ori_start\n",
    "print('time costs:{} s'.format(ori_cost_time))\n",
    "\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "\n",
    "\n",
    "EA(net,attack_size=10, iter_num=50)\n",
    "\n",
    "Fixate_with_val(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# credit\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "BATCH_SIZE=128\n",
    "\n",
    "df=pd.read_csv('data/Credit/german_credit',sep=' ')\n",
    "df_binary, Y, S, Y_true = transform_dataset_credit(df)\n",
    "print(np.mean(Y))\n",
    "\n",
    "l_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "x_tensor = torch.tensor(df_binary.to_numpy().astype(np.float32))\n",
    "y_tensor = torch.tensor(Y.reshape(-1, 1).astype(np.float32))\n",
    "s_tensor = torch.tensor(preprocessing.OneHotEncoder().fit_transform(np.array(S).reshape(-1, 1)).toarray())\n",
    "\n",
    "dataset = TensorDataset(x_tensor, y_tensor, l_tensor, s_tensor)  # dataset = CustomDataset(x_tensor, y_tensor)\n",
    "\n",
    "base_size = len(dataset) // 10\n",
    "split = [7 * base_size, 1 * base_size, len(dataset) - 8 * base_size]  # Train, validation, test\n",
    "\n",
    "train_dataset, val_dataset, test_dataset = random_split(dataset, split)\n",
    "\n",
    "train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
    "# train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE)\n",
    "val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE)\n",
    "test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE)\n",
    "\n",
    "x_train_tensor = train_dataset[:][0]\n",
    "y_train_tensor = train_dataset[:][1]\n",
    "l_train_tensor = train_dataset[:][2]\n",
    "s_train_tensor = train_dataset[:][3]\n",
    "\n",
    "global_results = []\n",
    "\n",
    "# get the classification threshold, we use the same scale for compas so 4 instead of 0.5\n",
    "ori_start=time.time()\n",
    "threshold = 0.5\n",
    "\n",
    "net, results = train_and_evaluate(train_loader, val_loader, test_loader, device, input_shape=x_tensor.shape[1],\n",
    "                                    grl_lambda=0,dataset='credit')\n",
    "ori_end=time.time()\n",
    "ori_cost_time=ori_end-ori_start\n",
    "print('time costs:{} s'.format(ori_cost_time))\n",
    "\n",
    "result = get_metrics(results, threshold, 0)\n",
    "global_results.append(result)\n",
    "\n",
    "\n",
    "EA(net,attack_size=10, iter_num=50)\n",
    "\n",
    "Fixate_with_val(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(global_results)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EA(net,attack_size=10, iter_num=50,dataset='census')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Fixate_with_val(10,dataset='census')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}