research_questions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "# DATASET IMPORT\n",
    "\n",
    "# Load dataLinux: 10k builds from Linux Testers containing flaky runs and failure runs\n",
    "dataLinux = pd.read_json('dataset.35past.Linux10k.json')\n",
    "\n",
    "# Load dataPass: 2 builds from Linux Testers containing flaky runs, failure runs but more importantly all pass runs\n",
    "dataPass = pd.read_json('dataset.pass.json')\n",
    "\n",
    "# Load nft: List of Passing tests that are never found to be flaky or legit elsewhere\n",
    "file121= open('nft-121.json')\n",
    "file123= open('nft-123.json')\n",
    "nft121 = json.load(file121)\n",
    "nft123 = json.load(file123)\n",
    "file121.close()\n",
    "file123.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pprint import pprint\n",
    "from collections import Counter\n",
    "\n",
    "# Get data about PASS (2), LEGIT (1) and FLAKY (0) run\n",
    "dataPass = dataPass[(dataPass[\"label\"] == 2) & (dataPass[\"testSource\"] != \"\")]\n",
    "dataFlaky = dataLinux[(dataLinux[\"label\"] == 0) & (dataLinux[\"testSource\"] != \"\")]\n",
    "dataLegit = dataLinux[(dataLinux[\"label\"] == 1) & (dataLinux[\"testSource\"] != \"\")]\n",
    "\n",
    "# Keep clean NFT\n",
    "dataPass121 = dataPass[(dataPass[\"buildId\"] == 121238) & (dataPass[\"testId\"].isin(nft121))]\n",
    "dataPass123 = dataPass[(dataPass[\"buildId\"] == 123038) & (dataPass[\"testId\"].isin(nft123))]\n",
    "dataPasses = pd.concat([dataPass121, dataPass123])\n",
    "\n",
    "# Building one set of pass legit and flaky\n",
    "data = pd.concat([dataPasses, dataFlaky, dataLegit])\n",
    "data[\"flakeRate\"] = data[\"flakeRate\"].fillna(0)\n",
    "print(\"Data:\", Counter(data[\"label\"]))\n",
    "\n",
    "# Split 80/20\n",
    "dataTrain = data[data[\"buildId\"] <= 121238]\n",
    "dataTest = data[(data[\"buildId\"] > 121238) & (data[\"buildId\"] <= 123038)]\n",
    "\n",
    "\n",
    "# TRAINING SET (Uncomment the RQ to check, comment the others)\n",
    "# RQ 1\n",
    "dataTrain = dataTrain.drop_duplicates(subset=[\"testSource\", \"label\"], keep='first')\n",
    "dataTrain[\"label\"] = dataTrain[\"label\"].map({0:1, 1:0, 2:0})\n",
    "\n",
    "# RQ 2\n",
    "# dataTrain[\"label\"] = dataTrain[\"label\"].map({0:1, 1:0, 2:0})\n",
    "\n",
    "# RQ 3\n",
    "# dataTrain[\"label\"] = dataTrain[\"label\"].map({0:1, 1:0, 2:0})\n",
    "\n",
    "\n",
    "# TEST SET\n",
    "dataTest = dataTest[(dataTest[\"label\"] == 1) | (dataTest[\"label\"] == 0)]\n",
    "# Adapt labels: Flaky == 1, Legit == 0\n",
    "dataTest[\"label\"] = dataTest[\"label\"].map({0:1, 1:0})\n",
    "\n",
    "print(\"Data Train:\", Counter(dataTrain[\"label\"]))\n",
    "print(\"Data Test:\", Counter(dataTest[\"label\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.preprocessing import OneHotEncoder, KBinsDiscretizer, Binarizer\n",
    "\n",
    "# Columns definition\n",
    "testSource_col = \"testSource\"\n",
    "cat_cols = [\"testSuite\"]\n",
    "std_cols = [\"flakeRate\"]\n",
    "other_cols = [\"runDuration\"]\n",
    "\n",
    "# Feature transformation (Uncomment the RQ to check, comment the others)\n",
    "# RQ1 or RQ2 \n",
    "cols_trans = ColumnTransformer([\n",
    "    ('testSource', CountVectorizer(max_features=100), testSource_col),\n",
    "], remainder='drop')\n",
    "\n",
    "# RQ 3 \n",
    "# cols_trans = ColumnTransformer([\n",
    "#     ('categories', OneHotEncoder(handle_unknown = \"ignore\"), cat_cols),\n",
    "#     ('testSource', CountVectorizer(max_features=100), testSource_col),\n",
    "# ], remainder='passthrough')\n",
    "\n",
    "X_train = dataTrain[other_cols + std_cols + cat_cols + [testSource_col]]\n",
    "X_test = dataTest[other_cols + std_cols + cat_cols + [testSource_col]]\n",
    "\n",
    "y_train = dataTrain[\"label\"]\n",
    "y_test = dataTest[\"label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NORMAL PIPELINE\n",
    "from numpy import mean\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.model_selection import cross_validate, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, matthews_corrcoef, r2_score\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.pipeline import Pipeline\n",
    "from sklearn import set_config\n",
    "set_config(display=\"diagram\")\n",
    "from sklearn.base import TransformerMixin, BaseEstimator\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Pipeline\n",
    "smote = SMOTE(sampling_strategy=0.4)\n",
    "featureSelection = SelectKBest(chi2, k=60)\n",
    "rfc = BalancedRandomForestClassifier(n_estimators=200, n_jobs=14, verbose=1)\n",
    "\n",
    "steps = [\n",
    "    ('trans', cols_trans),\n",
    "#     ('fs', featureSelection),\n",
    "#     ('s', smote), \n",
    "    ('m', rfc)\n",
    "]\n",
    "pipe = Pipeline(steps=steps)\n",
    "display(pipe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit and test\n",
    "pipe.fit(X_train, y_train)\n",
    "y_pred = pipe.predict(X_test)\n",
    "\n",
    "# Scores\n",
    "precision = precision_score(y_test, y_pred)\n",
    "recall = recall_score(y_test, y_pred)\n",
    "mcc = matthews_corrcoef(y_test, y_pred)\n",
    "f1 = f1_score(y_test, y_pred)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "\n",
    "print(\"\\nPrecision\", precision)\n",
    "print(\"Recall\", recall)\n",
    "print(\"MCC\", mcc)\n",
    "print(\"F1\", f1)\n",
    "print(\"R2\", r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics as metrics\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Confusion matrix\n",
    "metrics.plot_confusion_matrix(pipe, X_test, y_test, normalize=None, cmap='Blues', \n",
    "                              display_labels=[\"Non-Flaky\", \"Flaky\"], values_format = '.0f')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Information about false positives\n",
    "fp = np.logical_and(y_test != y_pred, y_pred == 1)\n",
    "tn = np.logical_and(y_test == y_pred, y_test == 0)\n",
    "X_fp = X_test[fp]\n",
    "X_tn = X_test[tn]\n",
    "data_fp = dataTest.loc[X_fp.index]\n",
    "data_fp_fr0 = data_fp[data_fp[\"flakeRate\"] > 0]\n",
    "\n",
    "print(\"Number of FP:\", len(X_fp))\n",
    "print(\"Number of FP:\", len(X_tn))\n",
    "print(\"Number of FP with flake Rate > 0:\", len(data_fp_fr0))\n",
    "print(\"FPR:\", len(X_fp) / (len(X_fp) + len(X_tn)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chromium",
   "language": "python",
   "name": "chromium"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}