refactor notebook examples into tabular, text and vision folders

notebooks/test_notebooks.py

@@ -18,6 +18,8 @@
 INDIVIDUAL_DASHBOARDS = 'individual-dashboards/'
 RESPONSIBLEAIDASHBOARD = 'responsibleaidashboard/'
 TEXT = RESPONSIBLEAIDASHBOARD + 'text/'
+TABULAR = RESPONSIBLEAIDASHBOARD + 'tabular/'
+VISION = RESPONSIBLEAIDASHBOARD + 'vision/'
 ERROR_ANALYSIS_DASHBOARD = INDIVIDUAL_DASHBOARDS + 'erroranalysis-dashboard/'
 EXPLANATION_DASHBOARD = INDIVIDUAL_DASHBOARDS + 'explanation-dashboard/'
 FAIRNESS_DASHBOARD = INDIVIDUAL_DASHBOARDS + 'fairness-dashboard/'
@@ -206,7 +208,7 @@ def test_erroranalysis_dashboard_housing():
 
 @pytest.mark.notebooks
 def test_responsibleaidashboard_census_classification_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = TABULAR
     nb_name = "responsibleaidashboard-census-classification-model-debugging"
 
     test_values = {}
@@ -215,7 +217,7 @@ def test_responsibleaidashboard_census_classification_model_debugging():
 
 @pytest.mark.notebooks
 def test_responsibleaidashboard_diabetes_decision_making():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = TABULAR
     nb_name = "responsibleaidashboard-diabetes-decision-making"
 
     test_values = {}
@@ -224,7 +226,7 @@ def test_responsibleaidashboard_diabetes_decision_making():
 
 @pytest.mark.notebooks
 def test_responsibleaidashboard_diabetes_regression_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = TABULAR
     nb_name = "responsibleaidashboard-diabetes-regression-model-debugging"
 
     test_values = {}
@@ -233,7 +235,7 @@ def test_responsibleaidashboard_diabetes_regression_model_debugging():
 
 @pytest.mark.notebooks
 def test_responsibleaidashboard_housing_classification_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = TABULAR
     nb_name = "responsibleaidashboard-housing-classification-model-debugging"
 
     test_values = {}
@@ -242,7 +244,7 @@ def test_responsibleaidashboard_housing_classification_model_debugging():
 
 @pytest.mark.notebooks
 def test_responsibleaidashboard_housing_decision_making():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = TABULAR
     nb_name = "responsibleaidashboard-housing-decision-making"
 
     test_values = {}
@@ -251,7 +253,7 @@ def test_responsibleaidashboard_housing_decision_making():
 
 @pytest.mark.notebooks
 def test_responsibleaidashboard_multiclass_dnn_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = TABULAR
     nb_name = "responsibleaidashboard-multiclass-dnn-model-debugging"
 
     test_values = {}
@@ -260,7 +262,7 @@ def test_responsibleaidashboard_multiclass_dnn_model_debugging():
 
 @pytest.mark.vision_notebooks
 def test_responsibleaidashboard_fridge_image_classification_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = VISION
     nb_name = (
         "responsibleaidashboard-fridge-"
         "image-classification-model-debugging"
@@ -272,7 +274,7 @@ def test_responsibleaidashboard_fridge_image_classification_model_debugging():
 
 @pytest.mark.vision_notebooks
 def test_responsibleaidashboard_fridge_multilabel_ic_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = VISION
     nb_name = (
         "responsibleaidashboard-fridge-multilabel-"
         "image-classification-model-debugging"
@@ -284,7 +286,7 @@ def test_responsibleaidashboard_fridge_multilabel_ic_model_debugging():
 
 @pytest.mark.vision_notebooks
 def test_responsibleaidashboard_fridge_object_detection_model_debugging():
-    nb_path = RESPONSIBLEAIDASHBOARD
+    nb_path = VISION
     nb_name = "responsibleaidashboard-fridge-object-detection-model-debugging"
 
     test_values = {}

scripts/e2e-widget.js

@@ -7,24 +7,44 @@ const { exit } = require("process");
 
 const nxPath = path.join(__dirname, "../node_modules/@nrwl/cli/bin/nx.js");
 const baseDir = path.join(__dirname, "../notebooks/responsibleaidashboard");
+const tabularDir = path.join(baseDir, "tabular");
+const visionDir = path.join(baseDir, "vision");
 const filePrefix = "responsibleaidashboard-";
-// Please add notebook name into 'fileNames' array only when you are adding e2e tests to that notebook.
+// Please add notebook name into the appropriate 'fileNames' array only when you are adding e2e tests to that notebook.
 // Keep this list in sync with .github/workflows/CI-e2e-notebooks.yml and/or .github/workflows/CI-e2e-notebooks-vision.yml
-const fileNames = [
+const tabularFileNames = [
   "responsibleaidashboard-census-classification-model-debugging",
   "responsibleaidashboard-diabetes-regression-model-debugging",
   "responsibleaidashboard-housing-classification-model-debugging",
   "responsibleaidashboard-diabetes-decision-making",
   "responsibleaidashboard-housing-decision-making",
-  "responsibleaidashboard-multiclass-dnn-model-debugging",
+  "responsibleaidashboard-multiclass-dnn-model-debugging"
+];
+const visionFileNames = [
   "responsibleaidashboard-fridge-image-classification-model-debugging",
   "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging",
   "responsibleaidashboard-fridge-object-detection-model-debugging"
 ];
+const fileNames = tabularFileNames.concat(visionFileNames);
 const notebookHostReg = /^ResponsibleAI started at (http:\/\/localhost:\d+)$/m;
 const serveHostReg = /Web Development Server is listening at\s+(.*)$/m;
 const timeout = 3600;
 
+/**
+ *
+ * @param {string} notebook
+ * @returns {string}
+ */
+function getDirForNotebook(notebook) {
+  if (visionFileNames.includes(notebook)) {
+    return visionDir;
+  } else if (tabularFileNames.includes(notebook)) {
+    return tabularDir;
+  } else {
+    throw new Error(`Notebook ${notebook} not found.`);
+  }
+}
+
 /**
  *
  * @param {string} host
@@ -73,7 +93,8 @@ async function runNotebook(name) {
   const timer = setTimeout(() => {
     throw new Error(`${name} timeout.`);
   }, timeout * 1000);
-  const nbProcess = spawn("python", ["-i", path.join(baseDir, name)]);
+  const dir = getDirForNotebook(name);
+  const nbProcess = spawn("python", ["-i", path.join(dir, name)]);
   nbProcess.on("exit", () => {
     throw new Error(`Failed to run notebook ${name}`);
   });
@@ -131,7 +152,8 @@ function addFlightsInFile(path, flights) {
 function checkIfAllNotebooksHaveTests() {
   console.log(`Checking if all notebooks under ${baseDir} have tests`);
   const files = fs
-    .readdirSync(baseDir)
+    .readdirSync(tabularDir)
+    .concat(fs.readdirSync(visionDir))
     .filter((f) => f.startsWith(filePrefix) && f.endsWith(".ipynb"))
     .map((f) => f.replace(".ipynb", ""));
   const allNotebooksHaveTests = _.isEqual(_.sortBy(files), _.sortBy(fileNames));
@@ -150,6 +172,7 @@ function convertNotebooks(notebook, flights) {
       console.log(`Skipping ${fileName}. Looking for ${notebook} only.`);
       continue;
     }
+    const dir = getDirForNotebook(fileName);
     if (flights) {
       // flights were passed (not just -f without flights arg)
       console.log(
@@ -161,7 +184,7 @@ function convertNotebooks(notebook, flights) {
     }
     const { status, stderr } = spawnSync(
       "jupyter",
-      ["nbconvert", path.join(baseDir, `${fileName}.ipynb`), "--to", "script"],
+      ["nbconvert", path.join(dir, `${fileName}.ipynb`), "--to", "script"],
       {
         stdio: "inherit"
       }
@@ -170,7 +193,7 @@ function convertNotebooks(notebook, flights) {
       throw new Error(`Failed to convert notebook:\r\n\r\n${stderr}`);
     }
     if (flights) {
-      addFlightsInFile(path.join(baseDir, `${fileName}.py`), flights);
+      addFlightsInFile(path.join(dir, `${fileName}.py`), flights);
     }
     console.log(`Converted notebook ${fileName}\r\n`);
   }
@@ -185,7 +208,8 @@ function convertNotebooks(notebook, flights) {
  */
 async function runNotebooks(selectedNotebook, host) {
   let files = fs
-    .readdirSync(baseDir)
+    .readdirSync(tabularDir)
+    .concat(fs.readdirSync(visionDir))
     .filter((f) => f.startsWith(filePrefix) && f.endsWith(".py"));
   console.log("Available notebooks:");
   files.forEach((file) => {