diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-census-classification-model-debugging.ipynb b/notebooks/responsibleaidashboard/tabular/responsibleaidashboard-census-classification-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-census-classification-model-debugging.ipynb rename to notebooks/responsibleaidashboard/tabular/responsibleaidashboard-census-classification-model-debugging.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-diabetes-decision-making.ipynb b/notebooks/responsibleaidashboard/tabular/responsibleaidashboard-diabetes-decision-making.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-diabetes-decision-making.ipynb rename to notebooks/responsibleaidashboard/tabular/responsibleaidashboard-diabetes-decision-making.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-diabetes-regression-model-debugging.ipynb b/notebooks/responsibleaidashboard/tabular/responsibleaidashboard-diabetes-regression-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-diabetes-regression-model-debugging.ipynb rename to notebooks/responsibleaidashboard/tabular/responsibleaidashboard-diabetes-regression-model-debugging.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-housing-classification-model-debugging.ipynb b/notebooks/responsibleaidashboard/tabular/responsibleaidashboard-housing-classification-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-housing-classification-model-debugging.ipynb rename to notebooks/responsibleaidashboard/tabular/responsibleaidashboard-housing-classification-model-debugging.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-housing-decision-making.ipynb b/notebooks/responsibleaidashboard/tabular/responsibleaidashboard-housing-decision-making.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-housing-decision-making.ipynb rename to notebooks/responsibleaidashboard/tabular/responsibleaidashboard-housing-decision-making.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-multiclass-dnn-model-debugging.ipynb b/notebooks/responsibleaidashboard/tabular/responsibleaidashboard-multiclass-dnn-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-multiclass-dnn-model-debugging.ipynb rename to notebooks/responsibleaidashboard/tabular/responsibleaidashboard-multiclass-dnn-model-debugging.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-fridge-image-classification-model-debugging.ipynb b/notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-image-classification-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-fridge-image-classification-model-debugging.ipynb rename to notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-image-classification-model-debugging.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.ipynb b/notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.ipynb rename to notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.ipynb diff --git a/notebooks/responsibleaidashboard/responsibleaidashboard-fridge-object-detection-model-debugging.ipynb b/notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-object-detection-model-debugging.ipynb similarity index 100% rename from notebooks/responsibleaidashboard/responsibleaidashboard-fridge-object-detection-model-debugging.ipynb rename to notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-object-detection-model-debugging.ipynb diff --git a/notebooks/test_notebooks.py b/notebooks/test_notebooks.py index 1306a0f403..4d3b6b4f87 100644 --- a/notebooks/test_notebooks.py +++ b/notebooks/test_notebooks.py @@ -18,6 +18,8 @@ INDIVIDUAL_DASHBOARDS = 'individual-dashboards/' RESPONSIBLEAIDASHBOARD = 'responsibleaidashboard/' TEXT = RESPONSIBLEAIDASHBOARD + 'text/' +TABULAR = RESPONSIBLEAIDASHBOARD + 'tabular/' +VISION = RESPONSIBLEAIDASHBOARD + 'vision/' ERROR_ANALYSIS_DASHBOARD = INDIVIDUAL_DASHBOARDS + 'erroranalysis-dashboard/' EXPLANATION_DASHBOARD = INDIVIDUAL_DASHBOARDS + 'explanation-dashboard/' FAIRNESS_DASHBOARD = INDIVIDUAL_DASHBOARDS + 'fairness-dashboard/' @@ -206,7 +208,7 @@ def test_erroranalysis_dashboard_housing(): @pytest.mark.notebooks def test_responsibleaidashboard_census_classification_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = TABULAR nb_name = "responsibleaidashboard-census-classification-model-debugging" test_values = {} @@ -215,7 +217,7 @@ def test_responsibleaidashboard_census_classification_model_debugging(): @pytest.mark.notebooks def test_responsibleaidashboard_diabetes_decision_making(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = TABULAR nb_name = "responsibleaidashboard-diabetes-decision-making" test_values = {} @@ -224,7 +226,7 @@ def test_responsibleaidashboard_diabetes_decision_making(): @pytest.mark.notebooks def test_responsibleaidashboard_diabetes_regression_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = TABULAR nb_name = "responsibleaidashboard-diabetes-regression-model-debugging" test_values = {} @@ -233,7 +235,7 @@ def test_responsibleaidashboard_diabetes_regression_model_debugging(): @pytest.mark.notebooks def test_responsibleaidashboard_housing_classification_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = TABULAR nb_name = "responsibleaidashboard-housing-classification-model-debugging" test_values = {} @@ -242,7 +244,7 @@ def test_responsibleaidashboard_housing_classification_model_debugging(): @pytest.mark.notebooks def test_responsibleaidashboard_housing_decision_making(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = TABULAR nb_name = "responsibleaidashboard-housing-decision-making" test_values = {} @@ -251,7 +253,7 @@ def test_responsibleaidashboard_housing_decision_making(): @pytest.mark.notebooks def test_responsibleaidashboard_multiclass_dnn_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = TABULAR nb_name = "responsibleaidashboard-multiclass-dnn-model-debugging" test_values = {} @@ -260,7 +262,7 @@ def test_responsibleaidashboard_multiclass_dnn_model_debugging(): @pytest.mark.vision_notebooks def test_responsibleaidashboard_fridge_image_classification_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = VISION nb_name = ( "responsibleaidashboard-fridge-" "image-classification-model-debugging" @@ -272,7 +274,7 @@ def test_responsibleaidashboard_fridge_image_classification_model_debugging(): @pytest.mark.vision_notebooks def test_responsibleaidashboard_fridge_multilabel_ic_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = VISION nb_name = ( "responsibleaidashboard-fridge-multilabel-" "image-classification-model-debugging" @@ -284,7 +286,7 @@ def test_responsibleaidashboard_fridge_multilabel_ic_model_debugging(): @pytest.mark.vision_notebooks def test_responsibleaidashboard_fridge_object_detection_model_debugging(): - nb_path = RESPONSIBLEAIDASHBOARD + nb_path = VISION nb_name = "responsibleaidashboard-fridge-object-detection-model-debugging" test_values = {} diff --git a/scripts/e2e-widget.js b/scripts/e2e-widget.js index 1550ed03d2..2ec4dc7169 100644 --- a/scripts/e2e-widget.js +++ b/scripts/e2e-widget.js @@ -7,24 +7,44 @@ const { exit } = require("process"); const nxPath = path.join(__dirname, "../node_modules/@nrwl/cli/bin/nx.js"); const baseDir = path.join(__dirname, "../notebooks/responsibleaidashboard"); +const tabularDir = path.join(baseDir, "tabular"); +const visionDir = path.join(baseDir, "vision"); const filePrefix = "responsibleaidashboard-"; -// Please add notebook name into 'fileNames' array only when you are adding e2e tests to that notebook. +// Please add notebook name into the appropriate 'fileNames' array only when you are adding e2e tests to that notebook. // Keep this list in sync with .github/workflows/CI-e2e-notebooks.yml and/or .github/workflows/CI-e2e-notebooks-vision.yml -const fileNames = [ +const tabularFileNames = [ "responsibleaidashboard-census-classification-model-debugging", "responsibleaidashboard-diabetes-regression-model-debugging", "responsibleaidashboard-housing-classification-model-debugging", "responsibleaidashboard-diabetes-decision-making", "responsibleaidashboard-housing-decision-making", - "responsibleaidashboard-multiclass-dnn-model-debugging", + "responsibleaidashboard-multiclass-dnn-model-debugging" +]; +const visionFileNames = [ "responsibleaidashboard-fridge-image-classification-model-debugging", "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging", "responsibleaidashboard-fridge-object-detection-model-debugging" ]; +const fileNames = tabularFileNames.concat(visionFileNames); const notebookHostReg = /^ResponsibleAI started at (http:\/\/localhost:\d+)$/m; const serveHostReg = /Web Development Server is listening at\s+(.*)$/m; const timeout = 3600; +/** + * + * @param {string} notebook + * @returns {string} + */ +function getDirForNotebook(notebook) { + if (visionFileNames.includes(notebook)) { + return visionDir; + } else if (tabularFileNames.includes(notebook)) { + return tabularDir; + } else { + throw new Error(`Notebook ${notebook} not found.`); + } +} + /** * * @param {string} host @@ -73,7 +93,8 @@ async function runNotebook(name) { const timer = setTimeout(() => { throw new Error(`${name} timeout.`); }, timeout * 1000); - const nbProcess = spawn("python", ["-i", path.join(baseDir, name)]); + const dir = getDirForNotebook(name); + const nbProcess = spawn("python", ["-i", path.join(dir, name)]); nbProcess.on("exit", () => { throw new Error(`Failed to run notebook ${name}`); }); @@ -131,7 +152,8 @@ function addFlightsInFile(path, flights) { function checkIfAllNotebooksHaveTests() { console.log(`Checking if all notebooks under ${baseDir} have tests`); const files = fs - .readdirSync(baseDir) + .readdirSync(tabularDir) + .concat(fs.readdirSync(visionDir)) .filter((f) => f.startsWith(filePrefix) && f.endsWith(".ipynb")) .map((f) => f.replace(".ipynb", "")); const allNotebooksHaveTests = _.isEqual(_.sortBy(files), _.sortBy(fileNames)); @@ -150,6 +172,7 @@ function convertNotebooks(notebook, flights) { console.log(`Skipping ${fileName}. Looking for ${notebook} only.`); continue; } + const dir = getDirForNotebook(fileName); if (flights) { // flights were passed (not just -f without flights arg) console.log( @@ -161,7 +184,7 @@ function convertNotebooks(notebook, flights) { } const { status, stderr } = spawnSync( "jupyter", - ["nbconvert", path.join(baseDir, `${fileName}.ipynb`), "--to", "script"], + ["nbconvert", path.join(dir, `${fileName}.ipynb`), "--to", "script"], { stdio: "inherit" } @@ -170,7 +193,7 @@ function convertNotebooks(notebook, flights) { throw new Error(`Failed to convert notebook:\r\n\r\n${stderr}`); } if (flights) { - addFlightsInFile(path.join(baseDir, `${fileName}.py`), flights); + addFlightsInFile(path.join(dir, `${fileName}.py`), flights); } console.log(`Converted notebook ${fileName}\r\n`); } @@ -185,7 +208,8 @@ function convertNotebooks(notebook, flights) { */ async function runNotebooks(selectedNotebook, host) { let files = fs - .readdirSync(baseDir) + .readdirSync(tabularDir) + .concat(fs.readdirSync(visionDir)) .filter((f) => f.startsWith(filePrefix) && f.endsWith(".py")); console.log("Available notebooks:"); files.forEach((file) => {