From 39bd27e21cdaac4ecf9b9253eaf6554f83a5b408 Mon Sep 17 00:00:00 2001 From: vzhang03 Date: Thu, 8 Aug 2024 16:49:32 -0700 Subject: [PATCH 1/6] Finished new optios to tag paths as flags and skip prompting --- package-lock.json | 10 +++--- packages/cli/package.json | 4 +-- packages/cli/src/data.ts | 4 ++- packages/cli/src/index.ts | 65 ++++++++++++++++++++++++++++++++------- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index 15e1971..ead0b39 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8677,10 +8677,10 @@ }, "packages/cli": { "name": "@jspsych/metadata-cli", - "version": "0.0.1", + "version": "0.0.2", "dependencies": { "@inquirer/prompts": "^5.1.2", - "@jspsych/metadata": "^0.0.1", + "@jspsych/metadata": "^0.0.2", "fuzzy": "^0.1.3", "inquirer": "^10.0.1", "yargs": "^17.7.2" @@ -8690,9 +8690,9 @@ } }, "packages/frontend": { - "version": "0.0.0", + "version": "0.0.1", "dependencies": { - "@jspsych/metadata": "^0.0.1", + "@jspsych/metadata": "^0.0.2", "react": "^18.3.1", "react-dom": "^18.3.1" }, @@ -9195,7 +9195,7 @@ }, "packages/metadata": { "name": "@jspsych/metadata", - "version": "0.0.1", + "version": "0.0.2", "license": "MIT", "dependencies": { "csv-parse": "^5.5.6" diff --git a/packages/cli/package.json b/packages/cli/package.json index 5d24259..12a315c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -30,7 +30,7 @@ "build:no-prompt:esm": "esbuild src/cli.ts --bundle --format=esm --platform=node --outfile=dist/esm/cli.js", "build:data:esm": "esbuild src/data.ts --packages=external --format=esm --platform=node --outfile=dist/esm/data.js", "build:validate:esm": "esbuild src/validatefunctions.ts --packages=external --format=esm --platform=node --outfile=dist/esm/validatefunctions.js", - "build:files:esm": "esbuild src/handlefiles.ts --packages=external --format=esm --platform=node --outfile=dist/esm/handlefiles.js", + "build:files:esm": "esbuild src/handleFiles.ts --packages=external --format=esm --platform=node --outfile=dist/esm/handleFiles.js", "build:utils:esm": "esbuild src/utils.ts --packages=external --format=esm --platform=node --outfile=dist/esm/utils.js", "build:cli:esm": "esbuild src/index.ts --packages=external --format=esm --platform=node --outfile=dist/esm/index.js", "cli": "node dist/esm/index.js", @@ -38,7 +38,7 @@ "build:no-prompt:cjs": "esbuild src/cli.ts --bundle --format=cjs --platform=node --outfile=dist/cjs/cli.js", "build:data:cjs": "esbuild src/data.ts --packages=external --format=cjs --platform=node --outfile=dist/cjs/data.js", "build:validate:cjs": "esbuild src/validatefunctions.ts --packages=external --format=cjs --platform=node --outfile=dist/cjs/validatefunctions.js", - "build:files:cjs": "esbuild src/handlefiles.ts --packages=external --format=cjs --platform=node --outfile=dist/cjs/handlefiles.js", + "build:files:cjs": "esbuild src/handleFiles.ts --packages=external --format=cjs --platform=node --outfile=dist/cjs/handleFiles.js", "build:utils:cjs": "esbuild src/utils.ts --packages=external --format=cjs --platform=node --outfile=dist/cjs/utils.js", "build:cli:cjs": "npm run build:utils:cjs && npm run build:files:cjs && npm run build:validate:cjs && npm run build:data:cjs && esbuild src/index.ts --packages=external --format=cjs --platform=node --outfile=dist/cjs/index.cjs" }, diff --git a/packages/cli/src/data.ts b/packages/cli/src/data.ts index d7a058f..34eb4fb 100644 --- a/packages/cli/src/data.ts +++ b/packages/cli/src/data.ts @@ -64,6 +64,7 @@ const processFile = async (metadata, directoryPath, file, verbose, targetDirecto // Processing directory recursively up to one level export const processDirectory = async (metadata, directoryPath, verbose=false, targetDirectoryPath?) => { + directoryPath = expandHomeDir(directoryPath); let total = 0; let failed = 0; @@ -107,7 +108,7 @@ export const processDirectory = async (metadata, directoryPath, verbose=false, t // Processing metadata options json export const processOptions = async (metadata, filePath, verbose=false) => { try { - const metadata_options_path = generatePath(filePath); + const metadata_options_path = expandHomeDir(generatePath(filePath)); const data = fs.readFileSync(metadata_options_path, "utf8"); // synchronous read if (verbose) console.log("\nmetadata options:", data, "\n"); // log the raw data @@ -149,6 +150,7 @@ export function saveTextToPath(textstr, filePath = './file.txt') { // function for loading metadata export const loadMetadata = async (metadata, filePath) => { + filePath = expandHomeDir(filePath); const fileName = path.basename(filePath).toLowerCase(); // Extract the file name from the filePath try { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index ba64744..3e104ff 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,13 +9,39 @@ import { validateDirectory, validateJson } from './validatefunctions.js'; import { createDirectoryWithStructure } from './handleFiles.js'; // Define a type for the parsed arguments -interface Args { +interface Argv { verbose?: boolean; - [x: string]: unknown; + 'psych-ds-dir'?: string; + 'data-dir'?: string; + 'metadata-options'?: string; + _: (string | number)[]; + $0: string; } -// Parse the arguments and cast them to the defined type -const argv = yargs(hideBin(process.argv)).argv as Args; +// Parse command-line arguments using yargs +const argv = yargs(hideBin(process.argv)) + .option('verbose', { + alias: 'v', + type: 'boolean', + description: 'Run with verbose logging', + }) + .option('psych-ds-dir', { + alias: 'em', + type: 'string', + description: 'Path to an existing Psych-DS directory.', + }) + .option('data-dir', { + alias: 'd', + type: 'string', + description: 'Path to a data-dir.', + }) + .option('metadata-options', { + alias: 'm', + type: 'string', + description: 'Path to a metadata-options.json file.', + }) + .help() + .argv as Argv; async function metadataOptionsPrompt(metadata, verbose){ const answer = await select({ @@ -52,7 +78,6 @@ async function metadataOptionsPrompt(metadata, verbose){ } -// -------------------------------------> new code const promptProjectStructure = async (metadata) => { const answer = await select({ message: 'Would you like to generate a new project directory or update an existing project directory?', @@ -136,11 +161,23 @@ const promptData = async (metadata, verbose, targetDirectoryPath) => { await processDirectory(metadata, data_path, verbose, targetDirectoryPath); // will check if already existing metadata and won't need to prompt } +// can seperate the process argv's out into seperate function const main = async () => { const verbose = argv.verbose ? argv.verbose : false; - const metadata = new JsPsychMetadata(verbose); - var [ project_path, new_project ] = await promptProjectStructure(metadata); // -> if reading from existing will want to look for if dataset_description file exists + + var project_path, new_project; + + if (argv['psych-ds-dir'] + && await validateDirectory(argv['psych-ds-dir']) + && await validateJson(argv['psych-ds-dir'] + "/dataset_description.json", "dataset_description.json")){ + project_path = argv['psych-ds-dir']; + new_project = false; + await loadMetadata(metadata, project_path + "/dataset_description.json"); + } + else { + [ project_path, new_project ] = await promptProjectStructure(metadata); + } if (new_project) { const project_name = await promptName(); @@ -148,12 +185,18 @@ const main = async () => { createDirectoryWithStructure(project_path); // May want to include this with project_name therefore will prevent errors metadata.setMetadataField("name", project_name); // same as above } - await promptData(metadata, verbose, `${project_path}/data`); - - await metadataOptionsPrompt(metadata, verbose); // passing in options file to overwite existing file + // check if it's a valid data directory and run it if it is possible + if (argv['data-dir'] && await validateDirectory(argv['data-dir'])) + await processDirectory(metadata, argv['data-dir'] , verbose, `${project_path}/data`); // will check if already existing metadata and won't need to prompt + else await promptData(metadata, verbose, `${project_path}/data`); + + // check if it's a valid path and then prompt the options + if (argv['metadata-options'] && validateJson(argv['metadata-options'])) + await processOptions(metadata, argv['metadata-options'], verbose); + else await metadataOptionsPrompt(metadata, verbose); // passing in options file to overwite existing file + const metadataString = JSON.stringify(metadata.getMetadata(), null, 2); // Assuming getMetadata() is the function that retrieves your metadata - if (argv.verbose) console.log("Final metadata string:\n\n", metadataString); saveTextToPath(metadataString,`${project_path}/dataset_description.json`); }; From 9dd598ef04e2a46634ca23098dc8fce8da585e45 Mon Sep 17 00:00:00 2001 From: vzhang03 Date: Thu, 8 Aug 2024 16:58:44 -0700 Subject: [PATCH 2/6] Fixed printing of metadata when loading --- packages/cli/src/data.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cli/src/data.ts b/packages/cli/src/data.ts index 34eb4fb..fd4565c 100644 --- a/packages/cli/src/data.ts +++ b/packages/cli/src/data.ts @@ -158,7 +158,6 @@ export const loadMetadata = async (metadata, filePath) => { if (fileName === "dataset_description.json"){ metadata.loadMetadata(content); - console.log("\nContents of dataset_description.json:\n", content) return true; } else console.error("dataset_description.json is not found at path:", filePath); From cd6d10c9ec1416441f97bb032b64f80ed160cfdf Mon Sep 17 00:00:00 2001 From: vzhang03 Date: Thu, 8 Aug 2024 17:08:10 -0700 Subject: [PATCH 3/6] Fixed verbose mode and added more functionality and legibility --- packages/cli/src/data.ts | 4 ++-- packages/cli/src/index.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/data.ts b/packages/cli/src/data.ts index fd4565c..cd093c2 100644 --- a/packages/cli/src/data.ts +++ b/packages/cli/src/data.ts @@ -141,9 +141,9 @@ export function saveTextToPath(textstr, filePath = './file.txt') { fs.writeFile(filePath, textstr, 'utf8', (err) => { if (err) { - console.error(`Error writing to file ${filePath}:`, err); + console.error(`\nError writing to file ${filePath}:`, err); } else { - console.log(`File ${filePath} has been saved.`); + console.log(`\nFile ${filePath} has been saved.`); } }); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 3e104ff..d3f5411 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -173,7 +173,8 @@ const main = async () => { && await validateJson(argv['psych-ds-dir'] + "/dataset_description.json", "dataset_description.json")){ project_path = argv['psych-ds-dir']; new_project = false; - await loadMetadata(metadata, project_path + "/dataset_description.json"); + await loadMetadata(metadata, project_path + "/dataset_description.json"); // maybe shoudl add verbose + if (verbose) console.log(`\n\n-------------------------- Reading existing metadata --------------------------\n\n${JSON.stringify(metadata.getMetadata(), null, 2)}`); } else { [ project_path, new_project ] = await promptProjectStructure(metadata); @@ -187,17 +188,21 @@ const main = async () => { } // check if it's a valid data directory and run it if it is possible - if (argv['data-dir'] && await validateDirectory(argv['data-dir'])) + if (argv['data-dir'] && await validateDirectory(argv['data-dir'])){ + if (verbose) console.log("\n\n-------------------------- Reading and writing data files --------------------------\n\n"); await processDirectory(metadata, argv['data-dir'] , verbose, `${project_path}/data`); // will check if already existing metadata and won't need to prompt + } else await promptData(metadata, verbose, `${project_path}/data`); // check if it's a valid path and then prompt the options - if (argv['metadata-options'] && validateJson(argv['metadata-options'])) + if (argv['metadata-options'] && validateJson(argv['metadata-options'])){ + if (verbose) console.log("\n\n-------------------------- Reading and writing metadata-option --------------------------n\n"); await processOptions(metadata, argv['metadata-options'], verbose); + } else await metadataOptionsPrompt(metadata, verbose); // passing in options file to overwite existing file const metadataString = JSON.stringify(metadata.getMetadata(), null, 2); // Assuming getMetadata() is the function that retrieves your metadata - if (argv.verbose) console.log("Final metadata string:\n\n", metadataString); + if (argv.verbose) console.log("\n\n-------------------------- Final metadata string --------------------------\n\n", metadataString); saveTextToPath(metadataString,`${project_path}/dataset_description.json`); }; From 556ac9574253d3232779b381a0b6eda87015233d Mon Sep 17 00:00:00 2001 From: vzhang03 Date: Thu, 8 Aug 2024 17:18:34 -0700 Subject: [PATCH 4/6] Adding changeset --- .changeset/sharp-chefs-speak.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/sharp-chefs-speak.md diff --git a/.changeset/sharp-chefs-speak.md b/.changeset/sharp-chefs-speak.md new file mode 100644 index 0000000..0ef9366 --- /dev/null +++ b/.changeset/sharp-chefs-speak.md @@ -0,0 +1,5 @@ +--- +"@jspsych/metadata-cli": minor +--- + +Adding flags for skipping steps in the prompting process and cleaning up verbose mode From f312de2a361be67e5a2790037b3802c29ee613d6 Mon Sep 17 00:00:00 2001 From: vzhang03 Date: Fri, 9 Aug 2024 15:12:31 -0700 Subject: [PATCH 5/6] Final updates to prompting messages making them more descriptive using checkmarks --- dev/fail/success.csv | 14 ++++++++++++++ packages/cli/src/data.ts | 14 ++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 dev/fail/success.csv diff --git a/dev/fail/success.csv b/dev/fail/success.csv new file mode 100644 index 0000000..271d280 --- /dev/null +++ b/dev/fail/success.csv @@ -0,0 +1,14 @@ +"success","timeout","failed_images","failed_audio","failed_video","trial_type","trial_index","plugin_version","time_elapsed","rt","stimulus","response" +"true","false","[]","[]","[]","preload","0","2.0.0","5","","","" +"","","","","","html-keyboard-response","1","2.0.0","1088","null","

+

","null" +"","","","","","image-keyboard-response","2","2.0.0","2739","1398","img/happy_face_2.jpg","y" +"","","","","","html-keyboard-response","3","2.0.0","3494","null","

+

","null" +"","","","","","image-keyboard-response","4","2.0.0","4002","255","img/happy_face_3.jpg","n" +"","","","","","html-keyboard-response","5","2.0.0","4756","null","

+

","null" +"","","","","","image-keyboard-response","6","2.0.0","5073","64","img/happy_face_1.jpg","y" +"","","","","","html-keyboard-response","7","2.0.0","5828","null","

+

","null" +"","","","","","image-keyboard-response","8","2.0.0","6462","381","img/happy_face_2.jpg","n" +"","","","","","html-keyboard-response","9","2.0.0","7217","null","

+

","null" +"","","","","","image-keyboard-response","10","2.0.0","7639","168","img/happy_face_3.jpg","y" +"","","","","","html-keyboard-response","11","2.0.0","8393","null","

+

","null" +"","","","","","image-keyboard-response","12","2.0.0","8697","50","img/happy_face_1.jpg","n" \ No newline at end of file diff --git a/packages/cli/src/data.ts b/packages/cli/src/data.ts index cd093c2..9a1b88a 100644 --- a/packages/cli/src/data.ts +++ b/packages/cli/src/data.ts @@ -49,7 +49,7 @@ const processFile = async (metadata, directoryPath, file, verbose, targetDirecto await metadata.generate(content, {}, 'csv'); break; default: - console.error(file, "is not .csv or .json format."); + console.error(`"${file}" is not .csv or .json format.`); return false; } @@ -101,7 +101,11 @@ export const processDirectory = async (metadata, directoryPath, verbose=false, t }; await processDirectoryRecursive(directoryPath, 0); - console.log("Files read with rate:", `${(total - failed)}/${total}`); + + if (failed === 0) console.log(`āœ” Reading data files was successful with ${total} files read.`); + else if (failed !== total) console.log(`? Data files was partially successful with ${(total - failed)}/${total} files read.`); + else if (failed === total) console.log(`x Data files was unsuccessful with 0 files read. Please try again with valid JsPsych generated data.`); + return { total, failed }; }; @@ -115,6 +119,7 @@ export const processOptions = async (metadata, filePath, verbose=false) => { var metadata_options = JSON.parse(data); // parse the JSON data metadata.updateMetadata(metadata_options); + console.log(`\nāœ” Successfully read and updated metadata according to options file.`); return true; } catch (error) { console.error("Error reading or parsing metadata options:", error); @@ -143,7 +148,7 @@ export function saveTextToPath(textstr, filePath = './file.txt') { if (err) { console.error(`\nError writing to file ${filePath}:`, err); } else { - console.log(`\nFile ${filePath} has been saved.`); + console.log(`\nāœ” File ${filePath} has been saved.`); } }); } @@ -157,7 +162,8 @@ export const loadMetadata = async (metadata, filePath) => { const content = await fs.promises.readFile(filePath, "utf8"); if (fileName === "dataset_description.json"){ - metadata.loadMetadata(content); + metadata.loadMetadata(content); + console.log(`\nāœ” Successfully loaded previous metadata.\n`); return true; } else console.error("dataset_description.json is not found at path:", filePath); From b26ed599b1e5d25c94185df6542448e107a463f8 Mon Sep 17 00:00:00 2001 From: vzhang03 Date: Fri, 9 Aug 2024 15:33:52 -0700 Subject: [PATCH 6/6] Updating documentation to include the flags --- packages/cli/README.md | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 8b0d46c..c7710ad 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -6,27 +6,39 @@ The metadata-cli module contains commands for interacting with the metadata-modu ## CLI with prompting (recommended) -To run the CLI with prompting you can call ```npm i @jspsych/metadata-cli```. This is the recommended way of running the package. +To run the CLI with prompting you can call ```npx @jspsych/metadata-cli```. This is the recommended way of running the package. This will then prompt you through the various CLI options, this includes generating from scratch or using an existing file. There are descriptions of how the files and directories should be structured below. -You can also run it through the repository by following the steps 1-3 below and then calling ```npm run build``` followed by ```npx .```. +## CLI without prompting and optional flags -## CLI without prompting +If you are findng the prompting steps to be more of a hinderance rather, then you can skip steps by directly passing in the arguments when calling npx. -Depending on the build step, when running it locally you may need to rename the package.json attribute ```"type": "commonjs"``` to ```"type": "module"```. To run the CLI without prompting, you would need to clone this repo and navigate to the CLI directory. To do this, you must have node installed and it is not recommended unless you have programming experience. +An example is: + +``` +npx @jspsych/metadata-cli --verbose --psych-ds-dir=/path/to/existing/metadata/dir --data-dir=/path/to/data/dir --metadata-options=/path/to/metadata-options.json +``` + +The ```--verbose``` flag when called will make the output more descriptive and will explain all the steps that are happening in more detail. This is only recommended when there is something going wrong and you need to get more information. + +The ```--psych-ds-dir --data-dir --metadata-options``` flags all correspond to the different prompting steps. Using the example above, you are able to skip all prompting steps by entering the correct information. If any of these inputs are not valid the CLI will prompt you until you enter a valid input. + +```--psych-ds-dir``` corresponds to an existing Psych-DS valid directory. This is to be used if want to update existing metadata. + +```--data-dir``` corresponds to the data folder that you want to use to update or create the metadata. + +```--metadata-options``` corresponds to the options.json that you want to use to overwrite the defaults and update the existing metadata. + +## Running the CLI locally + +To run the CLI locally, you will need to clone this repo and navigate to the CLI directory. To do this, you must have node installed and it is not recommended unless you have programming experience. This is best if you want to customize the CLI according to your specific use case or want to avoid using npm/npx. 1. Clone the repository. 2. Navigate to the CLI directory. 3. Install the necessary packages with ```npm install```. 4. Build the cli file with ```npm run build```. -5. Run the cli with ```node /dist/cli.js /path/to/data/dir /path/to/metadata_options.json```. - -Within the package.json, there are examples of running this using existing data files. You can run those commands with ```npm run data``` or ```npm run options```. - -If the metadata file has already been created, ensure that ```dataset_description.json``` is located in the root directory. This CLI will successfully load the old data prior to updating it with new data and will then overwrite the original data. - -This will likely +5. Run the cli with ```npx .```. ## Common errors