From f2114a8554502794aa80afa57abd6d754bf4f6e6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Anderson" Date: Sun, 12 May 2024 16:34:51 -0400 Subject: [PATCH] Many updates. * Improve documentation. * Show Job ID column only when showing submitted or completed jobs. * Validate submit_whole. * Cache workspace directory mtime and list it only when changed. * New subcommands: `row init` and `row clean`. * Show requested job statuses in `show directories`. --- .github/CODEOWNERS | 1 + .gitignore | 6 +- .pre-commit-config.yaml | 16 +- Cargo.lock | 19 + Cargo.toml | 1 + DESIGN.md | 21 +- README.md | 82 ++-- demo/.gitignore | 1 + demo/demo.sh | 19 + demo/workflow.toml | 26 ++ doc/src/SUMMARY.md | 12 +- doc/src/clusters/cluster.md | 4 +- doc/src/contributors.md | 5 + doc/src/developers/contributing.md | 90 +++++ doc/src/developers/style.md | 31 +- doc/src/developers/testing.md | 10 +- doc/src/guide/concepts/cache.md | 71 +++- doc/src/guide/concepts/json-pointers.md | 2 +- doc/src/guide/python/actions.md | 21 +- doc/src/guide/python/signac.sh | 6 +- doc/src/guide/tutorial/group.md | 6 +- doc/src/guide/tutorial/group.sh | 7 +- doc/src/guide/tutorial/hello.md | 11 +- doc/src/guide/tutorial/hello.sh | 4 +- doc/src/guide/tutorial/index.md | 5 +- doc/src/guide/tutorial/submit.md | 11 +- doc/src/images/umich-block-M.svg | 79 ++++ doc/src/launchers/built-in.md | 2 +- doc/src/license.md | 498 ++++++++++++++++++++++++ doc/src/release-notes.md | 5 + doc/src/row/clean.md | 40 ++ doc/src/row/index.md | 2 +- doc/src/row/init.md | 52 ++- doc/src/row/scan.md | 23 +- doc/src/row/show/cluster.md | 31 +- doc/src/row/show/directories.md | 61 ++- doc/src/row/show/launchers.md | 27 +- doc/src/row/show/status.md | 41 +- doc/src/row/submit.md | 35 +- doc/src/row/uncomplete.md | 3 - doc/theme/index.hbs | 359 +++++++++++++++++ src/builtin.rs | 5 + src/cli.rs | 192 ++++++++- src/cli/clean.rs | 115 ++++++ src/cli/cluster.rs | 3 + src/cli/directories.rs | 114 ++++-- src/cli/init.rs | 122 ++++++ src/cli/launchers.rs | 3 + src/cli/scan.rs | 3 + src/cli/status.rs | 9 +- src/cli/submit.rs | 16 +- src/cluster.rs | 3 + src/expr.rs | 3 + src/format.rs | 3 + src/launcher.rs | 3 + src/lib.rs | 33 +- src/main.rs | 9 + src/progress_styles.rs | 3 + src/project.rs | 3 + src/scheduler.rs | 3 + src/scheduler/bash.rs | 3 + src/scheduler/slurm.rs | 3 + src/state.rs | 223 +++++++---- src/ui.rs | 38 +- src/workflow.rs | 17 +- src/workspace.rs | 3 + tests/cli.rs | 104 +++++ validate/validate.py | 4 +- 68 files changed, 2475 insertions(+), 311 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 demo/.gitignore create mode 100644 demo/demo.sh create mode 100644 demo/workflow.toml create mode 100644 doc/src/contributors.md create mode 100644 doc/src/developers/contributing.md create mode 100644 doc/src/images/umich-block-M.svg create mode 100644 doc/src/license.md create mode 100644 doc/src/release-notes.md create mode 100644 doc/src/row/clean.md delete mode 100644 doc/src/row/uncomplete.md create mode 100644 doc/theme/index.hbs create mode 100644 src/cli/clean.rs create mode 100644 src/cli/init.rs diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..3c6c1aa --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* joaander diff --git a/.gitignore b/.gitignore index ea74663..b6e08f7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /target -/workspace /workflow.toml -/.row -/.signac +workspace +.row +.signac diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c6faa6..fdcd5dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,5 +34,17 @@ repos: hooks: - id: ruff-format - id: ruff - -# TODO: add fix-license-header +- repo: https://github.com/glotzerlab/fix-license-header + rev: v0.3.2 + hooks: + - id: fix-license-header + name: Fix license headers (rust) + types_or: [rust] + args: + - --license-file=LICENSE + - --add=Part of row, released under the BSD 3-Clause License. + - --comment-prefix=// +- repo: https://github.com/crate-ci/typos + rev: v1.21.0 + hooks: + - id: typos diff --git a/Cargo.lock b/Cargo.lock index 274864f..cc4d72c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -638,6 +638,24 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -770,6 +788,7 @@ dependencies = [ "log", "memchr", "nix", + "path-absolutize", "postcard", "predicates", "serde", diff --git a/Cargo.toml b/Cargo.toml index 1a6a84b..ac112b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ indicatif-log-bridge = "0.2.2" log = "0.4.21" memchr = "2.7.2" nix = { version = "0.28.0", features = ["signal"] } +path-absolutize = "3.1.1" postcard = { version = "1.0.8", default-features = false, features = ["use-std"] } serde = { version = "1.0.197", features = ["derive"] } serde_json = "1.0.114" diff --git a/DESIGN.md b/DESIGN.md index cb16e73..79d67de 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -52,11 +52,6 @@ Row is yet another workflow engine that automates the process of executing **act Ideas: * List scheduler jobs and show useful information. * Cancel scheduler jobs specific to actions and/or directories. -* Command to uncomplete an action for a set of directories. This would remove the product files and - update the cache. -* Some method to clear any cache (maybe this instead of uncomplete?). This would allow - users to discover changed action names, changed products, manually uncompleted - actions, and deal with corrupt cache files. ## Overview @@ -138,7 +133,8 @@ completed **directories** is read. ### The cache files Row maintains the state of the workflow in several files: -* `values.json` +* `directories.json` + * Last time the workspace was modified. * Cached copies of the user-provided static value file. * `completed.postcard` * Completion status for each **action**. @@ -263,13 +259,14 @@ status may take a long time, so it should display a progress bar. ## Subcommands -* `init` - create `workflow.toml` and `workspace` if they do not yet exist. (TODO: write init) +* `init` - create `workflow.toml` and `workspace` if they do not yet exist. * `scan` - scan the workspace for directories that have completed actions. * `show` - show properties of the workflow: * `status` - summarize the status of the workflow. * `directories` - list directories in the workflow. - -Ideas for other commands, `uncomplete` + * `clsuter` - show the currently selected cluster configuration. + * `launchers` - list the launchers for the current cluster. +* `clean` - delete row cache files. ## Definitions @@ -296,3 +293,9 @@ Ideas for other commands, `uncomplete` - **whole group**: A **submission group** that is identical to the **group** found without applying the additional submission filters. - **workspace**: The location on the file system that contains **directories**. + +# TODO: Pull request template +# TODO: Issue templates? +# TODO: Dependabot configuration +# TODO: readthedocs builds +# TODO: logo diff --git a/README.md b/README.md index e87696c..92277ed 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,79 @@ # Row +[![GitHub Actions](https://github.com/glotzerlab/row/actions/workflows/test.yaml/badge.svg?branch=trunk)](https://github.com/glotzerlab/row/actions/workflows/test.yaml) +[![Read the Docs](https://img.shields.io/readthedocs/row/latest.svg)](https://row.readthedocs.io/) +[![Contributors](https://img.shields.io/github/contributors-anon/glotzerlab/row.svg?style=flat)](https://row.readthedocs.io/en/latest/contributors.html) +[![License](https://img.shields.io/badge/license-BSD--3--Clause-green.svg)](https://row.readthedocs.io/en/latest/license.html) + Row is a command line tool that helps you manage workflows on HPC resources. Define **actions** in a workflow configuration file that apply to **groups** of **directories** -in your **workspace**. **Submit** actions to your HPC **scheduler**. Row tracks which -actions have been submitted on which directories so that you don't submit the same work -twice. Once a job completes, subsequent actions become eligible allowing you to process -your entire workflow to completion over many submissions. +in your **workspace**. **Submit** actions to your cluster's **scheduler**. Row tracks +which actions have been submitted on which directories so that you don't submit the same +work twice. Once a job completes, subsequent actions become eligible allowing you to +process your entire workflow to completion over many submissions. -The name is "row" as in "row, row, row your boat". +The name is **row** as in *"row, row, row your boat"*. Notable features: -* Support both arbitrary directories and [signac](https://signac.io) workspaces. -* Execute actions via arbitrary shell commands. +* Support arbitrary directories and [signac](https://signac.io) workspaces. +* Define your workflow in a configuration file. +* Execute actions via user-defined shell commands. * Flexible group definitions: * Select directories with conditions on their value. * Split directories by their value and/or into fixed size groups. * Execute groups in serial or parallel. * Schedule CPU and GPU resources. -* Automatically determine the partition based on the batch job size. +* Automatically determine the partition based on the job's resources and size. * Built-in configurations for many national and university HPC systems. * Add custom cluster definitions for your resources. +* Row is **fast**. -TODO: better demo script to get output for README and row show documentation examples. +## Demonstration -For example: ```bash -> row show status -Action Completed Submitted Eligible Waiting Remaining cost -one 1000 100 900 0 24K CPU-hours -two 0 200 800 1000 8K GPU-hours +$ row submit --action=step1 -n 1 +[1/1] Submitting action 'step1' on directory dir12 and 3 more (0ms). +Row submitted job 5095791. ``` ```bash -> row show directories --value "/value" -Directory Status Job ID /value -dir1 submitted 1432876 0.9 -dir2 submitted 1432876 0.8 -dir3 submitted 1432876 0.7 - -dir4 completed 0.5 -dir5 completed 0.4 -dir6 completed 0.3 +$ row show status +Action Completed Submitted Eligible Waiting Remaining cost +initialize 50 0 50 0 8 CPU-hours +step1 4 4 42 50 2K CPU-hours +step2 0 0 4 96 800 GPU-hours +``` + +```bash +$ row show directories step1 -n 3 --value="/value" +Directory Status Job ID /value +dir1 completed 116 +dir10 completed 952 +dir100 completed 139 +dir11 completed 998 + +dir12 submitted anvil/5095791 950 +dir13 submitted anvil/5095791 107 +dir14 submitted anvil/5095791 127 +dir15 submitted anvil/5095791 122 + +dir16 eligible 682 +dir17 eligible 816 +dir18 eligible 803 +dir19 eligible 691 ``` -**Row** is a spiritual successor to -[signac-flow](https://docs.signac.io/projects/flow/en/latest/). +## Resources + +- [Documentation](https://row.readthedocs.io/): + Tutorial, command line interface documentation, and configuration file specifications. +- [Row discussion board](https://github.com/glotzerlab/row/discussions/): + Ask the **row** user community for help. +- [signac](https://signac.io): + Python package to help you manage your workspace. + +## History + +**Row** is a spiritual successor to [signac-flow][flow]. + +[flow]: https://docs.signac.io/projects/flow/en/latest/. diff --git a/demo/.gitignore b/demo/.gitignore new file mode 100644 index 0000000..f47cb20 --- /dev/null +++ b/demo/.gitignore @@ -0,0 +1 @@ +*.out diff --git a/demo/demo.sh b/demo/demo.sh new file mode 100644 index 0000000..ce9501a --- /dev/null +++ b/demo/demo.sh @@ -0,0 +1,19 @@ +mkdir workspace || exit 1 +cd workspace + +for i in {1..100} +do + mkdir dir$i || exit 1 + v=$((1 + RANDOM % 1000)) + echo "{\"value\": $v}" > dir$i/value.json || exit 1 +done + +row submit --cluster none --action=initialize -n 5 --yes || exit 1 + +row submit --cluster none --action=step1 -n 1 --yes || exit 1 + +row submit --action=step1 -n 1 --yes || exit 1 + +row show status || exit 1 + +row show directories step1 -n 3 --value="/value" diff --git a/demo/workflow.toml b/demo/workflow.toml new file mode 100644 index 0000000..1175cf3 --- /dev/null +++ b/demo/workflow.toml @@ -0,0 +1,26 @@ +[workspace] +value_file = "value.json" + +[[action]] +name = "initialize" +command = "touch workspace/{directory}/initialize.out" +products = ["initialize.out"] +resources.walltime.per_directory = "00:10:00" +group.maximum_size = 10 + +[[action]] +name = "step1" +command = "touch workspace/{directory}/step1.out" +products = ["step1.out"] +previous_actions = ["initialize"] +resources.walltime.per_directory = "1 day, 00:00:00" +group.maximum_size = 4 + +[[action]] +name = "step2" +command = "touch workspace/{directory}/step2.out" +previous_actions = ["step1"] +products = ["step2.out"] +resources.walltime.per_directory = "08:00:00" +resources.gpus_per_process = 1 +group.maximum_size = 4 diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index c7f2757..bce7bd0 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -21,7 +21,7 @@ - [Thread parallelism](guide/concepts/thread-parallelism.md) - [Directory status](guide/concepts/status.md) - [JSON pointers](guide/concepts/json-pointers.md) - - [The row cache](guide/concepts/cache.md) + - [Cache files](guide/concepts/cache.md) # Reference - [row](row/index.md) @@ -33,7 +33,7 @@ - [show cluster](row/show/cluster.md) - [show launchers](row/show/launchers.md) - [scan](row/scan.md) - - [uncomplete](row/uncomplete.md) + - [clean](row/clean.md) - [`workflow.toml`](workflow/index.md) - [workspace](workflow/workspace.md) @@ -52,14 +52,14 @@ # Appendix -- [Change log]() +- [Release notes](release-notes.md) - [Migrating from signac-flow](signac-flow.md) - [For developers](developers/index.md) - - [Contributing]() + - [Contributing](developers/contributing.md) - [Code style](developers/style.md) - [Testing](developers/testing.md) - [Documentation](developers/documentation.md) -- [License]() +- [License](license.md) ----- -[Contributors]() +[Contributors](contributors.md) diff --git a/doc/src/clusters/cluster.md b/doc/src/clusters/cluster.md index 2c087f9..a06885f 100644 --- a/doc/src/clusters/cluster.md +++ b/doc/src/clusters/cluster.md @@ -75,7 +75,7 @@ total_cpus <= maximum_cpus_per_job `cluster.partition.require_cpus_multiple_of`: **integer** - All jobs submitted to this partition **must** use an integer multiple of the given number of cpus: -``` +```plaintext total_cpus % require_cpus_multiple_of == 0 ``` @@ -118,7 +118,7 @@ total_gpus <= maximum_gpus_per_job `cluster.partition.require_gpus_multiple_of`: **integer** - All jobs submitted to this partition **must** use an integer multiple of the given number of gpus: -``` +```plaintext total_gpus % require_gpus_multiple_of == 0 ``` diff --git a/doc/src/contributors.md b/doc/src/contributors.md new file mode 100644 index 0000000..c5b1d4b --- /dev/null +++ b/doc/src/contributors.md @@ -0,0 +1,5 @@ +# Contributors + +The following people have contributed to the development of **row**: + +* Joshua A. Anderson, University of Michigan diff --git a/doc/src/developers/contributing.md b/doc/src/developers/contributing.md new file mode 100644 index 0000000..fe8cf03 --- /dev/null +++ b/doc/src/developers/contributing.md @@ -0,0 +1,90 @@ +# Contributing + +Contributions are welcomed via [pull requests on GitHub][github]. Contact the **row** +developers before starting work to ensure it meshes well with the planned development +direction and standards set for the project. + +[github]: https://github.com/glotzerlab/gsd/row + +## Features + +### Implement functionality in a general and flexible fashion + +New features should be applicable to a variety of use-cases. The **row** developers can +assist you in designing flexible interfaces. + +### Maintain performance of existing code paths + +Expensive code paths should only execute when requested. + +## Version control + +### Base your work off the correct branch + +- Base all new work on `trunk`. + +### Propose a minimal set of related changes + +All changes in a pull request should be closely related. Multiple change sets that are +loosely coupled should be proposed in separate pull requests. + +### Agree to the Contributor Agreement + +All contributors must agree to the Contributor Agreement before their pull request can +be merged. + +### Set your git identity + +Git identifies every commit you make with your name and e-mail. [Set your identity][id] +to correctly identify your work and set it identically on all systems and accounts where +you make commits. + +[id]: http://www.git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup + +## Source code + +### Use a consistent style + +The **Code style** section of the documentation sets the style guidelines for **row** +code. + +### Document code with comments + +Use **Rust** documentation comments for classes, functions, etc. Also comment complex +sections of code so that other developers can understand them. + +### Compile without warnings + +Your changes should compile without warnings. + +## Tests + +### Write unit tests + +Add unit tests for all new functionality. + +### Validity tests + +The developer should run research-scale simulations using the new functionality and +ensure that it behaves as intended. When appropriate, add a new test to `validate.py`. + +## User documentation + +### Write user documentation + +Document all new configuration keys, command line options, command line tools, +and any important user-facing change in the mdBook documentation. + +### Tutorial + +When applicable, update or write a new tutorial. + + +### Add developer to the credits + +Update the contributors documentation to name each developer that has contributed to the +code. + +### Propose a change log entry + +Propose a short concise entry describing the change in the pull request description. diff --git a/doc/src/developers/style.md b/doc/src/developers/style.md index b0bb84a..6718f61 100644 --- a/doc/src/developers/style.md +++ b/doc/src/developers/style.md @@ -1,7 +1,30 @@ # Code style -**Row** code follows the -[Rust style guide](https://doc.rust-lang.org/style-guide/index.html). +## Rust -**Row's** [pre-commit](https://pre-commit.com/) configuration applies style fixes -with `rustfmt` checks for common errors with `clippy`. +**Row's** rust code follows the [Rust style guide][1]. **Row's** [pre-commit][2] +configuration applies style fixes with `rustfmt` checks for common errors with `clippy`. + +[1]: https://doc.rust-lang.org/style-guide/index.html +[2]: https://pre-commit.com/ + +## Python + +**Row's** pre-commit configuration both formats and checks Python code with `ruff`. + +## Markdown + +Wrap **Markdown** files at 88 characters wide, except when not possible (e.g. when +formatting a table). Follow layout and design patterns established in existing markdown +files. + +## Spelling/grammar + +Contributors **must** configure their editors to perform spell checking (and preferably +grammar checking as well). **Row's** pre-commit runs +[typos](https://github.com/crate-ci/typos) which has a low rate of false positives. +Developers *should* also configure a more thorough checker of their choice to ensure +that code comments and documentation are free of errors. Suggested tools: +* [typos](https://github.com/crate-ci/typos) +* [ltex-ls](https://github.com/valentjn/ltex-ls) +* [cspell](https://cspell.org/) diff --git a/doc/src/developers/testing.md b/doc/src/developers/testing.md index eca87af..4f67fa6 100644 --- a/doc/src/developers/testing.md +++ b/doc/src/developers/testing.md @@ -14,8 +14,9 @@ with any test that is automatically run concurrently. Check for this with: ```bash rg --multiline "#\[test\]\n *fn" ``` -(see the [saftey discussion](https://doc.rust-lang.org/std/env/fn.set_var.html) in -`std::env` for details. +(see the [safety discussion][1] in `std::env` for details). + +[1]: https://doc.rust-lang.org/std/env/fn.set_var.html ## Cluster-specific tests @@ -26,8 +27,9 @@ describes how to run the tests. ## Tutorial tests The tutorial scripts in `doc/src/guide/*.sh` are runnable. These are described in the -documentation using mdBook's anchor feature to include -[portions of files](https://rust-lang.github.io/mdBook/format/mdbook.html) in the +documentation using mdBook's anchor feature to include [portions of files][2] in the documentation as needed. This way, the tutorial can be tested by executing the script. This type of testing validates that the script *runs*, not that it produces the correct output. Developers should manually check the tutorial script output as needed. + +[2]: https://rust-lang.github.io/mdBook/format/mdbook.html diff --git a/doc/src/guide/concepts/cache.md b/doc/src/guide/concepts/cache.md index 5e01ebc..1fdbc04 100644 --- a/doc/src/guide/concepts/cache.md +++ b/doc/src/guide/concepts/cache.md @@ -1,3 +1,70 @@ -# The row cache +# Cache files -TODO: Write this document. +**Row** stores cache files in `/.row` to improve performance. In most +usage environments **row** will automatically update the cache and keep it synchronized +with the state of the workflow and workspace. The rest of this document describes +some scenarios where they cache may not be updated and how you fix the problem. + +## Directory values + +**Row** caches the **value** of every directory in the workspace. The cache will be +invalid when: +* *You change the contents* of any value file. +* *You change* `value_file` in `workflow.toml`. + +> To recover from such a change, execute: +> ```bash +> row clean --directory +> ``` + +## Submitted jobs + +**Row** caches the job ID, directory, and cluster name for every job it submits +to a cluster via `row submit`. **Row** will be unaware of any jobs that you manually +submit with `sbatch`. + +> You should submit all jobs via: +> ```bash +> `row submit` +> ``` + +Copying a project directory (including `.row/`) from one cluster to another (or from +a cluster to a workstation) will preserve the submitted cache. The 2nd cluster cannot +access the job queue of the first, so all jobs will remain in the cache. *Submitting* +jobs on the 2nd cluster will inevitably lead to changes in the submitted cache on both +clusters that cannot be merged. + +> Before you copy your project directory, wait for all jobs to finish, then execute +> ```bash +> row show status +> ``` +> to update the cache. + +## Completed directories + +Jobs submitted by `row submit` check if they completed any directories on exit and +update the completed cache accordingly. A completed directory may not be discovered +if: +* *The job is killed* (e.g. due to walltime limits). +* *You execute an action manually* (e.g. `python action.py action directory`). +* *You change products* in `workflow.toml`. +* *You change the name of an action* in `workflow.toml`. + +> To discover new completed directories, execute +> ```bash +> row scan +> ``` +> This is safe to run any time, including at the same time as any running jobs. + +`row scan` only discovers **completed** actions. It *does not* check if a currently +**complete** directory no longer contains an action's products. Therefore, **row** will +still consider directories complete even when: +* *You change products* in `workflow.toml`. +* *You delete product files* in a directory. + +> To completely reset the completed cache, execute: +> ```bash +> row clean --completed +> row scan +> ``` +> `row clean` will require that you wait until all submitted jobs have completed first. diff --git a/doc/src/guide/concepts/json-pointers.md b/doc/src/guide/concepts/json-pointers.md index d24debd..36e6685 100644 --- a/doc/src/guide/concepts/json-pointers.md +++ b/doc/src/guide/concepts/json-pointers.md @@ -4,7 +4,7 @@ way for you to access elements of a directory's JSON value. For example, given the JSON document: -``` +```json { "a": 1, "b": { diff --git a/doc/src/guide/python/actions.md b/doc/src/guide/python/actions.md index d19cd71..692ba2a 100644 --- a/doc/src/guide/python/actions.md +++ b/doc/src/guide/python/actions.md @@ -8,8 +8,10 @@ This guide will show you how to structure all of your actions in a single file: `actions.py`. This layout is inspired by **row's** predecessor: **signac-flow** and its `project.py`. -> Note: If you are familiar with **signac-fow**, see -> [migrating from signac-flow](../../signac-flow.md) for many helpful tips. +> Note: If you are familiar with **signac-fow**, see [migrating from signac-flow][1] +> for many helpful tips. + +[1]: ../../signac-flow.md To demonstrate the structure of a project, let's build a workflow that computes the sum of squares. The focus of this guide is on structure and best practices. You need to @@ -35,11 +37,12 @@ Execute: ``` to initialize the signac workspace and populate it with directories. -> Note: If you aren't familiar with **signac**, then go read the -> [*basic* tutorial](https://docs.signac.io/en/latest/tutorial.html#basics). Come back -> to the **row** documentation when you get to the section on *workflows*. Or, for -> extra credit, reimplement the **signac** tutorial workflow in **row** after you finish -> reading this guide. +> Note: If you aren't familiar with **signac**, then go read the [*basic* tutorial][2]. +> Come back to the **row** documentation when you get to the section on *workflows*. Or, +> for extra credit, reimplement the **signac** tutorial workflow in **row** after you +> finish reading this guide. + +[2]: https://docs.signac.io/en/latest/tutorial.html#basics ## Write actions.py @@ -92,7 +95,7 @@ and you should see: ```plaintext Submitting 1 job that may cost up to 0 CPU-hours. Proceed? [Y/n]: y -[1/1] Submitting action 'square' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more (0 seconds). +[1/1] Submitting action 'square' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more. ``` Next, submit the *compute_sum* action: @@ -103,7 +106,7 @@ and you should see: ```plaintext Submitting 1 job that may cost up to 0 CPU-hours. Proceed? [Y/n]: y -[1/1] Submitting action 'compute_sum' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more (0 seconds). +[1/1] Submitting action 'compute_sum' on directory 04bb77c1bbbb40e55ab9eb22d4c88447 and 9 more. 285 ``` diff --git a/doc/src/guide/python/signac.sh b/doc/src/guide/python/signac.sh index 015aa07..f43dadb 100644 --- a/doc/src/guide/python/signac.sh +++ b/doc/src/guide/python/signac.sh @@ -1,8 +1,6 @@ # ANCHOR: row_init -# $ row init sum_squares --signac # TODO: Write row init -mkdir -p sum_squares/workspace +row init sum_squares --signac cd sum_squares -echo 'workspace.value_file = "signac_statepoint.json"' > workflow.toml # ANCHOR_END: row_init cp ../populate_workspace.py . @@ -19,5 +17,5 @@ row submit --action square # ANCHOR_END: submit_square # ANCHOR: submit_sum -row submit --action sum +row submit --action compute_sum # ANCHOR_END: submit_sum diff --git a/doc/src/guide/tutorial/group.md b/doc/src/guide/tutorial/group.md index d708871..6cde54f 100644 --- a/doc/src/guide/tutorial/group.md +++ b/doc/src/guide/tutorial/group.md @@ -200,13 +200,13 @@ you will see that `row submit` launches 3 jobs: ```plaintext Submitting 3 jobs that may cost up to 6 CPU-hours. Proceed? [Y/n]: -[1/3] Submitting action 'process_point' on directory directory1 and 2 more (0 seconds). +[1/3] Submitting action 'process_point' on directory directory1 and 2 more. directory1 directory3 directory5 -[2/3] Submitting action 'process_point' on directory directory6 (0 seconds). +[2/3] Submitting action 'process_point' on directory directory6. directory6 -[3/3] Submitting action 'process_point' on directory directory2 and 1 more (0 seconds). +[3/3] Submitting action 'process_point' on directory directory2 and 1 more. directory2 directory4 ``` diff --git a/doc/src/guide/tutorial/group.sh b/doc/src/guide/tutorial/group.sh index 11bfbc0..94b61b7 100644 --- a/doc/src/guide/tutorial/group.sh +++ b/doc/src/guide/tutorial/group.sh @@ -1,11 +1,8 @@ # ANCHOR: init -# $ row init group-workflow # TODO: Write row init -mkdir -p group-workflow/workspace -cd group-workflow -touch workflow.toml +row init group-workflow +cd group-workflow/workspace -cd workspace mkdir directory1 && echo '{"type": "point", "x": 0, "y": 10}' > directory1/value.json mkdir directory2 && echo '{"type": "point", "x": 3, "y": 8}' > directory2/value.json mkdir directory3 && echo '{"type": "point", "x": 0, "y": 4}' > directory3/value.json diff --git a/doc/src/guide/tutorial/hello.md b/doc/src/guide/tutorial/hello.md index 3694449..bd2e75e 100644 --- a/doc/src/guide/tutorial/hello.md +++ b/doc/src/guide/tutorial/hello.md @@ -30,11 +30,10 @@ empty `workflow.toml` file that `row init` created with: ```toml {{#include hello-workflow.toml}} ``` -`workflow.toml` is a [TOML](https://toml.io) file. Don't be afraid to read the -spec, it is written in clear an accessible language and includes many examples. In -`workflow.toml`, `action` is an array of tables and each `[[action]]` line adds a -new element. Each element **requires** the keys `name` and `command`. There are many -optional keys you will learn about in later tutorials, or you can skip ahead and +`workflow.toml` is a [TOML](https://toml.io) file. In `workflow.toml`, `action` is an +array of tables and each `[[action]]` line adds a new element. Each element **requires** +the keys `name` and `command`. There are many optional keys you will learn about in +later tutorials, or you can skip ahead and [read the action reference documentation](../../workflow/action/index.md). `name` is a string that sets the name of the action. `command` is a template for a shell @@ -60,7 +59,7 @@ Later sections in this tutorial will cover resource costs in more detail. `echo "Hello, {directory}!"` is certainly not going to take that long, so confirm with `y` and then press enter. You should then see the action execute: ```plaintext -[1/1] Submitting action 'hello' on directory directory0 and 2 more (0 seconds). +[1/1] Submitting action 'hello' on directory directory0 and 2 more. Hello, directory0! Hello, directory1! Hello, directory2! diff --git a/doc/src/guide/tutorial/hello.sh b/doc/src/guide/tutorial/hello.sh index 71221e3..e73d4a5 100644 --- a/doc/src/guide/tutorial/hello.sh +++ b/doc/src/guide/tutorial/hello.sh @@ -1,8 +1,6 @@ # ANCHOR: init -# $ row init hello-workflow # TODO: Write row init -mkdir -p hello-workflow/workspace +row init hello-workflow cd hello-workflow -touch workflow.toml # ANCHOR_END: init # ANCHOR: create diff --git a/doc/src/guide/tutorial/index.md b/doc/src/guide/tutorial/index.md index 250aad3..a553a25 100644 --- a/doc/src/guide/tutorial/index.md +++ b/doc/src/guide/tutorial/index.md @@ -2,6 +2,7 @@ This section is a tutorial aimed at users who are new to not only **row**, but also HPC resources in general. It assumes that you have already learned the basics of the shell. -If you are unfamiliar with the shell, check out -[Software Carpentry: The Unix Shell](https://swcarpentry.github.io/shell-novice/) +If you are unfamiliar with the shell, check out [Software Carpentry: The Unix Shell][1] or any other book/tutorial you prefer. + +[1]: https://swcarpentry.github.io/shell-novice/ diff --git a/doc/src/guide/tutorial/submit.md b/doc/src/guide/tutorial/submit.md index 5478ecb..84c45c3 100644 --- a/doc/src/guide/tutorial/submit.md +++ b/doc/src/guide/tutorial/submit.md @@ -30,10 +30,11 @@ node, GPU, etc...). Your final script must request the correct `--partition` to the command and charge accounts properly. `clusters.toml` describes rules by which **row** automatically selects partitions when it generates job scripts. -> Note: Feel free to ask on the -> [discussion board](https://github.com/glotzerlab/row/discussions) if you need help +> Note: Feel free to ask on the [discussion board][discussion] if you need help > writing configuration files for your cluster. +[discussion]: https://github.com/glotzerlab/row/discussions + Check that the output of `row show cluster` and `row show launchers` is what you expect before continuing. @@ -54,7 +55,7 @@ Make sure that the script is requesting the correct resources and is routed to t correct **partition**. For example, the example workflow might generate a job script like this on Anvil: -``` +```bash #!/bin/bash #SBATCH --job-name=hello-directory0+2 #SBATCH --partition=shared @@ -89,7 +90,7 @@ on the number of CPU cores quested. ### Submitting jobs When you are *sure* that the **job script** is correct, submit it with: -``` +```bash row submit ``` @@ -122,7 +123,7 @@ Similarly, row show directories hello ``` will show something like: -``` +```plaintext Directory Status Job ID directory0 submitted anvil/5044933 directory1 submitted anvil/5044933 diff --git a/doc/src/images/umich-block-M.svg b/doc/src/images/umich-block-M.svg new file mode 100644 index 0000000..f5fa9ab --- /dev/null +++ b/doc/src/images/umich-block-M.svg @@ -0,0 +1,79 @@ + + + + + + + + + + + diff --git a/doc/src/launchers/built-in.md b/doc/src/launchers/built-in.md index 63a8c3c..6a90dac 100644 --- a/doc/src/launchers/built-in.md +++ b/doc/src/launchers/built-in.md @@ -8,6 +8,6 @@ to see the current launcher configuration. When using OpenMP/MPI hybrid applications, place `"openmp"` first in the list of launchers (`launchers = ["openmp", "mpi"]`) to generate the appropriate command: -``` +```bash OMP_NUM_THREADS=T srun --ntasks=N --cpus-per-task=T command $directory ``` diff --git a/doc/src/license.md b/doc/src/license.md new file mode 100644 index 0000000..07ff395 --- /dev/null +++ b/doc/src/license.md @@ -0,0 +1,498 @@ +# License + +**Row** is available under the following license: + +```plaintext +{{#include ../../LICENSE}} +``` + +## Libraries + +**Row** uses the following open source software packages under their respective +licenses. + +* [clap](https://github.com/clap-rs/clap) + ```plaintext + Copyright (c) Individual contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [clap-verbosity-flag](https://github.com/clap-rs/clap-verbosity-flag) + ```plaintext + Copyright (c) Individual contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [console](https://github.com/console-rs/console) + ```plaintext + Copyright (c) 2017 Armin Ronacher + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [env_logger](https://github.com/rust-cli/env_logger) + ```plaintext + Copyright (c) Individual contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [home](https://github.com/rust-lang/cargo) + ```plaintext + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [human_format](https://github.com/BobGneu/human-format-rs) + ```plaintext + Copyright (c) 2018 Bob Chatman + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [indicatif](https://github.com/console-rs/indicatif) + ```plaintext + Copyright (c) 2017 Armin Ronacher + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [log](https://github.com/rust-lang/log) + ```plaintext + Copyright (c) 2014 The Rust Project Developers + + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [memchr](https://github.com/BurntSushi/memchr) + ```plaintext + Copyright (c) 2015 Andrew Gallant + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + ``` +* [nix](https://github.com/nix-rust/nix) + ```plaintext + Copyright (c) 2015 Carl Lerche + nix-rust Authors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + ``` +* [path-absolutize](https://github.com/magiclen/path-absolutize) + ```plaintext + Copyright (c) 2018 magiclen.org (Ron Li) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [postcard](https://github.com/jamesmunns/postcard) + ```plaintext + Copyright (c) 2019 Anthony James Munns + + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [serde](https://github.com/serde-rs/serde) + ```plaintext + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [serde_json](https://github.com/serde-rs/json) + ```plaintext + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [signal-hook](https://github.com/vorner/signal-hook) + ```plaintext + Copyright (c) 2017 tokio-jsonrpc developers + + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [speedate](https://github.com/pydantic/speedate/) + ```plaintext + Copyright (c) 2022 Samuel Colvin + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` +* [thiserror](https://github.com/dtolnay/thiserror) + ```plaintext + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [toml](https://github.com/toml-rs/toml) + ```plaintext + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [uuid](https://github.com/uuid-rs/uuid) + ```plaintext + Copyright (c) 2014 The Rust Project Developers + Copyright (c) 2018 Ashley Mannix, Christopher Armstrong, Dylan DPC, Hunar Roop Kahlon + + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + ``` +* [wildmatch](https://github.com/becheran/wildmatch) + ```plaintext + Copyright (c) 2020 Armin Becher + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ``` diff --git a/doc/src/release-notes.md b/doc/src/release-notes.md new file mode 100644 index 0000000..864a062 --- /dev/null +++ b/doc/src/release-notes.md @@ -0,0 +1,5 @@ +# Release notes + +## 0.1.0 (not yet released) + +* Initial release. diff --git a/doc/src/row/clean.md b/doc/src/row/clean.md new file mode 100644 index 0000000..83744bc --- /dev/null +++ b/doc/src/row/clean.md @@ -0,0 +1,40 @@ +# clean + +Usage +```bash +row clean [OPTIONS] <--directory|--submitted|--completed|--all> +``` + +`row clean` safely removes cache files generated by **row**. The +[cache concepts page](../guide/concepts/cache.md) describes cases where you might need +to clean the cache. + +## `[OPTIONS]` + +### `--all` + +Remove all caches. + +### `--completed` + +Remove the cache of completed actions on directories. + +### `--directory` + +Remove the directory value cache. + +### `--submitted` + +Remove the cache of submitted jobs. + +### `--force` + +Force an unsafe removal. + + +## Examples + +* Remove the completed cache: + ```bash + row clean --completed + ``` diff --git a/doc/src/row/index.md b/doc/src/row/index.md index ce20f3c..b72e251 100644 --- a/doc/src/row/index.md +++ b/doc/src/row/index.md @@ -10,7 +10,7 @@ row [OPTIONS] * [`show`](show/index.md) * [`submit`](submit.md) * [`scan`](scan.md) -* [`uncomplete`](uncomplete.md) +* [`clean`](clean.md)
You should execute only one instance of row at a time for a given project. diff --git a/doc/src/row/init.md b/doc/src/row/init.md index a3e31ee..4412cd2 100644 --- a/doc/src/row/init.md +++ b/doc/src/row/init.md @@ -1,3 +1,53 @@ # init -TODO: implement and document `row init`. +Usage +```bash +row init [OPTIONS] +``` + +`row init` creates `workflow.toml` and the workspace directory in the given DIRECTORY. +It creates the directory if needed. The default workspace path name is `workspace`. Use +the `--workspace` option to change this. + +Set the `--signac` option to create a project compatible with signac. You must +separately initialize the signac project. + +## `` + +The project directory to create. May be absolute or relative to the current directory. + +## `[OPTIONS]` + +### `--signac` + +Create a signac compatible project. + +* Sets workspace directory to `workspace`. +* Adds `value_file = "signac_statepoint.json"` to the `[workspace]` + configuration. + +### `--workspace` + +(also: `-w`) + +Set the name of the workspace directory. May not be used in combination with +`--signac`. + +## Errors + +`row init` returns an error when a row project already exists at the given DIRECTORY. + +## Examples + +* Create a project in the current directory: + ```bash + row init . + ``` +* Create a signac compatible project in the directory `project`: + ```bash + row init --signac project + ``` +* Create a project where the workspace is named `data`: + ```bash + row init --workspace data project + ``` diff --git a/doc/src/row/scan.md b/doc/src/row/scan.md index f5b7561..7539778 100644 --- a/doc/src/row/scan.md +++ b/doc/src/row/scan.md @@ -7,14 +7,14 @@ row scan [OPTIONS] [DIRECTORIES] `row scan` scans the selected directories for action [products](../workflow/action/index.md#products) and updates the cache -of completed directories appropriately. +of completed directories accordingly. Under normal usage, you should not need to execute `row scan` manually. [`row submit`](submit.md) automatically scans the submitted directories after it executes the action's command. -> Note: `row scan` only **adds** new completed directories. To mark directories as -> no longer completed, use [`row uncomplete`](uncomplete.md). +> Note: `row scan` only **adds** new completed directories. To mark directories +> as no longer completed, use [`row clean`](clean.md). ## `[DIRECTORIES]` @@ -30,4 +30,19 @@ Pass a single `-` to read the directories from stdin (separated by newlines). Set `--action ` to choose which action to scan. By default, **row** scans for products from all actions. -> Note: Unlike other commands, `--action` is **not** a regular expression for *scan*. +> Note: Unlike other commands, `--action` is **not** a wildcard. + +## Examples + +* Scan all directories for all actions: + ```bash + row scan + ``` +* Scan a specific action: + ```bash + row scan --action=action + ``` +* Scan specific directories: + ```bash + row scan directory1 directory2 + ``` diff --git a/doc/src/row/show/cluster.md b/doc/src/row/show/cluster.md index 3fd1cfb..98b6b4e 100644 --- a/doc/src/row/show/cluster.md +++ b/doc/src/row/show/cluster.md @@ -5,21 +5,7 @@ Usage: row show cluster [OPTIONS] ``` -Print the [current cluster configuration](../../clusters/index.md) (or for the cluster -given in `--cluster`). - -Example output: -``` -name = "none" -scheduler = "bash" - -[identify] -always = true - -[[partition]] -name = "none" -prevent_auto_select = false -``` +Print the [current cluster configuration](../../clusters/index.md) in TOML format. ## `[OPTIONS]` @@ -30,3 +16,18 @@ Show the configuration of all clusters: both user-defined and built-in. ### `--name` Show only the cluster's name. + +## Examples + +* Show the autodetected cluster: + ```bash + row show cluster + ``` +* Show the configuration of a specific cluster: + ```bash + row show cluster --cluster=anvil + ``` +* Show all clusters: + ```bash + row show cluster --all + ``` diff --git a/doc/src/row/show/directories.md b/doc/src/row/show/directories.md index d50fdfc..0b79087 100644 --- a/doc/src/row/show/directories.md +++ b/doc/src/row/show/directories.md @@ -5,32 +5,20 @@ Usage: row show directories [OPTIONS] [DIRECTORIES] ``` -Example output: -```plaintext -Directory Status Job /value -dir1 submitted 1432876 0.9 -dir2 submitted 1432876 0.8 -dir3 submitted 1432876 0.7 - -dir4 completed 0.5 -dir5 completed 0.4 -dir6 completed 0.3 -``` - `row show directories` lists each selected directory with its [status](../../guide/concepts/status.md) and scheduler job ID (when submitted) for the given ``. You can also show elements from the directory's value, accessed by [JSON pointer](../../guide/concepts/json-pointers.md). Blank lines separate [groups](../../workflow/action/group.md). +By default, `row show status` displays directories with any status. Set one or more +of `--completed`, `--submitted`, `--eligible`, and `--waiting` to show specific +directories that have specific statuses. + ## `[DIRECTORIES]` List these specific directories. By default, **row** shows all directories that match the action's [include condition](../../workflow/action/group.md#include) -For example: -```bash -row show directories action dir1 dir2 dir3 -``` Pass a single `-` to read the directories from stdin (separated by newlines): ```bash @@ -39,6 +27,20 @@ echo "dir1" | row show directories action - ## `[OPTIONS]` +### `--completed` + +Show directories with the *completed* status. + +### `--eligible` + +Show directories with the *eligible* status. + +### `--n-groups` + +(also: `-n`) + +Limit the number of groups displayed. + ### `--no-header` Hide the header in the output. @@ -47,8 +49,35 @@ Hide the header in the output. Do not write blank lines between groups. +### `--submitted` + +Show directories with the *submitted* status. + ### `--value` Pass `--value ` to add a column of output that shows an element of the directory's value as a JSON string. You may pass `--value` multiple times to include additional columns. + +### `--waiting` + +Show directories with the *waiting* status. + +## Examples + +* Show all the directories for action `one`: + ```bash + row show directories one + ``` +* Show the directory value element `/value`: + ```bash + row show directories action --value=/value + ``` +* Show specific directories: + ```bash + row show directories action directory1 directory2 + ``` +* Show eligible directories + ```bash + row show directories action --eligible + ``` diff --git a/doc/src/row/show/launchers.md b/doc/src/row/show/launchers.md index c1db970..fedd069 100644 --- a/doc/src/row/show/launchers.md +++ b/doc/src/row/show/launchers.md @@ -6,17 +6,7 @@ row show launchers [OPTIONS] ``` Print the [launchers](../../launchers/index.md) defined for the current cluster (or the -cluster given in `--cluster`). - -Example output: -``` -[mpi] -executable = "mpirun" -processes = "-n " - -[openmp] -threads_per_process = "OMP_NUM_THREADS=" -``` +cluster given in `--cluster`). The output is TOML formatted. This includes the user-provided launchers in [`launchers.toml`](../../launchers/index.md) and the built-in launchers (or the user-provided overrides). @@ -26,3 +16,18 @@ and the built-in launchers (or the user-provided overrides). ### `--all` Show the launcher configurations for all clusters. + +## Examples + +* Show the launchers for the autodetected cluster: + ```bash + row show launchers + ``` +* Show the launchers for a specific cluster: + ```bash + row show launchers --cluster=anvil + ``` +* Show all launchers: + ```bash + row show launchers --all + ``` diff --git a/doc/src/row/show/status.md b/doc/src/row/show/status.md index ccadfed..b43947c 100644 --- a/doc/src/row/show/status.md +++ b/doc/src/row/show/status.md @@ -5,26 +5,16 @@ Usage: row show status [OPTIONS] [DIRECTORIES] ``` -Example output: -```plaintext -Action Completed Submitted Eligible Waiting Remaining cost -one 1000 100 900 0 24K CPU-hours -two 0 200 800 1000 8K GPU-hours -``` - -For each action, the summary details the number of directories in each -[status](../../guide/concepts/status.md). -`row show status` also estimates the remaining cost in either CPU-hours or GPU-hours -based on the number of submitted, eligible, and waiting jobs and the -[resources used by the action](../../workflow/action/resources.md). +`row show status` prints a summary of all directories in the workspace. +The summary includes the number of directories in each +[status](../../guide/concepts/status.md) and an estimate of the remaining cost in either +CPU-hours or GPU-hours based on the number of submitted, eligible, and waiting jobs and +the [resources used by the action](../../workflow/action/resources.md). ## `[DIRECTORIES]` Show the status of these specific directories. By default, **row** shows the status for -the entire workspace. For example: -```bash -row show status dir1 dir2 dir3 -``` +the entire workspace. Pass a single `-` to read the directories from stdin (separated by newlines): ```bash @@ -43,3 +33,22 @@ shows the status of all actions. `` is a wildcard pattern. ### `--no-header` Hide the header in the output. + +## Examples + +* Show the status of the entire workspace: + ```bash + row show status + ``` +* Show the status of a specific action: + ```bash + row show status --action=action + ``` +* Show the status of all action names that match a wildcard pattern: + ```bash + row show status --action='project*' + ``` +* Show the status of specific directories in the workspace: + ```bash + row show status directory1 directory2 + ``` diff --git a/doc/src/row/submit.md b/doc/src/row/submit.md index 295d944..66a58df 100644 --- a/doc/src/row/submit.md +++ b/doc/src/row/submit.md @@ -8,16 +8,12 @@ row submit [OPTIONS] [DIRECTORIES] `row submit` submits jobs to the scheduler. First it determines the [status](../guide/concepts/status.md) of all the given directories for the selected actions. Then it forms [groups](../workflow/action/group.md) and submits one job for -each group. Pass `--dry-run` to see what will be submitted. +each group. Pass `--dry-run` to see the script(s) that will be submitted. ## `[DIRECTORIES]` Submit eligible jobs for these specific directories. By default, **row** submits -the entire workspace. For example: -```bash -row submit dir1 dir2 dir3 -``` - +the entire workspace. ## `[OPTIONS]` ### `--action` @@ -39,3 +35,30 @@ Set `-n ` to limit the number of submitted jobs. **Row** will submit up to th ### `--yes` Skip the interactive confirmation. + +## Examples + +* Print the job script(s) that will be submitted: + ```bash + row submit --dry-run + ``` +* Submit jobs for all eligible directories: + ```bash + row submit + ``` +* Submit the first eligible job: + ```bash + row submit -n 1 + ``` +* Submit jobs for a specific action: + ```bash + row submit --action=action + ``` +* Submit jobs for all actions that match a wildcard pattern: + ```bash + row submit --action='project*' + ``` +* Submit jobs on specific directories: + ```bash + row submit directory1 directory2 + ``` diff --git a/doc/src/row/uncomplete.md b/doc/src/row/uncomplete.md deleted file mode 100644 index 92f2284..0000000 --- a/doc/src/row/uncomplete.md +++ /dev/null @@ -1,3 +0,0 @@ -# uncomplete - -TODO: implement and document `row uncomplete`. diff --git a/doc/theme/index.hbs b/doc/theme/index.hbs new file mode 100644 index 0000000..f86b1c2 --- /dev/null +++ b/doc/theme/index.hbs @@ -0,0 +1,359 @@ + + + + + + {{ title }} + {{#if is_print }} + + {{/if}} + {{#if base_url}} + + {{/if}} + + + + {{> head}} + + + + + + {{#if favicon_svg}} + + {{/if}} + {{#if favicon_png}} + + {{/if}} + + + + {{#if print_enable}} + + {{/if}} + + + + {{#if copy_fonts}} + + {{/if}} + + + + + + + + {{#each additional_css}} + + {{/each}} + + {{#if mathjax_support}} + + + {{/if}} + + +
+ + + + + + + + + + + + + + + + + + + +
+ +
+ {{> header}} + + + + {{#if search_enabled}} + + {{/if}} + + + + +
+
+ {{{ content }}} + + +
+
+

Development of row is led by the Glotzer Group at the University of Michigan. +

Copyright © 2024 The Regents of the University of Michigan. +

+
+ + University of Michigan logo + +
+
+
+ + +
+
+ + + +
+ + {{#if live_reload_endpoint}} + + + {{/if}} + + {{#if google_analytics}} + + + {{/if}} + + {{#if playground_line_numbers}} + + {{/if}} + + {{#if playground_copyable}} + + {{/if}} + + {{#if playground_js}} + + + + + + {{/if}} + + {{#if search_js}} + + + + {{/if}} + + + + + + + {{#each additional_js}} + + {{/each}} + + {{#if is_print}} + {{#if mathjax_support}} + + {{else}} + + {{/if}} + {{/if}} + +
+ + diff --git a/src/builtin.rs b/src/builtin.rs index 2ce01bc..dbebbb0 100644 --- a/src/builtin.rs +++ b/src/builtin.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use std::collections::HashMap; use crate::cluster::{self, Cluster, IdentificationMethod, Partition, SchedulerType}; @@ -247,6 +250,8 @@ fn greatlakes() -> Cluster { } } +// TODO: Add/test Frontier and Andes. + fn none() -> Cluster { // Fallback none cluster. Cluster { diff --git a/src/cli.rs b/src/cli.rs index 517185d..3d95cfd 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,5 +1,10 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + +pub mod clean; pub mod cluster; pub mod directories; +pub mod init; pub mod launchers; pub mod scan; pub mod status; @@ -42,7 +47,9 @@ pub struct GlobalOptions { #[arg(long, global = true, env = "ROW_CLEAR_PROGRESS", display_order = 2)] pub clear_progress: bool, - /// Check the job submission status on the given cluster. Autodetected by default. + /// Check the job submission status on the given cluster. + /// + /// Autodetected by default. #[arg(long, global = true, env = "ROW_CLUSTER", display_order = 2)] cluster: Option, } @@ -62,29 +69,212 @@ pub enum ColorMode { #[derive(Subcommand, Debug)] pub enum ShowCommands { /// Show the current state of the workflow. + /// + /// `row show status` prints a summary of all directories in the workspace. + /// The summary includes the number of directories in each status and an + /// estimate of the remaining cost in either CPU-hours or GPU-hours based + /// on the number of submitted, eligible, and waiting jobs and the + /// resources used by the action. + /// + /// EXAMPLES + /// + /// * Show the status of the entire workspace: + /// + /// row show status + /// + /// * Show the status of a specific action: + /// + /// row show status --action=action + /// + /// * Show the status of all action names that match a wildcard pattern: + /// + /// row show status --action='project*' + /// + /// * Show the status of specific directories in the workspace: + /// + /// row show status directory1 directory2 + /// Status(status::Arguments), /// List directories in the workspace. + /// + /// `row show directories` lists each selected directory with its status + /// and scheduler job ID (when submitted). for the given `. You + /// can also show elements from the directory's value, accessed by JSON + /// pointer. Blank lines separate groups. + /// + /// By default, `row show status` displays directories with any status. Set + /// one or more of `--completed`, `--submitted`, `--eligible`, and + /// `--waiting` to show specific directories that have specific statuses. + /// + /// EXAMPLES + /// + /// * Show all the directories for action `one`: + /// + /// row show directories one + /// + /// * Show the directory value element `/value`: + /// + /// row show directories action --value=/value + /// + /// * Show specific directories: + /// + /// row show directories action directory1 directory2 + /// + /// * Show eligible directories + /// + /// row show directories action --eligible + /// Directories(directories::Arguments), /// Show the cluster configuration. + /// + /// Print the current cluster configuration in TOML format. + /// + /// EXAMPLES + /// + /// * Show the autodetected cluster: + /// + /// row show cluster + /// + /// * Show the configuration of a specific cluster: + /// + /// row show cluster --cluster=anvil + /// + /// * Show all clusters: + /// + /// row show cluster --all + /// Cluster(cluster::Arguments), /// Show launcher configurations. + /// + /// Print the launchers defined for the current cluster (or the cluster + /// given in `--cluster`). The output is TOML formatted. + /// + /// This includes the user-provided launchers in `launchers.toml` and the + /// built-in launchers (or the user-provided overrides). + /// + /// EXAMPLES + /// + ///* Show the launchers for the autodetected cluster: + /// + /// row show launchers + /// + ///* Show the launchers for a specific cluster: + /// + /// row show launchers --cluster=anvil + /// + ///* Show all launchers: + /// + /// row show launchers --all + /// Launchers(launchers::Arguments), } #[derive(Subcommand, Debug)] pub enum Commands { + /// Initialize a new project. + /// + /// `row init` creates `workflow.toml` and the workspace directory in the + /// given DIRECTORY. It creates the directory if needed. The default workspace + /// path name is `workspace`. Use the `--workspace` option to change this. + /// + /// Set the `--signac` option to create a project compatible with signac. + /// You must separately initialize the signac project. + /// + /// ERRORS + /// + /// `row init` returns an error when a row project already exists at the + /// given DIRECTORY. + /// + /// EXAMPLES + /// + /// * Create a project in the current directory: + /// + /// row init . + /// + /// * Create a signac compatible project in the directory `project`: + /// + /// row init --signac project + /// + /// * Create a project where the workspace is named `data`: + /// + /// row init --workspace data project + /// + Init(init::Arguments), + /// Show properties of the workspace. #[command(subcommand)] Show(ShowCommands), /// Scan the workspace for completed actions. + /// + /// `row scan` scans the selected directories for action products and + /// updates the cache of completed directories accordingly. + /// + /// EXAMPLES + /// + /// * Scan all directories for all actions: + /// + /// row scan + /// + /// * Scan a specific action: + /// + /// row scan --action=action + /// + /// * Scan specific directories: + /// + /// row scan directory1 directory2 + /// Scan(scan::Arguments), /// Submit workflow actions to the scheduler. + /// + /// `row submit` submits jobs to the scheduler. First it determines the + /// status of all the given directories for the selected actions. Then it + /// forms groups and submits one job for each group. Pass `--dry-run` to see + /// the script(s) that will be submitted. + /// + /// EXAMPLES + /// + /// * Print the job script(s) that will be submitted: + /// + /// row submit --dry-run + /// + /// * Submit jobs for all eligible directories: + /// + /// row submit + /// + /// * Submit the first eligible job: + /// + /// row submit -n 1 + /// + /// * Submit jobs for a specific action: + /// + /// row submit --action=action + /// + /// * Submit jobs for all actions that match a wildcard pattern: + /// + /// row submit --action='project*' + /// + /// * Submit jobs on specific directories: + /// + /// row submit directory1 directory2 + /// Submit(submit::Arguments), + + /// Remove cache files. + /// + /// `row clean` safely removes cache files generated by row. + /// + /// EXAMPLES + /// + /// * Remove the completed cache: + /// + /// row clean --completed + /// + Clean(clean::Arguments), } /// Parse directories passed in on the command line. diff --git a/src/cli/clean.rs b/src/cli/clean.rs new file mode 100644 index 0000000..48b2cfe --- /dev/null +++ b/src/cli/clean.rs @@ -0,0 +1,115 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + +use clap::Args; +use log::{debug, info, warn}; +use std::error::Error; +use std::{fs, io}; + +use crate::cli::GlobalOptions; +use row::project::Project; +use row::MultiProgressContainer; +use row::{ + COMPLETED_CACHE_FILE_NAME, DATA_DIRECTORY_NAME, DIRECTORY_CACHE_FILE_NAME, + SUBMITTED_CACHE_FILE_NAME, +}; + +#[derive(Args, Debug)] +pub struct Arguments { + #[command(flatten)] + selection: Selection, + + /// Force removal of the completed and/or submitted cache when there are submitted jobs. + #[arg(long, display_order = 0)] + force: bool, +} + +#[derive(Args, Debug)] +#[group(required = true, multiple = false)] +#[allow(clippy::struct_excessive_bools)] +pub struct Selection { + /// Remove the directory cache. + #[arg(long, display_order = 0)] + directory: bool, + + /// Remove the submitted cache. + #[arg(long, display_order = 0)] + submitted: bool, + + /// Remove the completed cache. + #[arg(long, display_order = 0)] + completed: bool, + + /// Remove all caches. + #[arg(long, display_order = 0)] + all: bool, +} + +/// Remove row cache files. +pub fn clean( + options: &GlobalOptions, + args: &Arguments, + multi_progress: &mut MultiProgressContainer, +) -> Result<(), Box> { + debug!("Cleaning cache files."); + let mut project = Project::open(options.io_threads, &options.cluster, multi_progress)?; + + // Delete all existing completion staging files. + project.close(multi_progress)?; + + let selection = &args.selection; + + let num_submitted = project.state().num_submitted(); + if num_submitted > 0 { + let force_needed = selection.completed || selection.submitted || selection.all; + + if force_needed { + warn!("There are {num_submitted} directories with submitted jobs."); + } + if selection.submitted || selection.all { + warn!("The submitted cache is not recoverable. Row may resubmit running jobs."); + } + if selection.completed || selection.all { + warn!("These jobs may add to the completed cache after it is cleaned."); + } + if force_needed && !args.force { + warn!("You should wait for these jobs to complete."); + return Err(Box::new(row::Error::ForceCleanNeeded)); + } + } + + let data_directory = project.workflow().root.join(DATA_DIRECTORY_NAME); + + if selection.submitted || selection.all { + let path = data_directory.join(SUBMITTED_CACHE_FILE_NAME); + info!("Removing '{}'.", path.display()); + if let Err(error) = fs::remove_file(&path) { + match error.kind() { + io::ErrorKind::NotFound => (), + _ => return Err(Box::new(row::Error::FileRemove(path.clone(), error))), + } + } + } + if selection.completed || selection.all { + let path = data_directory.join(COMPLETED_CACHE_FILE_NAME); + info!("Removing '{}'.", path.display()); + if let Err(error) = fs::remove_file(&path) { + match error.kind() { + io::ErrorKind::NotFound => (), + _ => return Err(Box::new(row::Error::FileRemove(path.clone(), error))), + } + } + } + if selection.directory || selection.all { + let path = data_directory.join(DIRECTORY_CACHE_FILE_NAME); + info!("Removing '{}'.", path.display()); + if let Err(error) = fs::remove_file(&path) { + match error.kind() { + io::ErrorKind::NotFound => (), + _ => return Err(Box::new(row::Error::FileRemove(path.clone(), error))), + } + } + } + + Ok(()) +} diff --git a/src/cli/cluster.rs b/src/cli/cluster.rs index 548cc82..b3f82e7 100644 --- a/src/cli/cluster.rs +++ b/src/cli/cluster.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use clap::Args; use log::{debug, info}; use std::error::Error; diff --git a/src/cli/directories.rs b/src/cli/directories.rs index 099e63c..69aecfc 100644 --- a/src/cli/directories.rs +++ b/src/cli/directories.rs @@ -1,17 +1,21 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use clap::Args; use console::Style; -use log::debug; +use log::{debug, warn}; use std::collections::HashSet; use std::error::Error; use std::io::Write; use std::path::PathBuf; use crate::cli::{self, GlobalOptions}; -use crate::ui::{Alignment, Item, Table}; +use crate::ui::{Alignment, Item, Row, Table}; use row::project::Project; use row::MultiProgressContainer; #[derive(Args, Debug)] +#[allow(clippy::struct_excessive_bools)] pub struct Arguments { /// Select the action to scan (defaults to all). action: String, @@ -30,12 +34,33 @@ pub struct Arguments { /// Show an element of each directory's value (repeat to show multiple elements). #[arg(long, value_name = "JSON POINTER", display_order = 0)] value: Vec, + + /// Limit the number of groups displayed. + #[arg(short, long, display_order = 0)] + n_groups: Option, + + /// Show completed directories. + #[arg(long, display_order = 0)] + completed: bool, + + /// Show submitted + #[arg(long, display_order = 0)] + submitted: bool, + + /// Show eligible directories. + #[arg(long, display_order = 0)] + eligible: bool, + + /// Show waiting directories. + #[arg(long, display_order = 0)] + waiting: bool, } /// Show directories that match an action. /// /// Print a human-readable list of directories, their status, job ID, and value(s). /// +#[allow(clippy::too_many_lines)] pub fn directories( options: &GlobalOptions, args: Arguments, @@ -58,20 +83,49 @@ pub fn directories( project.find_matching_directories(action, query_directories.clone())?; let status = project.separate_by_status(action, matching_directories.clone())?; - let completed = HashSet::::from_iter(status.completed); - let submitted = HashSet::::from_iter(status.submitted); - let eligible = HashSet::::from_iter(status.eligible); - let waiting = HashSet::::from_iter(status.waiting); + let completed = HashSet::::from_iter(status.completed.clone()); + let submitted = HashSet::::from_iter(status.submitted.clone()); + let eligible = HashSet::::from_iter(status.eligible.clone()); + let waiting = HashSet::::from_iter(status.waiting.clone()); + + // Show directories with selected statuses. + let mut show_completed = args.completed; + let mut show_submitted = args.submitted; + let mut show_eligible = args.eligible; + let mut show_waiting = args.waiting; + if !show_completed && !show_submitted && !show_eligible && !show_waiting { + show_completed = true; + show_submitted = true; + show_eligible = true; + show_waiting = true; + } - // TODO: filter shown directories by status, also add --n_groups option - let groups = project.separate_into_groups(action, matching_directories)?; + let mut selected_directories = Vec::with_capacity(matching_directories.len()); + if show_completed { + selected_directories.extend(status.completed); + } + if show_submitted { + selected_directories.extend(status.submitted); + } + if show_eligible { + selected_directories.extend(status.eligible); + } + if show_waiting { + selected_directories.extend(status.waiting); + } + + let groups = project.separate_into_groups(action, selected_directories)?; let mut table = Table::new().with_hide_header(args.no_header); table.header = vec![ Item::new("Directory".to_string(), Style::new().underlined()), Item::new("Status".to_string(), Style::new().underlined()), - Item::new("Job ID".to_string(), Style::new().underlined()), ]; + if show_submitted || show_completed { + table + .header + .push(Item::new("Job ID".to_string(), Style::new().underlined())); + } for pointer in &args.value { table .header @@ -79,6 +133,12 @@ pub fn directories( } for (group_idx, group) in groups.iter().enumerate() { + if let Some(n) = args.n_groups { + if group_idx >= n { + break; + } + } + for directory in group { // Format the directory status. let status = if completed.contains(directory) { @@ -105,17 +165,24 @@ pub fn directories( row.push(status); // Job ID - let submitted = project.state().submitted(); - - // Values - if let Some((cluster, job_id)) = - submitted.get(&action.name).and_then(|d| d.get(directory)) - { - row.push(Item::new(format!("{cluster}/{job_id}"), Style::new())); - } else { - row.push(Item::new(String::new(), Style::new())); + if show_submitted || show_completed { + let submitted = project.state().submitted(); + + // Values + if let Some((cluster, job_id)) = + submitted.get(&action.name).and_then(|d| d.get(directory)) + { + row.push(Item::new(format!("{cluster}/{job_id}"), Style::new())); + } else { + row.push(Item::new(String::new(), Style::new())); + } } + for pointer in &args.value { + if !pointer.is_empty() && !pointer.starts_with('/') { + warn!("The JSON pointer '{pointer}' does not appear valid. Did you mean '/{pointer}'?"); + } + let value = project.state().values()[directory] .pointer(pointer) .ok_or_else(|| { @@ -126,18 +193,11 @@ pub fn directories( ); } - table.items.push(row); + table.rows.push(Row::Items(row)); } if !args.no_separate_groups && group_idx != groups.len() - 1 { - let mut row = vec![ - Item::new(String::new(), Style::new()), - Item::new(String::new(), Style::new()), - ]; - for _ in &args.value { - row.push(Item::new(String::new(), Style::new())); - } - table.items.push(row); + table.rows.push(Row::Separator); } } diff --git a/src/cli/init.rs b/src/cli/init.rs new file mode 100644 index 0000000..19c6fe4 --- /dev/null +++ b/src/cli/init.rs @@ -0,0 +1,122 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + +use clap::Args; +use log::{debug, info, trace, warn}; +use path_absolutize::Absolutize; +use std::fmt::Write as _; +use std::fs; +use std::io::Write; +use std::path::{self, Path, PathBuf}; + +use crate::cli::GlobalOptions; +use row::{Error, DATA_DIRECTORY_NAME}; + +#[derive(Args, Debug)] +pub struct Arguments { + /// Configure `workflow.toml` for signac. + #[arg(long, group = "workspace_group", display_order = 0)] + signac: bool, + + /// The workspace directory name. + #[arg(short, long, default_value_t=String::from("workspace"), group="workspace_group", display_order = 0)] + workspace: String, + + /// Directory to initialize. + #[arg(display_order = 0)] + directory: PathBuf, +} + +fn is_project(path: &Path) -> Result<(bool, PathBuf), Error> { + let mut path = PathBuf::from(path); + + let found = loop { + path.push("workflow.toml"); + trace!("Checking {}.", path.display()); + + if path + .try_exists() + .map_err(|e| Error::DirectoryRead(path.clone(), e))? + { + break true; + } + + path.pop(); + if !path.pop() { + break false; + } + }; + + path.pop(); + + Ok((found, path)) +} + +/// Initialize a new row project directory. +pub fn init( + _options: &GlobalOptions, + args: &Arguments, + output: &mut W, +) -> Result<(), Box> { + debug!("Scanning the workspace for completed actions."); + + if args.workspace.contains(path::MAIN_SEPARATOR_STR) { + return Err(Box::new(row::Error::WorkspacePathNotRelative( + args.workspace.clone(), + ))); + } + + let project_directory = args.directory.absolutize()?; + let (project_found, existing_path) = is_project(&project_directory)?; + + match (project_found, existing_path == project_directory) { + (true, true) => return Err(Box::new(row::Error::ProjectExists(existing_path))), + (true, false) => return Err(Box::new(row::Error::ParentProjectExists(existing_path))), + (_, _) => (), + } + + if project_directory.clone().join(DATA_DIRECTORY_NAME).exists() { + return Err(Box::new(row::Error::ProjectCacheExists( + project_directory.into(), + ))); + } + + if project_directory.exists() { + warn!("'{}' already exists.", project_directory.display()); + } + + let workspace_directory = project_directory.clone().join(&args.workspace); + info!("Creating directory '{}'", workspace_directory.display()); + fs::create_dir_all(&workspace_directory) + .map_err(|e| Error::DirectoryCreate(workspace_directory.clone(), e))?; + + let mut workflow = String::new(); + if args.signac { + let _ = writeln!( + workflow, + r#"[workspace] +value_file = "signac_statepoint.json""# + ); + } + + if args.workspace != "workspace" { + let _ = writeln!( + workflow, + r#"[workspace] +path = '{}'"#, + args.workspace + ); + } + + let workflow_path = project_directory.clone().join("workflow.toml"); + info!("Creating file '{}'", workflow_path.display()); + fs::write(&workflow_path, &workflow).map_err(|e| Error::FileWrite(workflow_path, e))?; + + writeln!( + output, + "Created row project in '{}'", + project_directory.display() + )?; + + Ok(()) +} diff --git a/src/cli/launchers.rs b/src/cli/launchers.rs index 2c044a1..8988ee5 100644 --- a/src/cli/launchers.rs +++ b/src/cli/launchers.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use clap::Args; use log::{debug, info}; use std::error::Error; diff --git a/src/cli/scan.rs b/src/cli/scan.rs index dcb5a04..544fd15 100644 --- a/src/cli/scan.rs +++ b/src/cli/scan.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use clap::Args; use log::{debug, info, trace, warn}; use postcard; diff --git a/src/cli/status.rs b/src/cli/status.rs index 9303b5b..b7a1e99 100644 --- a/src/cli/status.rs +++ b/src/cli/status.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use clap::Args; use console::Style; use indicatif::HumanCount; @@ -8,7 +11,7 @@ use std::path::PathBuf; use wildmatch::WildMatch; use crate::cli::{self, GlobalOptions}; -use crate::ui::{Alignment, Item, Table}; +use crate::ui::{Alignment, Item, Row, Table}; use row::project::{Project, Status}; use row::workflow::ResourceCost; use row::MultiProgressContainer; @@ -131,7 +134,9 @@ pub fn status( cost = cost + action.resources.cost(group.len()); } - table.items.push(make_row(&action.name, &status, &cost)); + table + .rows + .push(Row::Items(make_row(&action.name, &status, &cost))); } if matching_action_count == 0 { diff --git a/src/cli/submit.rs b/src/cli/submit.rs index 76c940f..bb07528 100644 --- a/src/cli/submit.rs +++ b/src/cli/submit.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use clap::Args; use console::style; use indicatif::HumanCount; @@ -83,6 +86,17 @@ pub fn submit( let status = project.separate_by_status(action, matching_directories)?; let groups = project.separate_into_groups(action, status.eligible)?; + if action.group.submit_whole { + let whole_groups = + project.separate_into_groups(action, project.state().list_directories())?; + for group in &groups { + if !whole_groups.contains(group) { + return Err(Box::new(row::Error::PartialGroupSubmission( + action.name.clone(), + ))); + } + } + } action_groups.push((&action, groups)); } @@ -133,8 +147,6 @@ pub fn submit( return Ok(()); } - // TODO: Validate submit_whole - if args.dry_run { let scheduler = project.scheduler(); info!("Would submit the following scripts..."); diff --git a/src/cluster.rs b/src/cluster.rs index 8cb50f7..e47a071 100644 --- a/src/cluster.rs +++ b/src/cluster.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use log::{debug, info, trace}; use serde::{Deserialize, Serialize}; use std::env; diff --git a/src/expr.rs b/src/expr.rs index fb1f7c1..f12c3b8 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use serde_json::Value; use std::cmp::Ordering; use std::iter; diff --git a/src/format.rs b/src/format.rs index a9bdfc2..31e2b28 100644 --- a/src/format.rs +++ b/src/format.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use std::fmt; use std::time::Duration; diff --git a/src/launcher.rs b/src/launcher.rs index 03421bc..4cd87b1 100644 --- a/src/launcher.rs +++ b/src/launcher.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use log::trace; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/src/lib.rs b/src/lib.rs index bd038f4..12ff797 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + #![warn(clippy::pedantic)] #![allow(clippy::cast_precision_loss)] #![allow(clippy::cast_possible_wrap)] @@ -27,9 +30,9 @@ pub const DATA_DIRECTORY_NAME: &str = ".row"; pub const COMPLETED_DIRECTORY_NAME: &str = "completed"; pub const MIN_PROGRESS_BAR_SIZE: usize = 1; -const VALUE_CACHE_FILE_NAME: &str = "values.json"; -const COMPLETED_CACHE_FILE_NAME: &str = "completed.postcard"; -const SUBMITTED_CACHE_FILE_NAME: &str = "submitted.postcard"; +pub const DIRECTORY_CACHE_FILE_NAME: &str = "directories.json"; +pub const COMPLETED_CACHE_FILE_NAME: &str = "completed.postcard"; +pub const SUBMITTED_CACHE_FILE_NAME: &str = "submitted.postcard"; /// Hold a `MultiProgress` and all of its progress bars. /// @@ -174,18 +177,30 @@ pub enum Error { #[error("Action '{0}' not found in the workflow.")] ActionNotFound(String), + #[error("A row project already exists in '{0}'.")] + ProjectExists(PathBuf), + + #[error("A row project already exists in the parent directory '{0}'.")] + ParentProjectExists(PathBuf), + + #[error("The cache directory '.row' already exists in '{0}'.")] + ProjectCacheExists(PathBuf), + + #[error("workspace must be a relative path name, got '{0}'.")] + WorkspacePathNotRelative(String), + + #[error("There are submitted jobs. Rerun with --force to bypass this check.")] + ForceCleanNeeded, + + #[error("Attempting partial submission of action '{0}' when `submit_whole=true`.")] + PartialGroupSubmission(String), + // thread errors #[error("Unexpected error communicating between threads in 'find_completed_directories'.")] CompletedDirectoriesSend(#[from] mpsc::SendError<(PathBuf, String)>), #[error("Unexpected error communicating between threads in 'read_values'.")] ReadValuesSend(#[from] mpsc::SendError<(PathBuf, Value)>), - // evalexpr errors - // #[error("Invalid number {0}")] - // InvalidNumber(String), - - // #[error("Evalexpr error: {0}")] - // Evalexpr(#[from] EvalexprError), } impl MultiProgressContainer { diff --git a/src/main.rs b/src/main.rs index 512eacf..5a062bb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + #![warn(clippy::pedantic)] use clap::Parser; @@ -67,6 +70,9 @@ fn main_detail() -> Result<(), Box> { let mut multi_progress_container = MultiProgressContainer::new(multi_progress.clone()); match options.command { + Some(Commands::Init(args)) => { + cli::init::init(&options.global, &args, &mut output)?; + } Some(Commands::Show(show)) => match show { ShowCommands::Status(args) => cli::status::status( &options.global, @@ -96,6 +102,9 @@ fn main_detail() -> Result<(), Box> { &mut multi_progress_container, &mut output, )?, + Some(Commands::Clean(args)) => { + cli::clean::clean(&options.global, &args, &mut multi_progress_container)?; + } None => (), } diff --git a/src/progress_styles.rs b/src/progress_styles.rs index 45ba33e..b490534 100644 --- a/src/progress_styles.rs +++ b/src/progress_styles.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use indicatif::{ProgressState, ProgressStyle}; use std::fmt::Write; diff --git a/src/project.rs b/src/project.rs index 4d7d91c..38a0798 100644 --- a/src/project.rs +++ b/src/project.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use indicatif::ProgressBar; use log::{debug, trace, warn}; use serde_json::Value; diff --git a/src/scheduler.rs b/src/scheduler.rs index 38997e4..6f2c8d6 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + pub mod bash; pub mod slurm; diff --git a/src/scheduler/bash.rs b/src/scheduler/bash.rs index 7ea5145..28c8357 100644 --- a/src/scheduler/bash.rs +++ b/src/scheduler/bash.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use log::{debug, error, trace}; use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; diff --git a/src/scheduler/slurm.rs b/src/scheduler/slurm.rs index f309227..df25492 100644 --- a/src/scheduler/slurm.rs +++ b/src/scheduler/slurm.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use log::{debug, error, trace}; use std::collections::{HashMap, HashSet}; use std::fmt::Write as _; diff --git a/src/state.rs b/src/state.rs index 09fb8c3..3f75423 100644 --- a/src/state.rs +++ b/src/state.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use indicatif::ProgressBar; use log::{debug, trace, warn}; use serde::{Deserialize, Serialize}; @@ -6,17 +9,31 @@ use std::collections::{HashMap, HashSet}; use std::fs::{self, File}; use std::io; use std::io::prelude::*; +use std::os::unix::fs::MetadataExt; use std::path::PathBuf; use crate::workflow::Workflow; use crate::{ progress_styles, workspace, Error, MultiProgressContainer, COMPLETED_CACHE_FILE_NAME, - COMPLETED_DIRECTORY_NAME, DATA_DIRECTORY_NAME, MIN_PROGRESS_BAR_SIZE, - SUBMITTED_CACHE_FILE_NAME, VALUE_CACHE_FILE_NAME, + COMPLETED_DIRECTORY_NAME, DATA_DIRECTORY_NAME, DIRECTORY_CACHE_FILE_NAME, + MIN_PROGRESS_BAR_SIZE, SUBMITTED_CACHE_FILE_NAME, }; type SubmittedJobs = HashMap>; +/// Directory cache +/// +/// Cache the directory values and store the last modified time. +/// +#[derive(Debug, Default, Deserialize, PartialEq, Serialize)] +pub struct DirectoryCache { + /// File system modification time of the workspace. + modified_time: (i64, i64), + + /// Directory values. + values: HashMap, +} + /// The state of the project. /// /// `State` collects the following information on the workspace and manages cache files @@ -30,8 +47,8 @@ type SubmittedJobs = HashMap>; /// #[derive(Debug, Default, Deserialize, PartialEq, Serialize)] pub struct State { - /// The cached value of each directory. - values: HashMap, + /// The directory cache. + directory_cache: DirectoryCache, /// Completed directories for each action. completed: HashMap>, @@ -43,7 +60,7 @@ pub struct State { completed_file_names: Vec, /// Set to true when `values` is modified from the on-disk cache. - values_modified: bool, + directories_modified: bool, /// Set to true when `completed` is modified from the on-disk cache. completed_modified: bool, @@ -55,7 +72,7 @@ pub struct State { impl State { /// Get the directory values. pub fn values(&self) -> &HashMap { - &self.values + &self.directory_cache.values } /// Get the set of directories completed for a given action. @@ -68,6 +85,16 @@ impl State { &self.submitted } + /// Get the number of submitted jobs. + pub fn num_submitted(&self) -> usize { + let mut result = 0; + + for v in self.submitted.values() { + result += v.len(); + } + result + } + /// Test whether a given directory has a submitted job for the given action. pub fn is_submitted(&self, action_name: &str, directory: &PathBuf) -> bool { if let Some(submitted_directories) = self.submitted.get(action_name) { @@ -130,8 +157,8 @@ impl State { /// List all directories in the state. pub fn list_directories(&self) -> Vec { trace!("Listing all directories in project."); - let mut result = Vec::with_capacity(self.values.len()); - result.extend(self.values.keys().cloned()); + let mut result = Vec::with_capacity(self.values().len()); + result.extend(self.values().keys().cloned()); result } @@ -142,11 +169,11 @@ impl State { /// pub fn from_cache(workflow: &Workflow) -> Result { let mut state = State { - values: Self::read_value_cache(workflow)?, + directory_cache: Self::read_directory_cache(workflow)?, completed: Self::read_completed_cache(workflow)?, submitted: Self::read_submitted_cache(workflow)?, completed_file_names: Vec::new(), - values_modified: false, + directories_modified: false, completed_modified: false, submitted_modified: false, }; @@ -161,17 +188,17 @@ impl State { Ok(state) } - /// Read the value cache from disk. - fn read_value_cache(workflow: &Workflow) -> Result, Error> { + /// Read the directory cache from disk. + fn read_directory_cache(workflow: &Workflow) -> Result { let data_directory = workflow.root.join(DATA_DIRECTORY_NAME); - let value_file = data_directory.join(VALUE_CACHE_FILE_NAME); + let directory_file = data_directory.join(DIRECTORY_CACHE_FILE_NAME); - match fs::read(&value_file) { + match fs::read(&directory_file) { Ok(bytes) => { - debug!("Reading cache '{}'.", value_file.display().to_string()); + debug!("Reading cache '{}'.", directory_file.display().to_string()); - let result = - serde_json::from_slice(&bytes).map_err(|e| Error::JSONParse(value_file, e))?; + let result = serde_json::from_slice(&bytes) + .map_err(|e| Error::JSONParse(directory_file, e))?; Ok(result) } @@ -179,12 +206,15 @@ impl State { io::ErrorKind::NotFound => { trace!( "'{}' not found, initializing default values.", - value_file.display().to_string() + directory_file.display().to_string() ); - Ok(HashMap::new()) + Ok(DirectoryCache { + modified_time: (0, 0), + values: HashMap::new(), + }) } - _ => Err(Error::FileRead(value_file, error)), + _ => Err(Error::FileRead(directory_file, error)), }, } } @@ -255,9 +285,9 @@ impl State { workflow: &Workflow, multi_progress: &mut MultiProgressContainer, ) -> Result<(), Error> { - if self.values_modified { - self.save_value_cache(workflow)?; - self.values_modified = false; + if self.directories_modified { + self.save_directory_cache(workflow)?; + self.directories_modified = false; } if self.completed_modified { @@ -273,22 +303,23 @@ impl State { Ok(()) } - /// Save the value cache to the filesystem. - fn save_value_cache(&self, workflow: &Workflow) -> Result<(), Error> { + /// Save the directory cache to the filesystem. + fn save_directory_cache(&self, workflow: &Workflow) -> Result<(), Error> { let data_directory = workflow.root.join(DATA_DIRECTORY_NAME); - let value_file = data_directory.join(VALUE_CACHE_FILE_NAME); + let directory_cache_file = data_directory.join(DIRECTORY_CACHE_FILE_NAME); debug!( - "Saving value cache: '{}'.", - value_file.display().to_string() + "Saving directory cache: '{}'.", + directory_cache_file.display().to_string() ); - let out_bytes: Vec = serde_json::to_vec(&self.values) - .map_err(|e| Error::JSONSerialize(value_file.clone(), e))?; + let out_bytes: Vec = serde_json::to_vec(&self.directory_cache) + .map_err(|e| Error::JSONSerialize(directory_cache_file.clone(), e))?; fs::create_dir_all(&data_directory) .map_err(|e| Error::DirectoryCreate(data_directory, e))?; - fs::write(&value_file, out_bytes).map_err(|e| Error::FileWrite(value_file.clone(), e))?; + fs::write(&directory_cache_file, out_bytes) + .map_err(|e| Error::FileWrite(directory_cache_file.clone(), e))?; Ok(()) } @@ -388,48 +419,60 @@ impl State { debug!("Synchronizing workspace '{}'.", workspace_path.display()); - // TODO: get workspace metadata. Store mtime in the cache. Then call `list_directories` - // only when the current mtime is different from the value in the cache. - let filesystem_directories: HashSet = - HashSet::from_iter(workspace::list_directories(workflow, multi_progress)?); - - //////////////////////////////////////////////// - // First, synchronize the values. - // Make a copy of the directories to remove. - let directories_to_remove: Vec = self - .values - .keys() - .filter(|&x| !filesystem_directories.contains(x)) - .cloned() - .collect(); + let mut directories_to_add = Vec::new(); - if directories_to_remove.is_empty() { - trace!("No directories to remove from the value cache."); + // Check if the workspace directory has been modified since we last updated the cache. + let metadata = fs::metadata(workspace_path.clone()) + .map_err(|e| Error::DirectoryRead(workspace_path.clone(), e))?; + let current_modified_time = (metadata.mtime(), metadata.mtime_nsec()); + if current_modified_time == self.directory_cache.modified_time { + trace!("The workspace has not been modified."); } else { - self.values_modified = true; - } + trace!("The workspace has been modified, updating the cache."); + self.directories_modified = true; + self.directory_cache.modified_time = current_modified_time; + + let filesystem_directories: HashSet = + HashSet::from_iter(workspace::list_directories(workflow, multi_progress)?); + + //////////////////////////////////////////////// + // First, synchronize the values. + // Make a copy of the directories to remove. + let directories_to_remove: Vec = self + .directory_cache + .values + .keys() + .filter(|&x| !filesystem_directories.contains(x)) + .cloned() + .collect(); - // Then remove them. - for directory in directories_to_remove { - trace!("Removing '{}' from the value cache", directory.display()); - self.values.remove(&directory); - } + if directories_to_remove.is_empty() { + trace!("No directories to remove from the directory cache."); + } + // Then remove them. + for directory in directories_to_remove { + trace!( + "Removing '{}' from the directory cache", + directory.display() + ); + self.directory_cache.values.remove(&directory); + } - // Make a copy of the directories to be added. - let directories_to_add: Vec = filesystem_directories - .iter() - .filter(|&x| !self.values.contains_key(x)) - .cloned() - .collect(); + // Make a copy of the directories to be added. + directories_to_add = filesystem_directories + .iter() + .filter(|&x| !self.directory_cache.values.contains_key(x)) + .cloned() + .collect(); - if directories_to_add.is_empty() { - trace!("No directories to add to the value cache."); - } else { - trace!( - "Adding {} directories to the workspace.", - directories_to_add.len() - ); - self.values_modified = true; + if directories_to_add.is_empty() { + trace!("No directories to add to the directory cache."); + } else { + trace!( + "Adding {} directories to the workspace.", + directories_to_add.len() + ); + } } // Read value files from the directories. @@ -455,7 +498,7 @@ impl State { /////////////////////////////////////////// // Wait for launched threads to finish and merge results. - self.values.extend(directory_values.get()?); + self.directory_cache.values.extend(directory_values.get()?); let new_complete = new_complete.get()?; if !new_complete.is_empty() { @@ -502,7 +545,7 @@ impl State { for directories in self.completed.values_mut() { let directories_to_remove: Vec = directories .iter() - .filter(|d| !self.values.contains_key(*d)) + .filter(|d| !self.directory_cache.values.contains_key(*d)) .cloned() .collect(); @@ -535,7 +578,7 @@ impl State { for directory_map in self.submitted.values_mut() { let directories_to_remove: Vec = directory_map .keys() - .filter(|d| !self.values.contains_key(*d)) + .filter(|d| !self.directory_cache.values.contains_key(*d)) .cloned() .collect(); @@ -688,7 +731,7 @@ mod tests { let mut state = State::default(); let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); assert!(result.is_ok()); - assert_eq!(state.values.len(), 0); + assert_eq!(state.values().len(), 0); } #[test] @@ -713,15 +756,18 @@ mod tests { let workflow = Workflow::open_str(temp.path(), workflow).unwrap(); let mut state = State::default(); - state.values.insert(PathBuf::from("dir4"), Value::Null); + state + .directory_cache + .values + .insert(PathBuf::from("dir4"), Value::Null); let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); assert!(result.is_ok()); - assert_eq!(state.values.len(), 3); - assert!(state.values.contains_key(&PathBuf::from("dir1"))); - assert!(state.values.contains_key(&PathBuf::from("dir2"))); - assert!(state.values.contains_key(&PathBuf::from("dir3"))); + assert_eq!(state.values().len(), 3); + assert!(state.values().contains_key(&PathBuf::from("dir1"))); + assert!(state.values().contains_key(&PathBuf::from("dir2"))); + assert!(state.values().contains_key(&PathBuf::from("dir3"))); } #[test] @@ -744,9 +790,9 @@ mod tests { let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); assert!(result.is_ok()); - assert_eq!(state.values.len(), 1); - assert!(state.values.contains_key(&PathBuf::from("dir1"))); - assert_eq!(state.values[&PathBuf::from("dir1")].as_i64(), Some(10)); + assert_eq!(state.values().len(), 1); + assert!(state.values().contains_key(&PathBuf::from("dir1"))); + assert_eq!(state.values()[&PathBuf::from("dir1")].as_i64(), Some(10)); } fn setup_completion_directories(temp: &TempDir, n: usize) -> String { @@ -794,13 +840,13 @@ products = ["g"] let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); assert!(result.is_ok()); - assert_eq!(state.values.len(), n); + assert_eq!(state.values().len(), n); assert!(state.completed.contains_key("b")); assert!(state.completed.contains_key("e")); for i in 0..n { let directory = PathBuf::from(format!("dir{i}")); #[allow(clippy::cast_sign_loss)] - let value = state.values[&directory].as_i64().unwrap() as usize; + let value = state.values()[&directory].as_i64().unwrap() as usize; assert_eq!(value, i); if i < n / 2 { @@ -831,6 +877,7 @@ products = ["g"] let mut state = State::default(); for i in 0..n { state + .directory_cache .values .insert(PathBuf::from(format!("dir{i}")), Value::Null); } @@ -840,7 +887,7 @@ products = ["g"] let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); assert!(result.is_ok()); - assert_eq!(state.values.len(), n); + assert_eq!(state.values().len(), n); assert!(!state.completed.contains_key("b")); assert!(!state.completed.contains_key("e")); } @@ -873,7 +920,7 @@ products = ["g"] let result = state.synchronize_workspace(&workflow, 2, &mut multi_progress); assert!(result.is_ok()); - assert_eq!(state.values.len(), n); + assert_eq!(state.values().len(), n); assert!(state.completed.contains_key("b")); assert!(state.completed.contains_key("e")); assert!(!state.completed.contains_key("z")); @@ -884,7 +931,7 @@ products = ["g"] for i in 0..n { let directory = PathBuf::from(format!("dir{i}")); #[allow(clippy::cast_sign_loss)] - let value = state.values[&directory].as_i64().unwrap() as usize; + let value = state.values()[&directory].as_i64().unwrap() as usize; assert_eq!(value, i); if i < n / 2 { @@ -918,6 +965,8 @@ products = ["g"] state.add_submitted("b", &["dir3".into(), "dir4".into()], "cluster2", 12); state.add_submitted("e", &["dir6".into(), "dir7".into()], "cluster2", 13); + assert_eq!(state.num_submitted(), 6); + assert!(state.is_submitted("b", &"dir1".into())); assert!(!state.is_submitted("b", &"dir2".into())); assert!(state.is_submitted("b", &"dir3".into())); @@ -968,6 +1017,8 @@ products = ["g"] state.add_submitted("b", &["dir1".into(), "dir2".into()], "cluster1", 19); state.add_submitted("f", &["dir3".into(), "dir4".into()], "cluster2", 27); + assert_eq!(state.num_submitted(), 6); + assert!(state.is_submitted("b", &"dir1".into())); assert!(state.is_submitted("b", &"dir2".into())); assert!(state.is_submitted("b", &"dir25".into())); @@ -1017,6 +1068,8 @@ products = ["g"] state.add_submitted("b", &["dir3".into(), "dir4".into()], "cluster2", 12); state.add_submitted("e", &["dir6".into(), "dir7".into()], "cluster2", 13); + assert_eq!(state.num_submitted(), 6); + assert!(state.is_submitted("b", &"dir1".into())); assert!(!state.is_submitted("b", &"dir2".into())); assert!(state.is_submitted("b", &"dir3".into())); diff --git a/src/ui.rs b/src/ui.rs index 59cb6eb..02a3dd5 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use console::Style; use indicatif::MultiProgress; use memchr::memmem; @@ -79,13 +82,19 @@ pub(crate) struct Item { alignment: Alignment, } +/// A table row is either a separator or a vector of items. +pub(crate) enum Row { + Separator, + Items(Vec), +} + /// The table pub(crate) struct Table { // The header row. pub header: Vec, - // The items. - pub items: Vec>, + // The table rows. + pub rows: Vec, // Hide the header when true. hide_header: bool, @@ -110,7 +119,7 @@ impl Table { pub(crate) fn new() -> Self { Table { header: Vec::new(), - items: Vec::new(), + rows: Vec::new(), hide_header: false, } } @@ -144,10 +153,12 @@ impl Table { .iter() .map(|h| console::measure_text_width(&h.text)) .collect(); - for row in &self.items { - for (i, item) in row.iter().enumerate() { - column_width[i] = - cmp::max(console::measure_text_width(&item.text), column_width[i]); + for row in &self.rows { + if let Row::Items(items) = row { + for (i, item) in items.iter().enumerate() { + column_width[i] = + cmp::max(console::measure_text_width(&item.text), column_width[i]); + } } } @@ -155,8 +166,17 @@ impl Table { Self::write_row(writer, &self.header, &column_width)?; } - for row in &self.items { - Self::write_row(writer, row, &column_width)?; + for (row_idx, row) in self.rows.iter().enumerate() { + match row { + Row::Items(items) => { + Self::write_row(writer, items, &column_width)?; + } + Row::Separator => { + if row_idx != self.rows.len() - 1 { + writeln!(writer)?; + } + } + } } Ok(()) diff --git a/src/workflow.rs b/src/workflow.rs index d440e88..17ef623 100644 --- a/src/workflow.rs +++ b/src/workflow.rs @@ -1,5 +1,8 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use human_format::Formatter; -use log::{debug, trace}; +use log::{debug, trace, warn}; use serde::{Deserialize, Deserializer}; use serde_json; use speedate::Duration; @@ -33,7 +36,7 @@ pub struct Workflow { /// The submission options #[serde(default)] pub submit_options: HashMap, - + // TODO: refactor handling of submit options into more general action defaults. /// The actions. #[serde(default)] pub action: Vec, @@ -236,9 +239,6 @@ impl ResourceCost { impl fmt::Display for ResourceCost { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut formatter = Formatter::new(); - // TODO: choose decimals more intelligently here. - // Currently: 4,499,000 will print as 4M, but 449,900 will print as 450K. - // It would be nice if we always kept 3 sig figs, giving 4.50M in the first case. formatter.with_decimals(0); formatter.with_separator(""); @@ -439,6 +439,13 @@ impl Workflow { .insert(name.clone(), global_options.clone()); } } + + // Warn for apparently invalid sort_by. + for pointer in &action.group.sort_by { + if !pointer.is_empty() && !pointer.starts_with('/') { + warn!("The JSON pointer '{pointer}' does not appear valid. Did you mean '/{pointer}'?"); + } + } } for action in &self.action { diff --git a/src/workspace.rs b/src/workspace.rs index 39bc722..313a5f9 100644 --- a/src/workspace.rs +++ b/src/workspace.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use indicatif::ProgressBar; use log::debug; use serde_json::Value; diff --git a/tests/cli.rs b/tests/cli.rs index dff47ce..08901b6 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -1,3 +1,6 @@ +// Copyright (c) 2024 The Regents of the University of Michigan. +// Part of row, released under the BSD 3-Clause License. + use assert_cmd::Command; use assert_fs::prelude::*; use assert_fs::TempDir; @@ -5,6 +8,8 @@ use predicates::prelude::*; use serial_test::parallel; use std::fs; +use row::DATA_DIRECTORY_NAME; + /// Create a sample workflow and workspace to use with the tests. fn setup_sample_workflow( temp: &TempDir, @@ -573,3 +578,102 @@ fn show_launchers() -> Result<(), Box> { Ok(()) } + +#[test] +#[parallel] +fn init_conflicting_args() -> Result<(), Box> { + let mut cmd = Command::cargo_bin("row")?; + let temp = TempDir::new()?; + + cmd.args(["init"]) + .arg("--signac") + .args(["--workspace", "test"]) + .current_dir(temp.path()); + cmd.assert() + .failure() + .stderr(predicate::str::contains("cannot be used with")); + + Ok(()) +} + +#[test] +#[parallel] +fn init_invalid_path() -> Result<(), Box> { + let mut cmd = Command::cargo_bin("row")?; + let temp = TempDir::new()?; + + cmd.args(["init"]) + .args(["--workspace", "/test/one"]) + .arg(".") + .current_dir(temp.path()); + cmd.assert() + .failure() + .stderr(predicate::str::contains("must be a relative")); + + Ok(()) +} + +#[test] +#[parallel] +fn init_workflow_exists() -> Result<(), Box> { + let mut cmd = Command::cargo_bin("row")?; + let temp = TempDir::new()?; + temp.child("workflow.toml").touch()?; + + cmd.args(["init"]).arg(".").current_dir(temp.path()); + cmd.assert() + .failure() + .stderr(predicate::str::contains("project already exists")); + + Ok(()) +} + +#[test] +#[parallel] +fn init_parent_exists() -> Result<(), Box> { + let mut cmd = Command::cargo_bin("row")?; + let temp = TempDir::new()?; + temp.child("workflow.toml").touch()?; + + let subdir = temp.child("subdir"); + subdir.create_dir_all()?; + + cmd.args(["init"]).arg(".").current_dir(subdir.path()); + cmd.assert().failure().stderr(predicate::str::contains( + "project already exists in the parent", + )); + + Ok(()) +} + +#[test] +#[parallel] +fn init_cache_exists() -> Result<(), Box> { + let mut cmd = Command::cargo_bin("row")?; + let temp = TempDir::new()?; + temp.child(DATA_DIRECTORY_NAME).touch()?; + + cmd.args(["init"]).arg(".").current_dir(temp.path()); + cmd.assert() + .failure() + .stderr(predicate::str::contains("cache directory")) + .stderr(predicate::str::contains("already exists")); + + Ok(()) +} + +#[test] +#[parallel] +fn init() -> Result<(), Box> { + let mut cmd = Command::cargo_bin("row")?; + let temp = TempDir::new()?; + + cmd.args(["init"]).arg(".").current_dir(temp.path()); + cmd.assert().success(); + + temp.child("workspace").assert(predicate::path::is_dir()); + temp.child("workflow.toml") + .assert(predicate::path::is_file()); + + Ok(()) +} diff --git a/validate/validate.py b/validate/validate.py index 7462d4e..e20a38f 100644 --- a/validate/validate.py +++ b/validate/validate.py @@ -10,12 +10,12 @@ * `cat /output/*.out` The submitted jobs check serial, threaded, MPI, MPI+threads, GPU, and -MPI+GPU jobs to ensure that they run sucessfully and are scheduled to the +MPI+GPU jobs to ensure that they run successfully and are scheduled to the selected resources. Check `*.out` for any error messages. Then check `/output/*.out` for `ERROR`, `WARN`, and `PASSED` lines. `validate.py` prints: `ERROR` when the launched job has a more restrictive binding than requested; `WARN` when the binding is less restrictive; and -'PASSED' when there are at least enough avaialble resources to execute. +'PASSED' when there are at least enough available resources to execute. To test a non-built-in cluster: * Configure your cluster in `cluster.toml`.