From bec8af7de9a0a154484c28bf4ced2af23f3b0d83 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 27 Nov 2024 12:02:16 +0100 Subject: [PATCH] enhancing 'cm pull repo' based on user feedback --- README.md | 21 ++++++++++++--------- cm/CHANGES.md | 3 +++ cm/README.md | 6 ++++-- cm/cmind/__init__.py | 2 +- cm/cmind/repo/automation/repo/module.py | 19 +++++++++++++------ cm/cmind/repos.py | 6 +++++- docs/history.md | 9 ++++++--- 7 files changed, 44 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index beb737a0e..b6c91ae84 100755 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ CK consists of several sub-projects: * CM/CM4Research/CM4MLPerf-results: [Grigori Fursin](https://cKnowledge.org/gfursin) * CM4MLOps: [Arjun Suresh](https://github.com/arjunsuresh) and [Anandhu Sooraj](https://github.com/anandhu-eng) -* CMX (the next generation of CM) [Grigori Fursin](https://cKnowledge.org/gfursin) +* CMX (the next generation of CM, CM4MLOps and CM4MLPerf): [Grigori Fursin](https://cKnowledge.org/gfursin) ### Citing our project @@ -73,19 +73,20 @@ To learn more about the motivation behind CK and CM technology, please explore t * "Enabling more efficient and cost-effective AI/ML systems with Collective Mind, virtualized MLOps, MLPerf, Collective Knowledge Playground and reproducible optimization tournaments": [ [ArXiv](https://arxiv.org/abs/2406.16791) ] * ACM REP'23 keynote about the MLCommons CM automation framework: [ [slides](https://doi.org/10.5281/zenodo.8105339) ] * ACM TechTalk'21 about Collective Knowledge project: [ [YouTube](https://www.youtube.com/watch?v=7zpeIVwICa4) ] [ [slides](https://learning.acm.org/binaries/content/assets/leaning-center/webinar-slides/2021/grigorifursin_techtalk_slides.pdf) ] +* Journal of Royal Society'20: [ [paper](https://royalsocietypublishing.org/doi/10.1098/rsta.2020.0211) ] ### CM Documentation * [CM installation GUI](https://access.cknowledge.org/playground/?action=install) -* [CM Getting Started Guide and FAQ](docs/getting-started.md) - * [Common CM interface to run MLPerf inference benchmarks](docs/mlperf/inference) - * [Common CM interface to re-run experiments from ML and Systems papers including MICRO'23 and the Student Cluster Competition @ SuperComputing'23](docs/tutorials/common-interface-to-reproduce-research-projects.md) +* [CM Getting Started Guide and FAQ](https://github.com/mlcommons/ck/tree/master/docs/getting-started.md) + * [Common CM interface to run MLPerf inference benchmarks](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference) + * [Common CM interface to re-run experiments from ML and Systems papers including MICRO'23 and the Student Cluster Competition @ SuperComputing'23](https://github.com/mlcommons/ck/tree/master/docs/tutorials/common-interface-to-reproduce-research-projects.md) * [CM automation recipes for MLOps and DevOps](https://access.cknowledge.org/playground/?action=scripts) - * [Other CM tutorials](docs/tutorials) -* [Full documentation](docs/README.md) -* [CM development tasks](docs/taskforce.md#current-tasks) -* [CM and CK history](docs/history.md) + * [Other CM tutorials](https://github.com/mlcommons/ck/tree/master/docs/tutorials) +* [Full documentation](https://github.com/mlcommons/ck/tree/master/docs/README.md) +* [CM taskforce](https://github.com/mlcommons/ck/tree/master/docs/taskforce.md) +* [CMX, CM and CK history](https://github.com/mlcommons/ck/tree/master/docs/history.md) ### Acknowledgments @@ -95,6 +96,8 @@ CM4Research and CMX) was created by [Grigori Fursin](https://cKnowledge.org/gfur and sponsored by cTuning.org, OctoAI and HiPEAC. Grigori donated CK to MLCommons to benefit the community and to advance its development as a collaborative, community-driven effort. -We thank MLCommons and FlexAI for supporting this project, + +We thank [MLCommons](https://mlcommons.org), [FlexAI](https://flex.ai) +and [cTuning](https://cTuning.org) for supporting this project, as well as our dedicated [volunteers and collaborators](https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md) for their feedback and contributions! diff --git a/cm/CHANGES.md b/cm/CHANGES.md index 358e4d357..8416126a1 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,3 +1,6 @@ +## V3.4.3.1 + - enhancing 'cm pull repo' based on user feedback + ## V3.4.3 - fixed pyproject.toml - fixed docs generation diff --git a/cm/README.md b/cm/README.md index e9b96edd0..953584a86 100644 --- a/cm/README.md +++ b/cm/README.md @@ -83,6 +83,8 @@ developed by [Grigori Fursin](https://cKnowledge.org/gfursin), as a part of the [Collective Knowledge educational initiative](https://cKnowledge.org), sponsored by [cTuning.org](https://cTuning.org) and [cKnowledge.org](https://cKnowledge.org), and contributed to MLCommons for the benefit of all. + This open-source technology, including CM4MLOps/CM4MLPerf, CM4ABTF, CM4Research, and more, -is a collaborative community-driven project made possible by our -[amazing volunteers, collaborators, and contributors](https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md)! +is a collaborative project supported by [MLCommons](https://mlcommons.org), +[FlexAI](https://flex.ai), [cTuning](https://cTuning.org) +and our [amazing volunteers, collaborators, and contributors](https://github.com/mlcommons/ck/blob/master/CONTRIBUTING.md)! diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py index b8f988f88..3e8c2744c 100644 --- a/cm/cmind/__init__.py +++ b/cm/cmind/__init__.py @@ -2,7 +2,7 @@ # # Written by Grigori Fursin -__version__ = "3.4.3" +__version__ = "3.4.3.1" from cmind.core import access from cmind.core import x diff --git a/cm/cmind/repo/automation/repo/module.py b/cm/cmind/repo/automation/repo/module.py index eae2345a9..2b7578eb1 100644 --- a/cm/cmind/repo/automation/repo/module.py +++ b/cm/cmind/repo/automation/repo/module.py @@ -34,6 +34,7 @@ def pull(self, i): (new_branch) (str): Create new Git branch (checkout) (str): Git checkout (checkout_only) (bool): only checkout existing repo + (dir) (str): use repository in this directory (depth) (int): Git depth (desc) (str): brief repository description (1 line) (prefix) (str): extra directory to keep CM artifacts @@ -54,11 +55,11 @@ def pull(self, i): console = i.get('out') == 'con' - alias = i.get('artifact','') - url = i.get('url','') - desc = i.get('desc','') - prefix = i.get('prefix','') - pat = i.get('pat','') + alias = i.get('artifact', '') + url = i.get('url', '') + desc = i.get('desc', '') + prefix = i.get('prefix', '') + pat = i.get('pat', '') extra_cmd_git = i.get('extra_cmd_git', '') extra_cmd_pip = i.get('extra_cmd_pip', '') @@ -132,13 +133,14 @@ def pull(self, i): branch = i.get('branch', '') new_branch = i.get('new_branch', '') checkout = i.get('checkout', '') + _dir = i.get('dir', '') r = net.request({'get': {'action': 'check-migration-repo-notes', 'repo': url, 'branch': branch, 'checkout': checkout}}) notes = r.get('dict', {}).get('notes','') if notes !='': print (notes) - if alias == 'mlcommons@ck' and branch == '' and checkout == '': + if alias == 'mlcommons@ck' and branch == '' and checkout == '' and _dir == '': print ('=========================================================================') print ('Warning: mlcommons@ck was automatically changed to mlcommons@cm4mlops.') print ('If you want to use older mlcommons@ck repository, use branch or checkout.') @@ -153,6 +155,7 @@ def pull(self, i): 'branch': branch, 'new_branch': new_branch, 'checkout': checkout, + 'dir': _dir, 'depth': i.get('depth', '')}] @@ -173,6 +176,7 @@ def pull(self, i): checkout = repo.get('checkout','') depth = repo.get('depth','') path_to_repo = repo.get('path_to_repo', None) + _dir = repo.get('dir', '') if console: print (self.cmind.cfg['line']) @@ -185,6 +189,8 @@ def pull(self, i): print ('New branch: {}'.format(new_branch)) if checkout!='': print ('Checkout: {}'.format(checkout)) + if _dir!='': + print ('Directory: {}'.format(_dir)) if depth!='' and depth!=None: print ('Depth: {}'.format(str(depth))) print ('') @@ -197,6 +203,7 @@ def pull(self, i): branch = branch, new_branch = new_branch, checkout = checkout, + _dir = _dir, console = console, desc=desc, prefix=prefix, diff --git a/cm/cmind/repos.py b/cm/cmind/repos.py index c86a641ee..e75fcf829 100644 --- a/cm/cmind/repos.py +++ b/cm/cmind/repos.py @@ -305,7 +305,7 @@ def process(self, repo_path, mode='add'): return rr ############################################################ - def pull(self, alias, url = '', branch = '', checkout = '', console = False, desc = '', prefix = '', depth = None, + def pull(self, alias, url = '', branch = '', checkout = '', _dir = '', console = False, desc = '', prefix = '', depth = None, path_to_repo = None, checkout_only = False, skip_zip_parent_dir = False, extra_cmd_git = '', extra_cmd_pip = '', new_branch = ''): """ @@ -318,6 +318,7 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des (new_branch) (str): Create new branch (checkout) (str): Git repository checkout (checkout_only) (bool): only checkout existing repo + (_dir) (str): use repository in this directory (depth) (int): Git repository depth (console) (bool): if True, print some info to console (desc) (str): optional repository description @@ -346,6 +347,9 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des if path_to_repo == None: path_to_repo = os.path.join(self.full_path_to_repos, alias) + if _dir != '': + path_to_repo = os.path.join(path_to_repo, _dir) + if console: print ('Local path: '+path_to_repo) print ('') diff --git a/docs/history.md b/docs/history.md index b92250b12..39b353d61 100644 --- a/docs/history.md +++ b/docs/history.md @@ -24,8 +24,11 @@ and systems (CM4MLOps scripts and workflows)](https://github.com/mlcommons/cm4ml and helped establish the [MLCommons Task Force on Automation and Reproducibility](taskforce.md) co-led with Arjun Suresh. -We continue extending CM to support different MLCommons projects to modularize and unify benchmarking -of ML/AI systems as a collaborative engineering effort based on [user feedback](../CONTRIBUTING.md). - You can learn more about the CM concept and motivation from the [keynote at ACM REP'23](https://doi.org/10.5281/zenodo.8105339) and this [white paper](https://arxiv.org/abs/2406.16791). + +# CMX (the next generation of CM and CM4MLOps) + +Grigori started prototyping CMX during summer 2024 to simplify CM interfaces based on user feedback. +It is available as a part of the standard [CM package at PYPI](https://pypi.org/project/cmind). +Please reach out to learn more about our plans.