From a0b29a2a5a2eefdb5f5d6841014e9602f1698464 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 16 Jul 2024 16:22:37 -0700 Subject: [PATCH 1/7] ENH: add ioc-deploy script --- scripts/ioc-deploy | 1 + scripts/ioc_deploy.py | 327 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 120000 scripts/ioc-deploy create mode 100755 scripts/ioc_deploy.py diff --git a/scripts/ioc-deploy b/scripts/ioc-deploy new file mode 120000 index 00000000..34cc4a13 --- /dev/null +++ b/scripts/ioc-deploy @@ -0,0 +1 @@ +ioc_deploy.py \ No newline at end of file diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py new file mode 100755 index 00000000..55680dac --- /dev/null +++ b/scripts/ioc_deploy.py @@ -0,0 +1,327 @@ +#!/usr/bin/python3 +""" +ioc-deploy is a script for building and deploying ioc tags from github. +It will create a shallow clone of your IOC in the standard release area at the correct path and "make" it. +""" + +import argparse +import enum +import logging +import os +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +EPICS_SITE_TOP_DEFAULT = "/cds/group/pcds/epics" +GITHUB_ORG_DEFAULT = "pcdshub" + +logger = logging.getLogger("ioc-deploy") + + +if sys.version_info >= (3, 7, 0): + import dataclasses + + @dataclasses.dataclass(frozen=True) + class CliArgs: + """ + Argparse argument types for type checking. + """ + + name: str = "" + release: str = "" + ioc_dir: str = "" + github_org: str = "" + auto_confirm: bool = False + dry_run: bool = False + verbose: bool = False + version: bool = False +else: + from types import SimpleNamespace as CliArgs + + +def get_parser() -> argparse.ArgumentParser: + current_default_target = str( + Path(os.environ.get("EPICS_SITE_TOP", EPICS_SITE_TOP_DEFAULT)) / "ioc" + ) + current_default_org = os.environ.get("GITHUB_ORG", GITHUB_ORG_DEFAULT) + parser = argparse.ArgumentParser(prog="ioc-deploy", description=__doc__) + parser.add_argument( + "--version", action="store_true", help="Show version number and exit." + ) + parser.add_argument( + "--name", + "-n", + action="store", + default="", + help="The name of the repository to deploy. This is a required argument. If it does not exist on github, we'll also try prepending with 'ioc-common-'.", + ) + parser.add_argument( + "--release", + "-r", + action="store", + default="", + help="The version of the IOC to deploy. This is a required argument.", + ) + parser.add_argument( + "--ioc-dir", + "-i", + action="store", + default=current_default_target, + help=f"The directory to deploy IOCs in. This defaults to $EPICS_SITE_TOP/ioc, or {EPICS_SITE_TOP_DEFAULT}/ioc if the environment variable is not set. With your current environment variables, this defaults to {current_default_target}.", + ) + parser.add_argument( + "--github_org", + "--org", + action="store", + default=current_default_org, + help=f"The github org to deploy IOCs from. This defaults to $GITHUB_ORG, or {GITHUB_ORG_DEFAULT} if the environment variable is not set. With your current environment variables, this defaults to {current_default_org}.", + ) + parser.add_argument( + "--auto-confirm", + "--confirm", + "--yes", + "-y", + action="store_true", + help="Skip the confirmation promps, automatically saying yes to each one.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Do not deploy anything, just print what would have been done.", + ) + parser.add_argument( + "--verbose", + "-v", + "--debug", + action="store_true", + help="Display additional debug information.", + ) + return parser + + +class ReturnCode(enum.IntEnum): + SUCCESS = 0 + EXCEPTION = 1 + NO_CONFIRM = 2 + + +def main(args: CliArgs) -> int: + """ + All main steps of the script. + + Will either return an int return code or raise. + """ + if not (args.name and args.release): + raise ValueError( + "Must provide both --name and --release. Check ioc-deploy --help for usage." + ) + + logger.info("Running ioc-deploy: checking inputs") + upd_name = finalize_name(name=args.name, github_org=args.github_org) + upd_rel = finalize_tag( + name=upd_name, github_org=args.github_org, release=args.release + ) + deploy_dir = get_target_dir(name=upd_name, ioc_dir=args.ioc_dir, release=upd_rel) + + logger.info(f"Deploying {args.github_org}/{upd_name} at {upd_rel} to {deploy_dir}") + if not args.auto_confirm: + user_text = input("Confirm release source and target? y/n\n") + if not user_text.strip().lower().startswith("y"): + return ReturnCode.NO_CONFIRM + if Path(deploy_dir).exists(): + logger.info(f"{deploy_dir} exists, skip git clone step.") + else: + rval = clone_repo_tag( + name=upd_name, + github_org=args.github_org, + release=upd_rel, + deploy_dir=deploy_dir, + dry_run=args.dry_run, + ) + if rval != ReturnCode.SUCCESS: + logger.error(f"Nonzero return value {rval} from git clone") + return rval + + logger.info(f"Building IOC at {deploy_dir}") + rval = make_in(deploy_dir=deploy_dir, dry_run=args.dry_run) + if rval != ReturnCode.SUCCESS: + logger.error(f"Nonzero return value {rval} from make") + return rval + return ReturnCode.SUCCESS + + +def finalize_name(name: str, github_org: str) -> str: + """ + Check if name is present in org and is well-formed. + + If the name is present, return it. + If the name is not present and the correct name can be guessed, guess. + If the name is not present and cannot be guessed, raise. + + A name is well-formed if it starts with "ioc", is hyphen-delimited, + and has at least three sections. + + For example, "ioc-common-gigECam" is a well-formed name for the purposes + of an IOC deployment. "ads-ioc" and "pcdsdevices" are not. + + However, "ads-ioc" will resolve to "ioc-common-ads-ioc". + Only common IOCs will be automatically discovered using this method. + """ + split_name = name.split("-") + if len(split_name) < 3 or split_name[0] != "ioc": + new_name = f"ioc-common-{name}" + logger.warning(f"{name} is not an ioc name, trying {new_name}") + name = new_name + logger.debug(f"Checking for {name} in org {github_org}") + try: + resp = urllib.request.urlopen(f"https://github.com/{github_org}/{name}") + if resp.code == 200: + logger.info(f"{name} exists in {github_org}") + return name + else: + logger.error(f"Unexpected http error code {resp.code}") + except urllib.error.HTTPError as exc: + logger.error(exc) + logger.debug("", exc_info=True) + raise ValueError(f"{name} does not exist in {github_org}") + + +def finalize_tag(name: str, github_org: str, release: str) -> str: + """ + Check if release is present in the org. + + We'll try a few prefix options in case the user has a typo. + In order of priority with no repeats: + - user's input + - R1.0.0 + - v1.0.0 + - 1.0.0 + """ + try_release = [release] + if release.startswith("R"): + try_release.extend([f"v{release[1:]}", f"{release[1:]}"]) + elif release.startswith("v"): + try_release.extend([f"R{release[1:]}", f"{release[1:]}"]) + elif release[0].isalpha(): + try_release.extend([f"R{release[1:]}", f"v{release[1:]}", f"{release[1:]}"]) + else: + try_release.extend([f"R{release}", f"v{release}"]) + + for rel in try_release: + logger.debug(f"Checking for release {rel} in {github_org}/{name}") + try: + resp = urllib.request.urlopen( + f"https://github.com/{github_org}/{name}/releases/tag/{rel}" + ) + if resp.code == 200: + logger.info(f"Release {rel} exists in {github_org}/{name}") + return rel + else: + logger.warning(f"Unexpected http error code {resp.code}") + except urllib.error.HTTPError: + logger.warning(f"Did not find release {rel} in {github_org}/{name}") + raise ValueError(f"Unable to find {release} in {github_org}/{name}") + + +def get_target_dir(name: str, ioc_dir: str, release: str) -> str: + """ + Return the directory we'll deploy the IOC in. + + This will look something like: + /cds/group/pcds/epics/ioc/common/gigECam/R1.0.0 + """ + split_name = name.split("-") + return str(Path(ioc_dir) / split_name[1] / "-".join(split_name[2:]) / release) + + +def clone_repo_tag( + name: str, github_org: str, release: str, deploy_dir: str, dry_run: bool +) -> int: + """ + Create a shallow clone of the git repository in the correct location. + """ + # Make sure the parent dir exists + parent_dir = Path(deploy_dir).resolve().parent + if dry_run: + logger.info(f"Dry-run: make {parent_dir} if not existing.") + else: + logger.debug(f"Ensure {parent_dir} exists") + parent_dir.mkdir(parents=True, exist_ok=True) + # Shell out to git clone + cmd = [ + "git", + "clone", + f"https://github.com/{github_org}/{name}", + "--depth", + "1", + "-b", + release, + deploy_dir, + ] + if dry_run: + logger.debug(f"Dry-run: skip shell command \"{' '.join(cmd)}\"") + return ReturnCode.SUCCESS + else: + return subprocess.run(cmd).returncode + + +def make_in(deploy_dir: str, dry_run: bool) -> int: + """ + Shell out to make in the deploy dir + """ + if dry_run: + logger.info(f"Dry-run: skipping make in {deploy_dir}") + return ReturnCode.SUCCESS + else: + return subprocess.run(["make"], cwd=deploy_dir).returncode + + +def get_version() -> str: + """ + Determine what version of engineering_tools is being used + """ + # Possibility 1: git clone (yours) + try: + return subprocess.check_output( + ["git", "-C", str(Path(__file__).resolve().parent), "describe", "--tags"], + universal_newlines=True, + ).strip() + except subprocess.CalledProcessError: + ... + # Possibility 2: release dir (someone elses) + ver = str(Path(__file__).resolve().parent.parent.stem) + if ver.startswith("R"): + return ver + else: + # We tried our best + return "unknown.dev" + + +def _main() -> int: + """ + Thin wrapper of main() for log setup, args handling, and high-level error handling. + """ + args = CliArgs(**vars(get_parser().parse_args())) + if args.verbose: + level = logging.DEBUG + fmt = "%(levelname)-8s %(name)s:%(lineno)d: %(message)s" + else: + level = logging.INFO + fmt = "%(levelname)-8s %(name)s: %(message)s" + logging.basicConfig(level=level, format=fmt) + logger.debug(f"args are {args}") + try: + if args.version: + print(get_version()) + return ReturnCode.SUCCESS + return main(args) + except Exception as exc: + logger.error(exc) + logger.debug("Traceback", exc_info=True) + return ReturnCode.EXCEPTION + + +if __name__ == "__main__": + exit(_main()) From fa7bbdb205245df2d726d3f4351dfa55a31ac3bd Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 16 Jul 2024 16:36:18 -0700 Subject: [PATCH 2/7] DOC: add ioc-deploy to the readme --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index d20a100d..7415899f 100644 --- a/README.md +++ b/README.md @@ -326,6 +326,47 @@ usage: grep_more_ioc [-h] [-d] patt hutch {print,search}
+ + ioc-deploy + +usage: ioc-deploy [-h] [--version] [--name NAME] [--release RELEASE] + [--ioc-dir IOC_DIR] [--github_org GITHUB_ORG] + [--auto-confirm] [--dry-run] [--verbose] + +ioc-deploy is a script for building and deploying ioc tags from github. It +will create a shallow clone of your IOC in the standard release area at the +correct path and "make" it. + +optional arguments: + -h, --help show this help message and exit + --version Show version number and exit. + --name NAME, -n NAME The name of the repository to deploy. This is a + required argument. If it does not exist on github, + we'll also try prepending with 'ioc-common-'. + --release RELEASE, -r RELEASE + The version of the IOC to deploy. This is a required + argument. + --ioc-dir IOC_DIR, -i IOC_DIR + The directory to deploy IOCs in. This defaults to + $EPICS_SITE_TOP/ioc, or /cds/group/pcds/epics/ioc if + the environment variable is not set. With your current + environment variables, this defaults to + /reg/g/pcds/epics/ioc. + --github_org GITHUB_ORG, --org GITHUB_ORG + The github org to deploy IOCs from. This defaults to + $GITHUB_ORG, or pcdshub if the environment variable is + not set. With your current environment variables, this + defaults to pcdshub. + --auto-confirm, --confirm, --yes, -y + Skip the confirmation promps, automatically saying yes + to each one. + --dry-run Do not deploy anything, just print what would have + been done. + --verbose, -v, --debug + Display additional debug information. + + + iocmanager From 219cf676496d6dc045ee0046c9b5507999dee6fe Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 16 Jul 2024 16:39:02 -0700 Subject: [PATCH 3/7] DOC: try manual html line breaks --- README.md | 62 +++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 7415899f..a61605fa 100644 --- a/README.md +++ b/README.md @@ -329,40 +329,40 @@ usage: grep_more_ioc [-h] [-d] patt hutch {print,search}
ioc-deploy -usage: ioc-deploy [-h] [--version] [--name NAME] [--release RELEASE] - [--ioc-dir IOC_DIR] [--github_org GITHUB_ORG] - [--auto-confirm] [--dry-run] [--verbose] - +usage: ioc-deploy [-h] [--version] [--name NAME] [--release RELEASE]
+ [--ioc-dir IOC_DIR] [--github_org GITHUB_ORG]
+ [--auto-confirm] [--dry-run] [--verbose]
+
ioc-deploy is a script for building and deploying ioc tags from github. It will create a shallow clone of your IOC in the standard release area at the correct path and "make" it. - -optional arguments: - -h, --help show this help message and exit - --version Show version number and exit. - --name NAME, -n NAME The name of the repository to deploy. This is a - required argument. If it does not exist on github, - we'll also try prepending with 'ioc-common-'. - --release RELEASE, -r RELEASE - The version of the IOC to deploy. This is a required - argument. - --ioc-dir IOC_DIR, -i IOC_DIR - The directory to deploy IOCs in. This defaults to - $EPICS_SITE_TOP/ioc, or /cds/group/pcds/epics/ioc if - the environment variable is not set. With your current - environment variables, this defaults to - /reg/g/pcds/epics/ioc. - --github_org GITHUB_ORG, --org GITHUB_ORG - The github org to deploy IOCs from. This defaults to - $GITHUB_ORG, or pcdshub if the environment variable is - not set. With your current environment variables, this - defaults to pcdshub. - --auto-confirm, --confirm, --yes, -y - Skip the confirmation promps, automatically saying yes - to each one. - --dry-run Do not deploy anything, just print what would have - been done. - --verbose, -v, --debug +
+optional arguments:
+ -h, --help show this help message and exit
+ --version Show version number and exit.
+ --name NAME, -n NAME The name of the repository to deploy. This is a
+ required argument. If it does not exist on github,
+ we'll also try prepending with 'ioc-common-'.
+ --release RELEASE, -r RELEASE
+ The version of the IOC to deploy. This is a required
+ argument.
+ --ioc-dir IOC_DIR, -i IOC_DIR
+ The directory to deploy IOCs in. This defaults to
+ $EPICS_SITE_TOP/ioc, or /cds/group/pcds/epics/ioc if
+ the environment variable is not set. With your current
+ environment variables, this defaults to
+ /reg/g/pcds/epics/ioc.
+ --github_org GITHUB_ORG, --org GITHUB_ORG
+ The github org to deploy IOCs from. This defaults to
+ $GITHUB_ORG, or pcdshub if the environment variable is
+ not set. With your current environment variables, this
+ defaults to pcdshub.
+ --auto-confirm, --confirm, --yes, -y
+ Skip the confirmation promps, automatically saying yes
+ to each one.
+ --dry-run Do not deploy anything, just print what would have
+ been done.
+ --verbose, -v, --debug
Display additional debug information. From 8f08f9539bb2aaa4706261a49e05e6faa05b7599 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 16 Jul 2024 16:47:36 -0700 Subject: [PATCH 4/7] DOC: secret sauce for preformatted spacing in readme table --- README.md | 68 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index a61605fa..6dfc4c64 100644 --- a/README.md +++ b/README.md @@ -328,43 +328,43 @@ usage: grep_more_ioc [-h] [-d] patt hutch {print,search}
ioc-deploy - -usage: ioc-deploy [-h] [--version] [--name NAME] [--release RELEASE]
- [--ioc-dir IOC_DIR] [--github_org GITHUB_ORG]
- [--auto-confirm] [--dry-run] [--verbose]
-
+ 
+usage: ioc-deploy [-h] [--version] [--name NAME] [--release RELEASE]
+                  [--ioc-dir IOC_DIR] [--github_org GITHUB_ORG]
+                  [--auto-confirm] [--dry-run] [--verbose]
+ 
 ioc-deploy is a script for building and deploying ioc tags from github. It
 will create a shallow clone of your IOC in the standard release area at the
 correct path and "make" it.
-

-optional arguments:

-  -h, --help            show this help message and exit

-  --version             Show version number and exit.

-  --name NAME, -n NAME  The name of the repository to deploy. This is a

-                        required argument. If it does not exist on github,

-                        we'll also try prepending with 'ioc-common-'.

-  --release RELEASE, -r RELEASE

-                        The version of the IOC to deploy. This is a required

-                        argument.

-  --ioc-dir IOC_DIR, -i IOC_DIR

-                        The directory to deploy IOCs in. This defaults to

-                        $EPICS_SITE_TOP/ioc, or /cds/group/pcds/epics/ioc if

-                        the environment variable is not set. With your current

-                        environment variables, this defaults to

-                        /reg/g/pcds/epics/ioc.

-  --github_org GITHUB_ORG, --org GITHUB_ORG

-                        The github org to deploy IOCs from. This defaults to

-                        $GITHUB_ORG, or pcdshub if the environment variable is

-                        not set. With your current environment variables, this

-                        defaults to pcdshub.

-  --auto-confirm, --confirm, --yes, -y

-                        Skip the confirmation promps, automatically saying yes

-                        to each one.

-  --dry-run             Do not deploy anything, just print what would have

-                        been done.

-  --verbose, -v, --debug

-                        Display additional debug information.
-    
+ 
+optional arguments:
+  -h, --help            show this help message and exit
+  --version             Show version number and exit.
+  --name NAME, -n NAME  The name of the repository to deploy. This is a
+                        required argument. If it does not exist on github,
+                        we'll also try prepending with 'ioc-common-'.
+  --release RELEASE, -r RELEASE
+                        The version of the IOC to deploy. This is a required
+                        argument.
+  --ioc-dir IOC_DIR, -i IOC_DIR
+                        The directory to deploy IOCs in. This defaults to
+                        $EPICS_SITE_TOP/ioc, or /cds/group/pcds/epics/ioc if
+                        the environment variable is not set. With your current
+                        environment variables, this defaults to
+                        /reg/g/pcds/epics/ioc.
+  --github_org GITHUB_ORG, --org GITHUB_ORG
+                        The github org to deploy IOCs from. This defaults to
+                        $GITHUB_ORG, or pcdshub if the environment variable is
+                        not set. With your current environment variables, this
+                        defaults to pcdshub.
+  --auto-confirm, --confirm, --yes, -y
+                        Skip the confirmation promps, automatically saying yes
+                        to each one.
+  --dry-run             Do not deploy anything, just print what would have
+                        been done.
+  --verbose, -v, --debug
+                        Display additional debug information.
+
From 37f8f896c5516a13e2145e95628f3464d2a7ab37 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 17 Jul 2024 16:24:04 -0700 Subject: [PATCH 5/7] ENH: exit if tag dir already exists --- scripts/ioc_deploy.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index 55680dac..78b4a34c 100755 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -126,23 +126,23 @@ def main(args: CliArgs) -> int: deploy_dir = get_target_dir(name=upd_name, ioc_dir=args.ioc_dir, release=upd_rel) logger.info(f"Deploying {args.github_org}/{upd_name} at {upd_rel} to {deploy_dir}") + if Path(deploy_dir).exists(): + raise RuntimeError(f"Deploy directory {deploy_dir} already exists! Aborting.") if not args.auto_confirm: user_text = input("Confirm release source and target? y/n\n") if not user_text.strip().lower().startswith("y"): return ReturnCode.NO_CONFIRM - if Path(deploy_dir).exists(): - logger.info(f"{deploy_dir} exists, skip git clone step.") - else: - rval = clone_repo_tag( - name=upd_name, - github_org=args.github_org, - release=upd_rel, - deploy_dir=deploy_dir, - dry_run=args.dry_run, - ) - if rval != ReturnCode.SUCCESS: - logger.error(f"Nonzero return value {rval} from git clone") - return rval + logger.info(f"Cloning IOC to {deploy_dir}") + rval = clone_repo_tag( + name=upd_name, + github_org=args.github_org, + release=upd_rel, + deploy_dir=deploy_dir, + dry_run=args.dry_run, + ) + if rval != ReturnCode.SUCCESS: + logger.error(f"Nonzero return value {rval} from git clone") + return rval logger.info(f"Building IOC at {deploy_dir}") rval = make_in(deploy_dir=deploy_dir, dry_run=args.dry_run) From 4bcdc8c70bc38e152f6520f847446a649cc6009c Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 17 Jul 2024 16:33:55 -0700 Subject: [PATCH 6/7] DOC: update script docstring and help text to include an example. --- scripts/ioc_deploy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index 78b4a34c..fc7d9c13 100755 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -2,6 +2,20 @@ """ ioc-deploy is a script for building and deploying ioc tags from github. It will create a shallow clone of your IOC in the standard release area at the correct path and "make" it. +If the tag directory already exists, the script will exit. + +Example command: +"ioc-deploy -n ioc-foo-bar -r R1.0.0" + +This will clone the repository to the default ioc directory and run make +using the currently set EPICS environment variables. + +With default settings this will clone + +from https://github.com/pcdshub/ioc-foo-bar +to /cds/group/pcds/epics/ioc/foo/bar/R1.0.0 + +then cd and make. """ import argparse From aff8f9f35a1908d3ed73094da2f48288a44c35b0 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 17 Jul 2024 16:35:26 -0700 Subject: [PATCH 7/7] DOC: re-paste help text into repo readme. --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6dfc4c64..6a8c5146 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,12 @@ usage: ioc-deploy [-h] [--version] [--name NAME] [--release RELEASE]   ioc-deploy is a script for building and deploying ioc tags from github. It will create a shallow clone of your IOC in the standard release area at the -correct path and "make" it. +correct path and "make" it. If the tag directory already exists, the script +will exit. Example command: "ioc-deploy -n ioc-foo-bar -r R1.0.0" This will +clone the repository to the default ioc directory and run make using the +currently set EPICS environment variables. With default settings this will +clone from https://github.com/pcdshub/ioc-foo-bar to +/cds/group/pcds/epics/ioc/foo/bar/R1.0.0 then cd and make.   optional arguments:   -h, --help show this help message and exit