From 082eca6d778cb1feae7aae68fa1f699f61928053 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 22 Jul 2024 17:17:47 -0700 Subject: [PATCH 01/14] FIX: use clone instead of http requests to allow us to check internal and private repos --- scripts/ioc_deploy.py | 88 +++++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index fc7d9c13..b5c7a27e 100755 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -24,9 +24,8 @@ import os import subprocess import sys -import urllib.error -import urllib.request from pathlib import Path +from tempfile import TemporaryDirectory EPICS_SITE_TOP_DEFAULT = "/cds/group/pcds/epics" GITHUB_ORG_DEFAULT = "pcdshub" @@ -189,17 +188,14 @@ def finalize_name(name: str, github_org: str) -> str: logger.warning(f"{name} is not an ioc name, trying {new_name}") name = new_name logger.debug(f"Checking for {name} in org {github_org}") - try: - resp = urllib.request.urlopen(f"https://github.com/{github_org}/{name}") - if resp.code == 200: - logger.info(f"{name} exists in {github_org}") - return name - else: - logger.error(f"Unexpected http error code {resp.code}") - except urllib.error.HTTPError as exc: - logger.error(exc) - logger.debug("", exc_info=True) - raise ValueError(f"{name} does not exist in {github_org}") + with TemporaryDirectory() as tmpdir: + try: + _clone(name=name, github_org=github_org, working_dir=tmpdir) + except subprocess.CalledProcessError as exc: + raise ValueError( + f"Error cloning repo, make sure {name} exists in {github_org} and check your permissions!" + ) from exc + return name def finalize_tag(name: str, github_org: str, release: str) -> str: @@ -223,19 +219,22 @@ def finalize_tag(name: str, github_org: str, release: str) -> str: else: try_release.extend([f"R{release}", f"v{release}"]) - for rel in try_release: - logger.debug(f"Checking for release {rel} in {github_org}/{name}") - try: - resp = urllib.request.urlopen( - f"https://github.com/{github_org}/{name}/releases/tag/{rel}" - ) - if resp.code == 200: + with TemporaryDirectory() as tmpdir: + for rel in try_release: + logger.debug(f"Checking for release {rel} in {github_org}/{name}") + try: + _clone( + name=name, + github_org=github_org, + release=release, + working_dir=tmpdir, + target_dir=release, + ) + except subprocess.CalledProcessError: + logger.warning(f"Did not find release {rel} in {github_org}/{name}") + else: logger.info(f"Release {rel} exists in {github_org}/{name}") return rel - else: - logger.warning(f"Unexpected http error code {resp.code}") - except urllib.error.HTTPError: - logger.warning(f"Did not find release {rel} in {github_org}/{name}") raise ValueError(f"Unable to find {release} in {github_org}/{name}") @@ -263,22 +262,14 @@ def clone_repo_tag( else: logger.debug(f"Ensure {parent_dir} exists") parent_dir.mkdir(parents=True, exist_ok=True) - # Shell out to git clone - cmd = [ - "git", - "clone", - f"https://github.com/{github_org}/{name}", - "--depth", - "1", - "-b", - release, - deploy_dir, - ] + if dry_run: - logger.debug(f"Dry-run: skip shell command \"{' '.join(cmd)}\"") + logger.debug("Dry-run: skip git clone") return ReturnCode.SUCCESS else: - return subprocess.run(cmd).returncode + return _clone( + name=name, github_org=github_org, release=release, target_dir=deploy_dir + ).returncode def make_in(deploy_dir: str, dry_run: bool) -> int: @@ -313,6 +304,29 @@ def get_version() -> str: return "unknown.dev" +def _clone( + name: str, + github_org: str, + release: str = "", + working_dir: str = "", + target_dir: str = "", +) -> subprocess.CompletedProcess: + """ + Clone the repo or raise a subprocess.CalledProcessError + """ + cmd = ["git", "clone", f"git@github.com:{github_org}/{name}", "--depth", "1"] + if release: + cmd.extend(["-b", release]) + if target_dir: + cmd.append(target_dir) + if working_dir: + logger.debug(f"Calling {' '.join(cmd)} in {working_dir}") + return subprocess.run(cmd, check=True, cwd=working_dir) + else: + logger.debug(f"Calling {' '.join(cmd)}") + return subprocess.run(cmd, check=True) + + def _main() -> int: """ Thin wrapper of main() for log setup, args handling, and high-level error handling. From 4111fd47b39a55a913ef70cfcac7a511c7d9d30e Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 22 Jul 2024 17:23:39 -0700 Subject: [PATCH 02/14] FIX: wrong variable in refactored clone --- scripts/ioc_deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index b5c7a27e..3ea88838 100755 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -226,9 +226,9 @@ def finalize_tag(name: str, github_org: str, release: str) -> str: _clone( name=name, github_org=github_org, - release=release, + release=rel, working_dir=tmpdir, - target_dir=release, + target_dir=rel, ) except subprocess.CalledProcessError: logger.warning(f"Did not find release {rel} in {github_org}/{name}") From 00a58d81bee741739ffc443d82fd6c4a90601f6f Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 22 Jul 2024 17:27:54 -0700 Subject: [PATCH 03/14] ENH: only show clone subprocess output in verbose mode --- scripts/ioc_deploy.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index 3ea88838..c294a429 100755 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -132,9 +132,9 @@ def main(args: CliArgs) -> int: ) logger.info("Running ioc-deploy: checking inputs") - upd_name = finalize_name(name=args.name, github_org=args.github_org) + upd_name = finalize_name(name=args.name, github_org=args.github_org, verbose=args.verbose) upd_rel = finalize_tag( - name=upd_name, github_org=args.github_org, release=args.release + name=upd_name, github_org=args.github_org, release=args.release, verbose=args.verbose, ) deploy_dir = get_target_dir(name=upd_name, ioc_dir=args.ioc_dir, release=upd_rel) @@ -152,6 +152,7 @@ def main(args: CliArgs) -> int: release=upd_rel, deploy_dir=deploy_dir, dry_run=args.dry_run, + verbose=args.verbose, ) if rval != ReturnCode.SUCCESS: logger.error(f"Nonzero return value {rval} from git clone") @@ -165,7 +166,7 @@ def main(args: CliArgs) -> int: return ReturnCode.SUCCESS -def finalize_name(name: str, github_org: str) -> str: +def finalize_name(name: str, github_org: str, verbose: bool) -> str: """ Check if name is present in org and is well-formed. @@ -190,7 +191,7 @@ def finalize_name(name: str, github_org: str) -> str: logger.debug(f"Checking for {name} in org {github_org}") with TemporaryDirectory() as tmpdir: try: - _clone(name=name, github_org=github_org, working_dir=tmpdir) + _clone(name=name, github_org=github_org, working_dir=tmpdir, verbose=verbose) except subprocess.CalledProcessError as exc: raise ValueError( f"Error cloning repo, make sure {name} exists in {github_org} and check your permissions!" @@ -198,7 +199,7 @@ def finalize_name(name: str, github_org: str) -> str: return name -def finalize_tag(name: str, github_org: str, release: str) -> str: +def finalize_tag(name: str, github_org: str, release: str, verbose: bool) -> str: """ Check if release is present in the org. @@ -229,6 +230,7 @@ def finalize_tag(name: str, github_org: str, release: str) -> str: release=rel, working_dir=tmpdir, target_dir=rel, + verbose=verbose, ) except subprocess.CalledProcessError: logger.warning(f"Did not find release {rel} in {github_org}/{name}") @@ -250,7 +252,7 @@ def get_target_dir(name: str, ioc_dir: str, release: str) -> str: def clone_repo_tag( - name: str, github_org: str, release: str, deploy_dir: str, dry_run: bool + name: str, github_org: str, release: str, deploy_dir: str, dry_run: bool, verbose: bool, ) -> int: """ Create a shallow clone of the git repository in the correct location. @@ -268,7 +270,7 @@ def clone_repo_tag( return ReturnCode.SUCCESS else: return _clone( - name=name, github_org=github_org, release=release, target_dir=deploy_dir + name=name, github_org=github_org, release=release, target_dir=deploy_dir, verbose=verbose, ).returncode @@ -310,6 +312,7 @@ def _clone( release: str = "", working_dir: str = "", target_dir: str = "", + verbose: bool = False, ) -> subprocess.CompletedProcess: """ Clone the repo or raise a subprocess.CalledProcessError @@ -321,10 +324,10 @@ def _clone( cmd.append(target_dir) if working_dir: logger.debug(f"Calling {' '.join(cmd)} in {working_dir}") - return subprocess.run(cmd, check=True, cwd=working_dir) + return subprocess.run(cmd, check=True, cwd=working_dir, capture_output=not verbose) else: logger.debug(f"Calling {' '.join(cmd)}") - return subprocess.run(cmd, check=True) + return subprocess.run(cmd, check=True, capture_output=not verbose) def _main() -> int: From 5a2e01f362f7ae8c1d72fa9e185afedaff13f7d6 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 22 Jul 2024 17:28:22 -0700 Subject: [PATCH 04/14] STY: ruff format ioc_deploy.py --- scripts/ioc_deploy.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index c294a429..47831f98 100755 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -132,9 +132,14 @@ def main(args: CliArgs) -> int: ) logger.info("Running ioc-deploy: checking inputs") - upd_name = finalize_name(name=args.name, github_org=args.github_org, verbose=args.verbose) + upd_name = finalize_name( + name=args.name, github_org=args.github_org, verbose=args.verbose + ) upd_rel = finalize_tag( - name=upd_name, github_org=args.github_org, release=args.release, verbose=args.verbose, + name=upd_name, + github_org=args.github_org, + release=args.release, + verbose=args.verbose, ) deploy_dir = get_target_dir(name=upd_name, ioc_dir=args.ioc_dir, release=upd_rel) @@ -191,7 +196,9 @@ def finalize_name(name: str, github_org: str, verbose: bool) -> str: logger.debug(f"Checking for {name} in org {github_org}") with TemporaryDirectory() as tmpdir: try: - _clone(name=name, github_org=github_org, working_dir=tmpdir, verbose=verbose) + _clone( + name=name, github_org=github_org, working_dir=tmpdir, verbose=verbose + ) except subprocess.CalledProcessError as exc: raise ValueError( f"Error cloning repo, make sure {name} exists in {github_org} and check your permissions!" @@ -252,7 +259,12 @@ def get_target_dir(name: str, ioc_dir: str, release: str) -> str: def clone_repo_tag( - name: str, github_org: str, release: str, deploy_dir: str, dry_run: bool, verbose: bool, + name: str, + github_org: str, + release: str, + deploy_dir: str, + dry_run: bool, + verbose: bool, ) -> int: """ Create a shallow clone of the git repository in the correct location. @@ -270,7 +282,11 @@ def clone_repo_tag( return ReturnCode.SUCCESS else: return _clone( - name=name, github_org=github_org, release=release, target_dir=deploy_dir, verbose=verbose, + name=name, + github_org=github_org, + release=release, + target_dir=deploy_dir, + verbose=verbose, ).returncode @@ -324,7 +340,9 @@ def _clone( cmd.append(target_dir) if working_dir: logger.debug(f"Calling {' '.join(cmd)} in {working_dir}") - return subprocess.run(cmd, check=True, cwd=working_dir, capture_output=not verbose) + return subprocess.run( + cmd, check=True, cwd=working_dir, capture_output=not verbose + ) else: logger.debug(f"Calling {' '.join(cmd)}") return subprocess.run(cmd, check=True, capture_output=not verbose) From 91a4823d4b97ba9a1d4ccc7f5a68be49871df4a4 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Mon, 22 Jul 2024 18:19:38 -0700 Subject: [PATCH 05/14] ENH: first pass at adapting ssh agent helper script --- scripts/ioc-deploy | 14 +++++++++++- scripts/ioc_deploy.py | 0 scripts/ssh_agent_helper.sh | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) mode change 120000 => 100755 scripts/ioc-deploy mode change 100755 => 100644 scripts/ioc_deploy.py create mode 100644 scripts/ssh_agent_helper.sh diff --git a/scripts/ioc-deploy b/scripts/ioc-deploy deleted file mode 120000 index 34cc4a13..00000000 --- a/scripts/ioc-deploy +++ /dev/null @@ -1 +0,0 @@ -ioc_deploy.py \ No newline at end of file diff --git a/scripts/ioc-deploy b/scripts/ioc-deploy new file mode 100755 index 00000000..9c825094 --- /dev/null +++ b/scripts/ioc-deploy @@ -0,0 +1,13 @@ +#!/usr/bin/bash +# Wrapper script for deploying IOCs from github. +# See ioc_deploy.py or try --help for usage +THIS_SCRIPT="$(realpath "${BASH_SOURCE[0]}")" +THIS_DIR="$(dirname "${THIS_SCRIPT}")" + +# Register the ssh key with the ssh agent if needed +source "${THIS_DIR}/ssh_agent_helper.sh" +# Stop the ssh agent at exit if we started it here +trap ssh_agent_helper_cleanup EXIT + +# Run the deploy script +/usr/bin/python3 "${THIS_DIR}"/ioc_deploy.py "$@" diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py old mode 100755 new mode 100644 diff --git a/scripts/ssh_agent_helper.sh b/scripts/ssh_agent_helper.sh new file mode 100644 index 00000000..a626fe72 --- /dev/null +++ b/scripts/ssh_agent_helper.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Helper script for starting the ssh agent if needed and doing an ssh-add. +# This will let anyone smoothly run github/ssh related scripts without multiple password prompts. +# This script is intended to be sourced. +# Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly. +# +# Expected usage: +# +# source ssh_agent_helper.sh +# trap ssh_agent_helper_cleanup EXIT +HELPER_STARTED_AGENT="NO" +export HELPER_STARTED_AGENT + +# Define an exportable helper for cleaning up the SSH agent +ssh_agent_helper_cleanup() { + if [ "${HELPER_STARTED_AGENT}" = "YES" ] && [ -n "${SSH_AGENT_PID}" ]; then + echo "Cleaning up SSH agent" + ssh-agent -k &> /dev/null + unset HELPER_STARTED_AGENT + export HELPER_STARTED_AGENT + export SSH_AGENT_PID + export SSH_AUTH_SOCK + fi +} +export ssh_agent_helper_cleanup +# Clean up immediately if something in this script fails +trap ssh_agent_helper_cleanup ERR SIGINT SIGTERM + +# SSH agent check: return code is 1 if there are no identities, 2 if cannot connect to agent. +# Only start the agent on return code 2, otherwise we can just add our identify. +# On return code 0 we don't have to do anything, the user already has this set up. +ssh-add -L &> /dev/null +rval=$? +set -e +if [ "$rval" -eq 2 ]; then + echo "Starting ssh agent" + eval "$(ssh-agent -s)" &> /dev/null + HELPER_STARTED_AGENT="YES" +fi +if [ "$rval" -gt 0 ]; then + echo "Running ssh-add, may prompt for ssh key password:" + ssh-add +fi From efa97996f486ede5efce1f0089360433a7338683 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 23 Jul 2024 17:49:58 -0700 Subject: [PATCH 06/14] ENH: maximally streamline, simplify, and fix ssh agent helper handling --- scripts/ioc-deploy | 15 ++++++++----- scripts/ssh-agent-helper | 31 ++++++++++++++++++++++++++ scripts/ssh_agent_helper.sh | 43 ------------------------------------- 3 files changed, 41 insertions(+), 48 deletions(-) create mode 100644 scripts/ssh-agent-helper delete mode 100644 scripts/ssh_agent_helper.sh diff --git a/scripts/ioc-deploy b/scripts/ioc-deploy index 9c825094..60770490 100755 --- a/scripts/ioc-deploy +++ b/scripts/ioc-deploy @@ -1,13 +1,18 @@ #!/usr/bin/bash # Wrapper script for deploying IOCs from github. # See ioc_deploy.py or try --help for usage +if ssh-add -L; then + echo "" + echo "WARNING: Your ssh key is not in the ssh agent." + echo " If you run ioc-deploy like this, you may need to input" + echo " your ssh key password multiple times!" + echo " Try 'source ssh-agent-helper' to input your ssh key" + echo " password only once per login." + echo "" +fi + THIS_SCRIPT="$(realpath "${BASH_SOURCE[0]}")" THIS_DIR="$(dirname "${THIS_SCRIPT}")" -# Register the ssh key with the ssh agent if needed -source "${THIS_DIR}/ssh_agent_helper.sh" -# Stop the ssh agent at exit if we started it here -trap ssh_agent_helper_cleanup EXIT - # Run the deploy script /usr/bin/python3 "${THIS_DIR}"/ioc_deploy.py "$@" diff --git a/scripts/ssh-agent-helper b/scripts/ssh-agent-helper new file mode 100644 index 00000000..7c03d341 --- /dev/null +++ b/scripts/ssh-agent-helper @@ -0,0 +1,31 @@ +#!/usr/bin/bash +# Helper script for starting the ssh agent if needed and doing an ssh-add. +# This will let anyone smoothly run github/ssh related scripts without multiple password prompts. +# This script is intended to be sourced. +# Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly. +# +# Expected usage: +# +# source ssh-agent-helper + +# SSH agent check: return code is 1 if there are no identities, 2 if cannot connect to agent. +# Only start the agent on return code 2, otherwise we can just add our identity. +# On return code 0 we don't have to do anything, the user already has this set up. +ssh-add -L &> /dev/null +rval=$? +if [ "$rval" -eq 2 ]; then + echo "Starting ssh agent" + eval "$(ssh-agent -s)" &> /dev/null + ssh_agent_helper_cleanup() { + if [ -n "${SSH_AGENT_PID}" ]; then + echo "Cleaning up SSH agent" + eval "$(ssh-agent -k)" &> /dev/null + fi + } + trap ssh_agent_helper_cleanup EXIT +fi +if [ "$rval" -gt 0 ]; then + echo "Running ssh-add, may prompt for ssh key password" + # Expire after 12h just in case to avoid infinite key storage + ssh-add -t 12h +fi diff --git a/scripts/ssh_agent_helper.sh b/scripts/ssh_agent_helper.sh deleted file mode 100644 index a626fe72..00000000 --- a/scripts/ssh_agent_helper.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Helper script for starting the ssh agent if needed and doing an ssh-add. -# This will let anyone smoothly run github/ssh related scripts without multiple password prompts. -# This script is intended to be sourced. -# Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly. -# -# Expected usage: -# -# source ssh_agent_helper.sh -# trap ssh_agent_helper_cleanup EXIT -HELPER_STARTED_AGENT="NO" -export HELPER_STARTED_AGENT - -# Define an exportable helper for cleaning up the SSH agent -ssh_agent_helper_cleanup() { - if [ "${HELPER_STARTED_AGENT}" = "YES" ] && [ -n "${SSH_AGENT_PID}" ]; then - echo "Cleaning up SSH agent" - ssh-agent -k &> /dev/null - unset HELPER_STARTED_AGENT - export HELPER_STARTED_AGENT - export SSH_AGENT_PID - export SSH_AUTH_SOCK - fi -} -export ssh_agent_helper_cleanup -# Clean up immediately if something in this script fails -trap ssh_agent_helper_cleanup ERR SIGINT SIGTERM - -# SSH agent check: return code is 1 if there are no identities, 2 if cannot connect to agent. -# Only start the agent on return code 2, otherwise we can just add our identify. -# On return code 0 we don't have to do anything, the user already has this set up. -ssh-add -L &> /dev/null -rval=$? -set -e -if [ "$rval" -eq 2 ]; then - echo "Starting ssh agent" - eval "$(ssh-agent -s)" &> /dev/null - HELPER_STARTED_AGENT="YES" -fi -if [ "$rval" -gt 0 ]; then - echo "Running ssh-add, may prompt for ssh key password:" - ssh-add -fi From 857f3b7016a002ca398e14d22c091a3bdb43d1c8 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 23 Jul 2024 17:56:08 -0700 Subject: [PATCH 07/14] ENH: improve stderr handling --- scripts/ioc-deploy | 16 ++++++++-------- scripts/ssh-agent-helper | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/ioc-deploy b/scripts/ioc-deploy index 60770490..5e56528c 100755 --- a/scripts/ioc-deploy +++ b/scripts/ioc-deploy @@ -1,14 +1,14 @@ #!/usr/bin/bash # Wrapper script for deploying IOCs from github. # See ioc_deploy.py or try --help for usage -if ssh-add -L; then - echo "" - echo "WARNING: Your ssh key is not in the ssh agent." - echo " If you run ioc-deploy like this, you may need to input" - echo " your ssh key password multiple times!" - echo " Try 'source ssh-agent-helper' to input your ssh key" - echo " password only once per login." - echo "" +if ! ssh-add -L &> /dev/null; then + >&2 echo "" + >&2 echo " WARNING: Your ssh key is not in the ssh agent." + >&2 echo " If you run ioc-deploy like this, you may need to input" + >&2 echo " your ssh key password multiple times!" + >&2 echo " Try 'source ssh-agent-helper' to input your ssh key" + >&2 echo " password only once per login." + >&2 echo "" fi THIS_SCRIPT="$(realpath "${BASH_SOURCE[0]}")" diff --git a/scripts/ssh-agent-helper b/scripts/ssh-agent-helper index 7c03d341..2f024662 100644 --- a/scripts/ssh-agent-helper +++ b/scripts/ssh-agent-helper @@ -27,5 +27,5 @@ fi if [ "$rval" -gt 0 ]; then echo "Running ssh-add, may prompt for ssh key password" # Expire after 12h just in case to avoid infinite key storage - ssh-add -t 12h + ssh-add -t 12h 2> /dev/null fi From 33d518f6d3acc6a55ee66d49813dbd6417e15edc Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 23 Jul 2024 17:58:59 -0700 Subject: [PATCH 08/14] ENH: gratuitious font bolding in warning --- scripts/ioc-deploy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ioc-deploy b/scripts/ioc-deploy index 5e56528c..6365d5e8 100755 --- a/scripts/ioc-deploy +++ b/scripts/ioc-deploy @@ -6,7 +6,7 @@ if ! ssh-add -L &> /dev/null; then >&2 echo " WARNING: Your ssh key is not in the ssh agent." >&2 echo " If you run ioc-deploy like this, you may need to input" >&2 echo " your ssh key password multiple times!" - >&2 echo " Try 'source ssh-agent-helper' to input your ssh key" + >&2 echo " Try '$(tput bold)source ssh-agent-helper$(tput sgr0)' to input your ssh key" >&2 echo " password only once per login." >&2 echo "" fi From b21c3157cb81560b5c140ac7856879df0e18ec30 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 23 Jul 2024 18:06:27 -0700 Subject: [PATCH 09/14] ENH: slight refactor for double call+ older py3 version compat --- scripts/ioc_deploy.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index 47831f98..6c093485 100644 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -338,14 +338,14 @@ def _clone( cmd.extend(["-b", release]) if target_dir: cmd.append(target_dir) + kwds = {"check": True} if working_dir: - logger.debug(f"Calling {' '.join(cmd)} in {working_dir}") - return subprocess.run( - cmd, check=True, cwd=working_dir, capture_output=not verbose - ) - else: - logger.debug(f"Calling {' '.join(cmd)}") - return subprocess.run(cmd, check=True, capture_output=not verbose) + kwds["cwd"] = working_dir + if not verbose: + kwds["stdin"] = subprocess.PIPE + kwds["stderr"] = subprocess.PIPE + logger.debug(f"Calling {' '.join(cmd)} with kwargs {kwds}") + return subprocess.run(cmd, **kwds) def _main() -> int: From 94e67c165ce290712ca706f1de2032e9eb6d52b1 Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Tue, 23 Jul 2024 18:07:50 -0700 Subject: [PATCH 10/14] ENH: slightly better debug output --- scripts/ioc_deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index 6c093485..a0ca6679 100644 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -344,7 +344,7 @@ def _clone( if not verbose: kwds["stdin"] = subprocess.PIPE kwds["stderr"] = subprocess.PIPE - logger.debug(f"Calling {' '.join(cmd)} with kwargs {kwds}") + logger.debug(f"Calling '{' '.join(cmd)}' with kwargs {kwds}") return subprocess.run(cmd, **kwds) From 71db53a73b37d2580e4c9807f6117022712a99da Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 24 Jul 2024 11:37:46 -0700 Subject: [PATCH 11/14] FIX: redirect stdout and stderr, not stdin and stderr (typo) --- scripts/ioc_deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ioc_deploy.py b/scripts/ioc_deploy.py index a0ca6679..ccc656b8 100644 --- a/scripts/ioc_deploy.py +++ b/scripts/ioc_deploy.py @@ -342,7 +342,7 @@ def _clone( if working_dir: kwds["cwd"] = working_dir if not verbose: - kwds["stdin"] = subprocess.PIPE + kwds["stdout"] = subprocess.PIPE kwds["stderr"] = subprocess.PIPE logger.debug(f"Calling '{' '.join(cmd)}' with kwargs {kwds}") return subprocess.run(cmd, **kwds) From 2b1a34ed1c95d009be648fa1531dced1da49d2ec Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 24 Jul 2024 11:46:00 -0700 Subject: [PATCH 12/14] DOC: expand docstrings and in-line explanations for ssh-agent-helper --- scripts/ssh-agent-helper | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/ssh-agent-helper b/scripts/ssh-agent-helper index 2f024662..b4af416c 100644 --- a/scripts/ssh-agent-helper +++ b/scripts/ssh-agent-helper @@ -1,27 +1,39 @@ #!/usr/bin/bash +# # Helper script for starting the ssh agent if needed and doing an ssh-add. # This will let anyone smoothly run github/ssh related scripts without multiple password prompts. +# # This script is intended to be sourced. # Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly. # # Expected usage: # # source ssh-agent-helper +# +ssh-add -L &> /dev/null +rval=$? # SSH agent check: return code is 1 if there are no identities, 2 if cannot connect to agent. # Only start the agent on return code 2, otherwise we can just add our identity. # On return code 0 we don't have to do anything, the user already has this set up. -ssh-add -L &> /dev/null -rval=$? +# If the user is already forwarding their SSH key via ssh agent forwarding, this +# helpfully returns 0 and nothing needs to be done. if [ "$rval" -eq 2 ]; then echo "Starting ssh agent" + # This ssh-agent -s command starts the agent and outputs some environment variable + # set and export commands to stdout that the user must execute in their shell via eval. + # This sets SSH_AGENT_PID to be used to kill the agent later, + # and SSH_AUTH_SOCK to be used to authorise ssh commands via the agent. eval "$(ssh-agent -s)" &> /dev/null + # If the agent hasn't already been killed, this helper will kill it. ssh_agent_helper_cleanup() { if [ -n "${SSH_AGENT_PID}" ]; then echo "Cleaning up SSH agent" + # Kills the agent and un-sets the environment variables eval "$(ssh-agent -k)" &> /dev/null fi } + # This will ensure that ssh_agent_helper_cleanup runs when the user's shell exits. trap ssh_agent_helper_cleanup EXIT fi if [ "$rval" -gt 0 ]; then From 1163bd8f4e3c38b48244a1e15ff54a099c7a2e8e Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 24 Jul 2024 11:55:04 -0700 Subject: [PATCH 13/14] DOC: document the logout condition --- scripts/ssh-agent-helper | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ssh-agent-helper b/scripts/ssh-agent-helper index b4af416c..23188e3f 100644 --- a/scripts/ssh-agent-helper +++ b/scripts/ssh-agent-helper @@ -2,6 +2,7 @@ # # Helper script for starting the ssh agent if needed and doing an ssh-add. # This will let anyone smoothly run github/ssh related scripts without multiple password prompts. +# An ssh-agent instance set up using this script will be automaticalls closed on logout. # # This script is intended to be sourced. # Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly. From be76659408c506544c8c42495e983f05fb9353ae Mon Sep 17 00:00:00 2001 From: Zachary Lentz Date: Wed, 24 Jul 2024 11:59:49 -0700 Subject: [PATCH 14/14] DOC: nitpick docstring comment and add to README.md --- README.md | 13 +++++++++++++ scripts/ssh-agent-helper | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ee76531f..6b7e7c0a 100644 --- a/README.md +++ b/README.md @@ -685,6 +685,19 @@ usage: serverStat servername options
+ + ssh-agent-helper + +usage: source ssh-agent-helper

+Helper script for starting the ssh agent if needed and doing an ssh-add -t 12h. +This will let anyone smoothly run github/ssh related scripts without multiple password prompts. +An ssh-agent process started by using this script will be automatically closed on logout. +

+This script is intended to be sourced. +Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly. + + + startami diff --git a/scripts/ssh-agent-helper b/scripts/ssh-agent-helper index 23188e3f..b78b9632 100644 --- a/scripts/ssh-agent-helper +++ b/scripts/ssh-agent-helper @@ -1,8 +1,8 @@ #!/usr/bin/bash # -# Helper script for starting the ssh agent if needed and doing an ssh-add. +# Helper script for starting the ssh agent if needed and doing an ssh-add -t 12h. # This will let anyone smoothly run github/ssh related scripts without multiple password prompts. -# An ssh-agent instance set up using this script will be automaticalls closed on logout. +# An ssh-agent process started by using this script will be automatically closed on logout. # # This script is intended to be sourced. # Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly.