Merge pull request #190 from ZLLentz/fix_ioc_deploy_auth

FIX/ENH: fix ioc-deploy auth, provide ssh-agent-helper

README.md

@@ -685,6 +685,19 @@ usage: serverStat servername options<br/>
     </td>
 </tr>
 
+<tr>
+    <td>ssh-agent-helper</td>
+    <td>
+usage: source ssh-agent-helper<br/><br/>
+Helper script for starting the ssh agent if needed and doing an ssh-add -t 12h.
+This will let anyone smoothly run github/ssh related scripts without multiple password prompts.
+An ssh-agent process started by using this script will be automatically closed on logout.
+<br/><br/>
+This script is intended to be sourced.
+Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly.
+    </td>
+</tr>
+
 <tr>
     <td>startami</td>
     <td>

scripts/ioc-deploy

@@ -0,0 +1,18 @@
+#!/usr/bin/bash
+# Wrapper script for deploying IOCs from github.
+# See ioc_deploy.py or try --help for usage
+if ! ssh-add -L &> /dev/null; then
+    >&2 echo ""
+    >&2 echo "  WARNING: Your ssh key is not in the ssh agent."
+    >&2 echo "           If you run ioc-deploy like this, you may need to input"
+    >&2 echo "           your ssh key password multiple times!"
+    >&2 echo "           Try '$(tput bold)source ssh-agent-helper$(tput sgr0)' to input your ssh key"
+    >&2 echo "           password only once per login."
+    >&2 echo ""
+fi
+
+THIS_SCRIPT="$(realpath "${BASH_SOURCE[0]}")"
+THIS_DIR="$(dirname "${THIS_SCRIPT}")"
+
+# Run the deploy script
+/usr/bin/python3 "${THIS_DIR}"/ioc_deploy.py "$@"

scripts/ioc_deploy.py

@@ -24,9 +24,8 @@
 import os
 import subprocess
 import sys
-import urllib.error
-import urllib.request
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 EPICS_SITE_TOP_DEFAULT = "/cds/group/pcds/epics"
 GITHUB_ORG_DEFAULT = "pcdshub"
@@ -133,9 +132,14 @@ def main(args: CliArgs) -> int:
         )
 
     logger.info("Running ioc-deploy: checking inputs")
-    upd_name = finalize_name(name=args.name, github_org=args.github_org)
+    upd_name = finalize_name(
+        name=args.name, github_org=args.github_org, verbose=args.verbose
+    )
     upd_rel = finalize_tag(
-        name=upd_name, github_org=args.github_org, release=args.release
+        name=upd_name,
+        github_org=args.github_org,
+        release=args.release,
+        verbose=args.verbose,
     )
     deploy_dir = get_target_dir(name=upd_name, ioc_dir=args.ioc_dir, release=upd_rel)
 
@@ -153,6 +157,7 @@ def main(args: CliArgs) -> int:
         release=upd_rel,
         deploy_dir=deploy_dir,
         dry_run=args.dry_run,
+        verbose=args.verbose,
     )
     if rval != ReturnCode.SUCCESS:
         logger.error(f"Nonzero return value {rval} from git clone")
@@ -166,7 +171,7 @@ def main(args: CliArgs) -> int:
     return ReturnCode.SUCCESS
 
 
-def finalize_name(name: str, github_org: str) -> str:
+def finalize_name(name: str, github_org: str, verbose: bool) -> str:
     """
     Check if name is present in org and is well-formed.
 
@@ -189,20 +194,19 @@ def finalize_name(name: str, github_org: str) -> str:
         logger.warning(f"{name} is not an ioc name, trying {new_name}")
         name = new_name
     logger.debug(f"Checking for {name} in org {github_org}")
-    try:
-        resp = urllib.request.urlopen(f"https://github.com/{github_org}/{name}")
-        if resp.code == 200:
-            logger.info(f"{name} exists in {github_org}")
-            return name
-        else:
-            logger.error(f"Unexpected http error code {resp.code}")
-    except urllib.error.HTTPError as exc:
-        logger.error(exc)
-        logger.debug("", exc_info=True)
-    raise ValueError(f"{name} does not exist in {github_org}")
+    with TemporaryDirectory() as tmpdir:
+        try:
+            _clone(
+                name=name, github_org=github_org, working_dir=tmpdir, verbose=verbose
+            )
+        except subprocess.CalledProcessError as exc:
+            raise ValueError(
+                f"Error cloning repo, make sure {name} exists in {github_org} and check your permissions!"
+            ) from exc
+    return name
 
 
-def finalize_tag(name: str, github_org: str, release: str) -> str:
+def finalize_tag(name: str, github_org: str, release: str, verbose: bool) -> str:
     """
     Check if release is present in the org.
 
@@ -223,19 +227,23 @@ def finalize_tag(name: str, github_org: str, release: str) -> str:
     else:
         try_release.extend([f"R{release}", f"v{release}"])
 
-    for rel in try_release:
-        logger.debug(f"Checking for release {rel} in {github_org}/{name}")
-        try:
-            resp = urllib.request.urlopen(
-                f"https://github.com/{github_org}/{name}/releases/tag/{rel}"
-            )
-            if resp.code == 200:
+    with TemporaryDirectory() as tmpdir:
+        for rel in try_release:
+            logger.debug(f"Checking for release {rel} in {github_org}/{name}")
+            try:
+                _clone(
+                    name=name,
+                    github_org=github_org,
+                    release=rel,
+                    working_dir=tmpdir,
+                    target_dir=rel,
+                    verbose=verbose,
+                )
+            except subprocess.CalledProcessError:
+                logger.warning(f"Did not find release {rel} in {github_org}/{name}")
+            else:
                 logger.info(f"Release {rel} exists in {github_org}/{name}")
                 return rel
-            else:
-                logger.warning(f"Unexpected http error code {resp.code}")
-        except urllib.error.HTTPError:
-            logger.warning(f"Did not find release {rel} in {github_org}/{name}")
     raise ValueError(f"Unable to find {release} in {github_org}/{name}")
 
 
@@ -251,7 +259,12 @@ def get_target_dir(name: str, ioc_dir: str, release: str) -> str:
 
 
 def clone_repo_tag(
-    name: str, github_org: str, release: str, deploy_dir: str, dry_run: bool
+    name: str,
+    github_org: str,
+    release: str,
+    deploy_dir: str,
+    dry_run: bool,
+    verbose: bool,
 ) -> int:
     """
     Create a shallow clone of the git repository in the correct location.
@@ -263,22 +276,18 @@ def clone_repo_tag(
     else:
         logger.debug(f"Ensure {parent_dir} exists")
         parent_dir.mkdir(parents=True, exist_ok=True)
-    # Shell out to git clone
-    cmd = [
-        "git",
-        "clone",
-        f"https://github.com/{github_org}/{name}",
-        "--depth",
-        "1",
-        "-b",
-        release,
-        deploy_dir,
-    ]
+
     if dry_run:
-        logger.debug(f"Dry-run: skip shell command \"{' '.join(cmd)}\"")
+        logger.debug("Dry-run: skip git clone")
         return ReturnCode.SUCCESS
     else:
-        return subprocess.run(cmd).returncode
+        return _clone(
+            name=name,
+            github_org=github_org,
+            release=release,
+            target_dir=deploy_dir,
+            verbose=verbose,
+        ).returncode
 
 
 def make_in(deploy_dir: str, dry_run: bool) -> int:
@@ -313,6 +322,32 @@ def get_version() -> str:
         return "unknown.dev"
 
 
+def _clone(
+    name: str,
+    github_org: str,
+    release: str = "",
+    working_dir: str = "",
+    target_dir: str = "",
+    verbose: bool = False,
+) -> subprocess.CompletedProcess:
+    """
+    Clone the repo or raise a subprocess.CalledProcessError
+    """
+    cmd = ["git", "clone", f"[email protected]:{github_org}/{name}", "--depth", "1"]
+    if release:
+        cmd.extend(["-b", release])
+    if target_dir:
+        cmd.append(target_dir)
+    kwds = {"check": True}
+    if working_dir:
+        kwds["cwd"] = working_dir
+    if not verbose:
+        kwds["stdout"] = subprocess.PIPE
+        kwds["stderr"] = subprocess.PIPE
+    logger.debug(f"Calling '{' '.join(cmd)}' with kwargs {kwds}")
+    return subprocess.run(cmd, **kwds)
+
+
 def _main() -> int:
     """
     Thin wrapper of main() for log setup, args handling, and high-level error handling.

scripts/ssh-agent-helper

@@ -0,0 +1,44 @@
+#!/usr/bin/bash
+#
+# Helper script for starting the ssh agent if needed and doing an ssh-add -t 12h.
+# This will let anyone smoothly run github/ssh related scripts without multiple password prompts.
+# An ssh-agent process started by using this script will be automatically closed on logout.
+#
+# This script is intended to be sourced.
+# Sourcing this script lets ssh-agent set the proper environment variables it needs to run properly.
+#
+# Expected usage:
+#
+# source ssh-agent-helper
+#
+
+ssh-add -L &> /dev/null
+rval=$?
+# SSH agent check: return code is 1 if there are no identities, 2 if cannot connect to agent.
+# Only start the agent on return code 2, otherwise we can just add our identity.
+# On return code 0 we don't have to do anything, the user already has this set up.
+# If the user is already forwarding their SSH key via ssh agent forwarding, this
+# helpfully returns 0 and nothing needs to be done.
+if [ "$rval" -eq 2 ]; then
+    echo "Starting ssh agent"
+    # This ssh-agent -s command starts the agent and outputs some environment variable
+    # set and export commands to stdout that the user must execute in their shell via eval.
+    # This sets SSH_AGENT_PID to be used to kill the agent later,
+    # and SSH_AUTH_SOCK to be used to authorise ssh commands via the agent.
+    eval "$(ssh-agent -s)" &> /dev/null
+    # If the agent hasn't already been killed, this helper will kill it.
+    ssh_agent_helper_cleanup() {
+        if [ -n "${SSH_AGENT_PID}" ]; then
+            echo "Cleaning up SSH agent"
+            # Kills the agent and un-sets the environment variables
+            eval "$(ssh-agent -k)" &> /dev/null
+        fi
+    }
+    # This will ensure that ssh_agent_helper_cleanup runs when the user's shell exits.
+    trap ssh_agent_helper_cleanup EXIT
+fi
+if [ "$rval" -gt 0 ]; then
+    echo "Running ssh-add, may prompt for ssh key password"
+    # Expire after 12h just in case to avoid infinite key storage
+    ssh-add -t 12h 2> /dev/null
+fi