Add new test: test_process.py

pythoncommons/test-scripts/download.sh

@@ -0,0 +1,16 @@
+function download-random-jars {
+	if [ $# -ne 1 ]; then
+    echo "Usage: $0 <target directory>"
+    return 1
+  fi
+	local target_dir=$1
+
+  rate_limit=""
+#  rate_limit="--limit-rate=50k"
+
+	wget $rate_limit -P $target_dir -nH --reject="index.html*"  --no-parent https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient/4.5.14/httpclient-4.5.14.jar
+	wget $rate_limit -P $target_dir -nH --reject="index.html*"  --no-parent https://repo.maven.apache.org/maven2/org/apache/httpcomponents/fluent-hc/4.5.14/fluent-hc-4.5.14.jar
+	wget $rate_limit -P $target_dir -nH --reject="index.html*"  --no-parent https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpmime/4.5.14/httpmime-4.5.14.jar
+	wget $rate_limit -P $target_dir -nH --reject="index.html*"  --no-parent https://repo.maven.apache.org/maven2/org/apache/httpcomponents/httpclient-cache/4.5.14/httpclient-cache-4.5.14.jar
+  echo "Downloaded to: $target_dir"
+}

pythoncommons/test-scripts/script_that_fails.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+echo "I am dumb and fail instantly"
+exit 1

pythoncommons/test-scripts/script_with_args.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+echo "arg: $1"
+echo "arg: $2"
+echo "arg: $3"

pythoncommons/test-scripts/sleeping_loop.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+. utils.sh
+
+for i in $(seq 1 10); do
+  echo "stdout: $i";
+  echoerr "stderr: $i"
+  sleep 0.3;
+done

pythoncommons/test-scripts/utils.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echoerr() { echo "$@" 1>&2; }

pythoncommons/tests/test_process.py

@@ -0,0 +1,169 @@
+import logging
+import os
+import sys
+import tempfile
+import unittest
+from datetime import datetime
+from typing import List
+
+from pythoncommons.file_utils import FileUtils
+from pythoncommons.logging_setup import SimpleLoggingSetup
+from pythoncommons.process import CommandRunner, SubprocessCommandRunner
+from pythoncommons.tests.test_project_utils import TEST_SCRIPTS_DIR
+
+REPO_ROOT_DIRNAME = "python-commons"
+PYTHONCOMMONS_MODULE_NAME = "pythoncommons"
+SCRIPT_WITH_ARGS_SH = "script_with_args.sh"
+SCRIPT_THAT_FAILS_SH = "script_that_fails.sh"
+UTILS_SH = "utils.sh"
+SLEEPING_LOOP_SH = "sleeping_loop.sh"
+
+
+class ProcessTests(unittest.TestCase):
+    """
+    IMPORTANT !
+    Specify this as additional CLI arguments if running from PyCharm:
+    -s  --capture=no --log-cli-level=10
+
+    Source: https://stackoverflow.com/a/71913594/1106893
+    """
+
+    def _get_test_name(self):
+        return os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
+
+    def get_test_dir(self, parent):
+        date_str = datetime.today().strftime('%Y%m%d_%H%M%S')
+        dir = os.path.join(parent, f"{self._get_test_name()}-{date_str}")
+        os.mkdir(dir)
+        return dir
+
+    @staticmethod
+    def find_script(name: str):
+        basedir = FileUtils.find_repo_root_dir(__file__, PYTHONCOMMONS_MODULE_NAME, raise_error=True)
+        return FileUtils.join_path(basedir, "test-scripts", name)
+
+    @staticmethod
+    def find_input_file(name: str):
+        basedir = FileUtils.find_repo_root_dir(__file__, PYTHONCOMMONS_MODULE_NAME, raise_error=True)
+        return FileUtils.join_path(basedir, "test-input-files", name)
+
+    def test_subprocessrunner_run_and_follow_stdout_stderr_does_not_hang(self):
+        script = self.find_script(SLEEPING_LOOP_SH)
+        basedir = FileUtils.get_parent_dir_name(script)
+        cmd = f"cd {basedir};{script}"
+        cmd = f"bash -c \"{cmd}\""
+        CMD_LOG = SimpleLoggingSetup.create_command_logger(__name__)
+        CMD_LOG.setLevel(logging.DEBUG)
+        SubprocessCommandRunner.run_and_follow_stdout_stderr(
+                        cmd, stdout_logger=CMD_LOG, exit_on_nonzero_exitcode=True
+                    )
+        # TODO verify if all printed to stdout / stderr
+
+    def test_subprocessrunner_run_and_follow_stdout_stderr_defaults(self):
+        script = self.find_script(SLEEPING_LOOP_SH)
+        basedir = FileUtils.get_parent_dir_name(script)
+        cmd = f"cd {basedir};{script}"
+        cmd = f"bash -c \"{cmd}\""
+        SubprocessCommandRunner.run_and_follow_stdout_stderr(cmd)
+        # TODO verify if all printed to stdout / stderr
+
+    def test_subprocessrunner_run_no_output(self):
+        script = self.find_script(SLEEPING_LOOP_SH)
+        parent_dir = FileUtils.get_parent_dir_name(script)
+        cmd = f"{script}"
+        cmd = f"bash -c \"{cmd}\""
+
+        command_result = SubprocessCommandRunner.run(cmd,
+                                                     working_dir=parent_dir,
+                                                     log_command_result=False,
+                                                     fail_on_error=True,
+                                                     fail_message="Failed to run script")
+        # TODO verify empty stdout / stderr
+
+    def test_commandrunner_execute_script_with_args(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            test_dir = self.get_test_dir(tmpdirname)
+            output_file = FileUtils.join_path(test_dir, "testfile")
+            script_path = ProcessTests.find_script(SCRIPT_WITH_ARGS_SH)
+            script_parent_dir = FileUtils.get_parent_dir_name(script_path)
+            script_name = os.path.basename(script_path)
+
+            args = f"arg1 arg2 arg3"
+
+            cli_cmd, cli_output = CommandRunner.execute_script(
+                script_name, args=args, working_dir=script_parent_dir, output_file=output_file, use_tee=True
+            )
+            file_contents = FileUtils.read_file(output_file)
+            self.assertEqual('arg: arg1\narg: arg2\narg: arg3\n', file_contents)
+            self.assertEqual('arg: arg1\narg: arg2\narg: arg3', cli_output)
+
+    def test_commandrunner_run_cli_command_with_args(self):
+        script_path = ProcessTests.find_script(SCRIPT_WITH_ARGS_SH)
+
+        args = f"arg1 arg2 arg3"
+        cmd = f"{script_path} {args}"
+        cmd, cli_output = CommandRunner.run_cli_command(
+            cmd, fail_on_empty_output=False, print_command=False, fail_on_error=False
+        )
+
+        self.assertEqual('arg: arg1\narg: arg2\narg: arg3', cli_output)
+
+
+    def test_commnadrunner_egrep_with_cli(self):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            test_dir = self.get_test_dir(tmpdirname)
+            output_file = FileUtils.join_path(test_dir, "testfile")
+
+            git_log_file = self.find_input_file("hadoop-git-repo-log.txt")
+            git_log: str = FileUtils.read_file(git_log_file)
+            git_log: List[str] = git_log.splitlines()
+
+            cli_command, output = CommandRunner.egrep_with_cli(
+                git_log,
+                file=output_file,
+                grep_for="backport",
+                escape_single_quotes=False,
+                escape_double_quotes=True,
+                fail_on_empty_output=False,
+                fail_on_error=True,
+            )
+            commit_messages = set()
+            for l in output.splitlines():
+                tmp = l.split("(")[0]
+                tmp = tmp.replace("*", "")
+                tmp = tmp.replace("|", "")
+                tmp = tmp.strip()
+                commit_messages.add(tmp)
+            self.assertIn("a90c7221436c - HADOOP-18724. [FOLLOW-UP] cherrypick changes from branch-3.3 backport", commit_messages)
+            self.assertIn("9676774e233f - HDFS-9273. Moving to 2.6.3 CHANGES section to reflect the backport.",
+                          commit_messages)
+            self.assertIn("c753617a48bf - Move HADOOP-11361, HADOOP-12348 and HADOOP-12482 from 2.8.0 to 2.7.3 in CHANGES.txt for backporting.",
+                          commit_messages)
+            self.assertIn("f3e5bc67661e - CHANGES.txt: Moving YARN-1884, YARN-3171, YARN-3740, YARN-3248, YARN-3544 to 2.6.1 given the backport.",
+                          commit_messages)
+            self.assertIn("fbbb7ff1ed11 - Updating all CHANGES.txt files to move entires from future releases into 2.6.1 section given the large number of backports to 2.6.1.",
+                          commit_messages)
+            self.assertIn("8770c82acc94 - MAPREDUCE-6286. Amend commit to CHANGES.txt for backport into 2.7.0.",
+                          commit_messages)
+            self.assertIn("7981908929a0 - backported HADOOP-10125 to branch2, update CHANGES.txt",
+                          commit_messages)
+            self.assertIn("b8f1cf31926d - HDFS-4817. Moving changelog to Release 2.2.0 section to reflect the backport.",
+                          commit_messages)
+            self.assertIn("2245fcc8c5c4 - Move HDFS-347 and related JIRAs to 2.0.5 section of CHANGES.txt after backport",
+                          commit_messages)
+            self.assertIn("0b9a1f908a57 - Moving MAPREDUCE-4678's changes line to 0.23 section to prepare for backport.",
+                          commit_messages)
+            self.assertIn("ebcc708d78ef - Move HADOOP-9004 to 2.0.3 section after backport",
+                          commit_messages)
+            self.assertIn("d6c50b4a67f6 - Move QJM-related backports into 2.0.3 release section in CHANGES.txt after backport to branch-2",
+                          commit_messages)
+            self.assertIn("c9ed8342f527 - Move HDFS-2330 and HDFS-3190 to branch-2 section, since they have been backported from trunk.",
+                          commit_messages)
+            self.assertIn("32431d25aed9 - HADOOP-7469 backporting to 0.23; moving in CHANGES.TXT",
+                          commit_messages)
+            print(commit_messages)
+
+    def test_subprocessrunner_run_script_fails(self):
+        script_path = ProcessTests.find_script(SCRIPT_THAT_FAILS_SH)
+        with self.assertRaises(ValueError):
+            SubprocessCommandRunner.run(script_path, fail_on_error=True, fail_message="Failed to run script")