Condor Fixes Pt-2 (#49)

Co-authored-by: github-actions <[email protected]>

.github/workflows/tests.yml

@@ -47,6 +47,7 @@ jobs:
         ]
     env:
       CLIENTS_PER_CPU: 1  # there isn't any improvement when >1
+      CI_TESTING_USE_LOCAL_DOCKER: 1
     steps:
       - uses: actions/checkout@v3
       - uses: cvmfs-contrib/github-action-cvmfs@v2

requirements.txt

@@ -14,7 +14,7 @@ certifi==2022.6.15
     # via
     #   pulsar-client
     #   requests
-charset-normalizer==2.0.12
+charset-normalizer==2.1.0
     # via requests
 coloredlogs==15.0.1
     # via wipac-dev-tools
@@ -49,7 +49,7 @@ pillow==9.1.1
     # via matplotlib
 protobuf==4.21.2
     # via apache-bookkeeper-client
-pulsar-client==2.10.0
+pulsar-client==2.10.1
     # via wipac-mqclient-pulsar
 pyerfa==2.0.0.1
     # via astropy
@@ -65,7 +65,7 @@ pytz==2022.1
     # via apache-bookkeeper-client
 pyyaml==6.0
     # via astropy
-requests==2.28.0
+requests==2.28.1
     # via wipac-dev-tools
 scipy==1.8.1
     # via healpy

scripts/condor/spawn_clients.py

@@ -33,7 +33,8 @@ def make_condor_file(  # pylint: disable=R0913,R0914
     gcd_dir: str,
     broker: str,
     auth_token: str,
-    log_level: str,
+    log: str,
+    log_third_party: str,
     timeout_to_clients: int,
     timeout_from_clients: int,
 ) -> str:
@@ -57,7 +58,8 @@ def make_condor_file(  # pylint: disable=R0913,R0914
             f"--gcd-dir {gcd_dir} "
             f"--broker {broker} "
             f"--auth-token {auth_token} "
-            f"--log {log_level} "
+            f"--log {log} "
+            f"--log-third-party {log_third_party} "
             f"--timeout-to-clients {timeout_to_clients} "
             f"--timeout-from-clients {timeout_from_clients}"
         )
@@ -87,12 +89,12 @@ def main() -> None:
     Make scratch directory and condor file.
     """
     if not (
-        os.getcwd().startswith("/home/")
+        os.getcwd().startswith(os.path.expanduser("~"))
         and os.getcwd().endswith("skymap_scanner")
         and "scripts" in os.listdir(".")
     ):
         raise RuntimeError(
-            "You must run this script from /home/ @ repo root (script uses relative paths)"
+            "You must run this script from home directory @ repo root (script uses relative paths)"
         )
 
     parser = argparse.ArgumentParser(
@@ -118,8 +120,19 @@ def main() -> None:
             "By default no accounting group is used."
         ),
     )
-    parser.add_argument("--cpus", type=int, help="number of CPUs", default=4)
-    parser.add_argument("--memory", help="amount of memory", default="8GB")
+    parser.add_argument(
+        "--cpus",
+        required=True,
+        type=int,
+        help="number of CPUs",
+        # default=4,
+    )
+    parser.add_argument(
+        "--memory",
+        required=True,
+        help="amount of memory",
+        # default="8GB",
+    )
 
     # skymap scanner args
     parser.add_argument(
@@ -144,20 +157,26 @@ def main() -> None:
     )
     parser.add_argument(
         "--timeout-to-clients",
-        default=60 * 1,
+        required=True,
         type=int,
         help="timeout (seconds) for messages TO client(s)",
     )
     parser.add_argument(
         "--timeout-from-clients",
-        default=60 * 30,
+        required=True,
         type=int,
         help="timeout (seconds) for messages FROM client(s)",
     )
     parser.add_argument(
-        "--log-level",
+        "-l",
+        "--log",
+        required=True,
+        help="Skymap Scanner: the output logging level (for first-party loggers)",
+    )
+    parser.add_argument(
+        "--log-third-party",
         required=True,
-        help="Skymap Scanner: the output logging level",
+        help="Skymap Scanner: the output logging level for third-party loggers",
     )
 
     args = parser.parse_args()
@@ -179,7 +198,8 @@ def main() -> None:
         args.gcd_dir,
         args.broker,
         args.auth_token,
-        args.log_level,
+        args.log,
+        args.log_third_party,
         args.timeout_to_clients,
         args.timeout_from_clients,
     )

scripts/launch_scripts/launch_client.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 ########################################################################
 #
@@ -48,8 +48,13 @@ PY_ARGS="$(echo $DOCKER_PY_ARGS | awk -F "#" '{print $2}')"
 
 set -x
 
+PULL_POLICY="--pull=always"
+if [ "$CI_TESTING_USE_LOCAL_DOCKER" == "1" ]; then
+    PULL_POLICY=""
+fi
+
 # Run
-docker run --network="host" --rm -i \
+docker run --network="host" $PULL_POLICY --rm -i \
     --shm-size=6gb \
     $DOCKERMOUNT_ARGS \
     --env PY_COLORS=1 \

scripts/launch_scripts/launch_server.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 ########################################################################
 #
@@ -56,8 +56,13 @@ PY_ARGS="$(echo $DOCKER_PY_ARGS | awk -F "#" '{print $2}')"
 
 set -x
 
+PULL_POLICY="--pull=always"
+if [ "$CI_TESTING_USE_LOCAL_DOCKER" == "1" ]; then
+    PULL_POLICY=""
+fi
+
 # Run
-docker run --network="host" --rm -i \
+docker run --network="host" $PULL_POLICY --rm -i \
     $DOCKERMOUNT_ARGS \
     --env PY_COLORS=1 \
     icecube/skymap_scanner:latest \

scripts/launch_scripts/pulsar_run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 set -x
 

tests/docker_scripts/_launch_scan_pixel.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 set -x
 

tests/docker_scripts/docker_local_build.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 set -x
 

tests/docker_scripts/pulsar_config.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # From the README.md