diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f61ec497d..d0deeaebe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,6 +47,7 @@ jobs: ] env: CLIENTS_PER_CPU: 1 # there isn't any improvement when >1 + CI_TESTING_USE_LOCAL_DOCKER: 1 steps: - uses: actions/checkout@v3 - uses: cvmfs-contrib/github-action-cvmfs@v2 diff --git a/requirements.txt b/requirements.txt index a1ff4ecb8..46541faf5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ certifi==2022.6.15 # via # pulsar-client # requests -charset-normalizer==2.0.12 +charset-normalizer==2.1.0 # via requests coloredlogs==15.0.1 # via wipac-dev-tools @@ -49,7 +49,7 @@ pillow==9.1.1 # via matplotlib protobuf==4.21.2 # via apache-bookkeeper-client -pulsar-client==2.10.0 +pulsar-client==2.10.1 # via wipac-mqclient-pulsar pyerfa==2.0.0.1 # via astropy @@ -65,7 +65,7 @@ pytz==2022.1 # via apache-bookkeeper-client pyyaml==6.0 # via astropy -requests==2.28.0 +requests==2.28.1 # via wipac-dev-tools scipy==1.8.1 # via healpy diff --git a/scripts/condor/spawn_clients.py b/scripts/condor/spawn_clients.py index b0a031f39..7de7bb266 100644 --- a/scripts/condor/spawn_clients.py +++ b/scripts/condor/spawn_clients.py @@ -33,7 +33,8 @@ def make_condor_file( # pylint: disable=R0913,R0914 gcd_dir: str, broker: str, auth_token: str, - log_level: str, + log: str, + log_third_party: str, timeout_to_clients: int, timeout_from_clients: int, ) -> str: @@ -57,7 +58,8 @@ def make_condor_file( # pylint: disable=R0913,R0914 f"--gcd-dir {gcd_dir} " f"--broker {broker} " f"--auth-token {auth_token} " - f"--log {log_level} " + f"--log {log} " + f"--log-third-party {log_third_party} " f"--timeout-to-clients {timeout_to_clients} " f"--timeout-from-clients {timeout_from_clients}" ) @@ -87,12 +89,12 @@ def main() -> None: Make scratch directory and condor file. """ if not ( - os.getcwd().startswith("/home/") + os.getcwd().startswith(os.path.expanduser("~")) and os.getcwd().endswith("skymap_scanner") and "scripts" in os.listdir(".") ): raise RuntimeError( - "You must run this script from /home/ @ repo root (script uses relative paths)" + "You must run this script from home directory @ repo root (script uses relative paths)" ) parser = argparse.ArgumentParser( @@ -118,8 +120,19 @@ def main() -> None: "By default no accounting group is used." ), ) - parser.add_argument("--cpus", type=int, help="number of CPUs", default=4) - parser.add_argument("--memory", help="amount of memory", default="8GB") + parser.add_argument( + "--cpus", + required=True, + type=int, + help="number of CPUs", + # default=4, + ) + parser.add_argument( + "--memory", + required=True, + help="amount of memory", + # default="8GB", + ) # skymap scanner args parser.add_argument( @@ -144,20 +157,26 @@ def main() -> None: ) parser.add_argument( "--timeout-to-clients", - default=60 * 1, + required=True, type=int, help="timeout (seconds) for messages TO client(s)", ) parser.add_argument( "--timeout-from-clients", - default=60 * 30, + required=True, type=int, help="timeout (seconds) for messages FROM client(s)", ) parser.add_argument( - "--log-level", + "-l", + "--log", + required=True, + help="Skymap Scanner: the output logging level (for first-party loggers)", + ) + parser.add_argument( + "--log-third-party", required=True, - help="Skymap Scanner: the output logging level", + help="Skymap Scanner: the output logging level for third-party loggers", ) args = parser.parse_args() @@ -179,7 +198,8 @@ def main() -> None: args.gcd_dir, args.broker, args.auth_token, - args.log_level, + args.log, + args.log_third_party, args.timeout_to_clients, args.timeout_from_clients, ) diff --git a/scripts/launch_scripts/launch_client.sh b/scripts/launch_scripts/launch_client.sh index 2855039da..18d4f4b58 100755 --- a/scripts/launch_scripts/launch_client.sh +++ b/scripts/launch_scripts/launch_client.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ######################################################################## # @@ -48,8 +48,13 @@ PY_ARGS="$(echo $DOCKER_PY_ARGS | awk -F "#" '{print $2}')" set -x +PULL_POLICY="--pull=always" +if [ "$CI_TESTING_USE_LOCAL_DOCKER" == "1" ]; then + PULL_POLICY="" +fi + # Run -docker run --network="host" --rm -i \ +docker run --network="host" $PULL_POLICY --rm -i \ --shm-size=6gb \ $DOCKERMOUNT_ARGS \ --env PY_COLORS=1 \ diff --git a/scripts/launch_scripts/launch_server.sh b/scripts/launch_scripts/launch_server.sh index be4dbd90d..4869d9b22 100755 --- a/scripts/launch_scripts/launch_server.sh +++ b/scripts/launch_scripts/launch_server.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash ######################################################################## # @@ -56,8 +56,13 @@ PY_ARGS="$(echo $DOCKER_PY_ARGS | awk -F "#" '{print $2}')" set -x +PULL_POLICY="--pull=always" +if [ "$CI_TESTING_USE_LOCAL_DOCKER" == "1" ]; then + PULL_POLICY="" +fi + # Run -docker run --network="host" --rm -i \ +docker run --network="host" $PULL_POLICY --rm -i \ $DOCKERMOUNT_ARGS \ --env PY_COLORS=1 \ icecube/skymap_scanner:latest \ diff --git a/scripts/launch_scripts/pulsar_run.sh b/scripts/launch_scripts/pulsar_run.sh index ab7687da1..51ee9c169 100755 --- a/scripts/launch_scripts/pulsar_run.sh +++ b/scripts/launch_scripts/pulsar_run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -x diff --git a/tests/docker_scripts/_launch_scan_pixel.sh b/tests/docker_scripts/_launch_scan_pixel.sh index 4d117e6c7..ef0599b73 100755 --- a/tests/docker_scripts/_launch_scan_pixel.sh +++ b/tests/docker_scripts/_launch_scan_pixel.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -x diff --git a/tests/docker_scripts/docker_local_build.sh b/tests/docker_scripts/docker_local_build.sh index 8539b1bd8..46806d4e3 100755 --- a/tests/docker_scripts/docker_local_build.sh +++ b/tests/docker_scripts/docker_local_build.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -x diff --git a/tests/docker_scripts/pulsar_config.sh b/tests/docker_scripts/pulsar_config.sh index 2e453b613..acd89fc50 100755 --- a/tests/docker_scripts/pulsar_config.sh +++ b/tests/docker_scripts/pulsar_config.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # From the README.md