diff --git a/.github/workflows/legal.py b/.github/workflows/legal.py index 8dfe5e9..800cc9c 100755 --- a/.github/workflows/legal.py +++ b/.github/workflows/legal.py @@ -77,3 +77,5 @@ def shell_style (text): return _format (text, prefix='# ') if not legal: sys.exit(1) + +print ('all legal preambles seem to be valid') diff --git a/.github/workflows/run.sh b/.github/workflows/run.sh new file mode 100755 index 0000000..c1ac000 --- /dev/null +++ b/.github/workflows/run.sh @@ -0,0 +1,153 @@ +#! /usr/bin/env bash +# +# Script runs the work flow locally. This is not a general script that runs any +# workflow like act. It is clearning and build to process these workflows since +# they are all all Ubuntu based and require a virtual Python 3.12 environment. + +do_job() +{ + python -m venv $3/venv + . $3/venv/bin/activate + python -m pip install pyyaml + python < $3/do.sh +import os +import sys +import yaml + +with open ("$1", 'rt') as file: y = yaml.safe_load(file) +lc = 0 +steps = [] +if "$2" == 'PyTesting': + print ('docker pull postgres:latest') + print ('docker run --detach --env POSTGRES_PASSWORD=password ' + '--env POSTGRES_USER=tester --name ga_postgres --network host ' + '--rm postgres:latest') + print ('sleep 5') + print ('docker exec -i ga_postgres createdb -U tester testspace') +for step in y['jobs']['$2']['steps']: + if 'run' in step: + for line in filter (len, step['run'].split('\n')): + if line.startswith ('sudo'): continue + if line.startswith ('createdb'): continue + if '&&' in line: line = line[:line.find('&&')] + if '||' in line: line = line[:line.find('||')] + cmd = line.split()[0] + lc += 1 + if cmd == 'black' and 'KEEP_STYLE' in os.environ: + line = line.replace ('--check --diff ','') + print (line,f'&& echo "result of github action step: success,{cmd}" ' + f'|| echo "result of github action step: failure,{cmd}"') +if lc: print (f'echo "github actions expected steps: {lc}"') +if "$2" == 'PyTesting': + print ('docker stop ga_postgres') + +EOF + chmod 755 $3/do.sh + $3/do.sh +} + +within() +{ + [[ $# -eq 1 ]] && return 0 + base=$1 + shift + while [[ $# -gt 0 ]] + do + [[ "$base" = "$1" ]] && return 0 + shift + done + return 1 +} + +if [ -z "$(which python)" ] +then + echo "'python' is not defined in your path. It should execute Python 3.12 or later." + exit -1 +fi +python <= 3 and sys.version_info.minor >= 12: + sys.exit(0) +else: sys.exit(-2) +EOF +if [ $? -ne 0 ] +then + echo "'python' is an older version. Need to use python 3.12 or later" + python --version + exit -2 +fi +expected_jobs="" +root=$(realpath $(dirname $0)/../..) +this=$(realpath $(dirname $0)) +wdir=$this/local +cd $root +rm -f $root/*.rpt.txt +trap "rm -rf $wdir $root/Python/build $root/Python/dawgie.egg-info $root/Python/dawgie/fe/requirements.txt" EXIT +for yaml in $(ls $this/*.yaml) +do + echo "yaml: $yaml" + for job in $(python < $root/$job.rpt.txt 2>&1 & + else + echo " skipping $job" + fi + done +done +echo "waiting for jobs to complete" +wait +rm -rf $wdir $root/Python/build $root/Python/dawgie.egg-info $root/Python/dawgie/fe/requirements.txt +declare -i summary=0 +for job in $expected_jobs +do + if [ -f $root/$job.rpt.txt ] + then + python <[str]: + result = [] + idx = frm.find(stmnt) + while idx >= 0: + result.append (frm[idx+len(stmnt):frm.find('\n',idx)].strip()) + idx = frm.find(stmnt, idx+len(stmnt)) + return result +with open ("$root/$job.rpt.txt", 'rt') as file: content = file.read() +count = int(extract (frm=content, + stmnt='github actions expected steps: ')[0]) +results = extract (frm=content, + stmnt='result of github action step: ') +summary = False +if len(results) != count: + print ('Failure: $job did not produce all of the exptected messages. See $root/$job.rpt.txt for details.') +else: + summary = all(r.startswith('success') for r in results) + if summary: print ('Success: $job') + else: print ('Failure: $job - see $root/$job.rpt.txt for details') +for result in results: + state,cmd = result.split(',') + print (' ', state+':', cmd) +if summary: + if 'KEEP_REPORTS' not in os.environ: os.unlink("$root/$job.rpt.txt") + sys.exit(0) +else: sys.exit(1) +EOF + summary=$summary+$? + else + echo "Failure: $job did not create expected report." + summary=1 + fi +done +[[ $summary -eq 0 ]] && echo "Summary: All test verifications were successful" || echo "Summary: Some or all test verifications failed." +trap - EXIT diff --git a/Python/dawgie/__init__.py b/Python/dawgie/__init__.py index 76da684..ca7b933 100644 --- a/Python/dawgie/__init__.py +++ b/Python/dawgie/__init__.py @@ -212,13 +212,13 @@ def diff(a, b, dt) -> METRIC: def measure(self, func, args=(), ds: 'Dataset' = None): c0 = resource.getrusage(resource.RUSAGE_CHILDREN) s0 = resource.getrusage(resource.RUSAGE_SELF) - t0 = datetime.datetime.utcnow() + t0 = datetime.datetime.now(datetime.UTC) value = func(*args) c1 = resource.getrusage(resource.RUSAGE_CHILDREN) s1 = resource.getrusage(resource.RUSAGE_SELF) child = _Metric.diff(c1, c0, 0) task = _Metric.diff( - s1, s0, (datetime.datetime.utcnow() - t0).total_seconds() + s1, s0, (datetime.datetime.now(datetime.UTC) - t0).total_seconds() ) self.__history.append({'child': child, 'task': task}) @@ -499,12 +499,18 @@ def do(self, goto: str = None) -> None: setattr(step, 'abort', self.abort) setattr(step, 'caller', self) - self.__timing['gather_' + step.name()] = datetime.datetime.utcnow() + self.__timing['gather_' + step.name()] = datetime.datetime.now( + datetime.UTC + ) aspect = dawgie_db.gather(step, self) - self.__timing['collect_' + step.name()] = datetime.datetime.utcnow() + self.__timing['collect_' + step.name()] = datetime.datetime.now( + datetime.UTC + ) aspect.collect(step.feedback()) aspect.collect(step.traits()) - self.__timing['start_' + self._name()] = datetime.datetime.utcnow() + self.__timing['start_' + self._name()] = datetime.datetime.now( + datetime.UTC + ) log.debug('Stepping into %s', step.name()) self.measure(step.run, args=(aspect,), ds=aspect.ds()) setattr(step, 'caller', None) @@ -821,12 +827,18 @@ def do(self, goto: str = None) -> None: setattr(step, 'abort', self.abort) setattr(step, 'caller', self) - self.__timing['retreat_' + step.name()] = datetime.datetime.utcnow() + self.__timing['retreat_' + step.name()] = datetime.datetime.now( + datetime.UTC + ) timeline = dawgie_db.retreat(step, self) - self.__timing['recede_' + step.name()] = datetime.datetime.utcnow() + self.__timing['recede_' + step.name()] = datetime.datetime.now( + datetime.UTC + ) timeline.recede(step.feedback()) timeline.recede(step.variables()) - self.__timing['start_' + self._name()] = datetime.datetime.utcnow() + self.__timing['start_' + self._name()] = datetime.datetime.now( + datetime.UTC + ) log.debug('Stepping into %s', step.name()) self.measure( step.run, args=(self._ps_hint(), timeline), ds=timeline.ds() @@ -1035,14 +1047,18 @@ def do(self, goto: str = None) -> None: setattr(step, 'caller', self) sname = f'{self._target()}.{self._name()}.{step.name()}' log.debug('Loading into %s', sname) - self.__timing['load_' + step.name()] = datetime.datetime.utcnow() + self.__timing['load_' + step.name()] = datetime.datetime.now( + datetime.UTC + ) ds = self._make_ds(step) for f in step.feedback(): ds.load(f) for p in step.previous(): ds.load(p) log.debug('Loaded into %s', sname) - self.__timing[f'start_{step.name()}'] = datetime.datetime.utcnow() + self.__timing[f'start_{step.name()}'] = datetime.datetime.now( + datetime.UTC + ) log.debug('Stepping into %s', sname) self.measure(step.run, args=(ds, self._ps_hint()), ds=ds) log.debug('Returned from %s', sname) diff --git a/Python/dawgie/fe/app.py b/Python/dawgie/fe/app.py index 4e7f6eb..7b68f25 100644 --- a/Python/dawgie/fe/app.py +++ b/Python/dawgie/fe/app.py @@ -50,9 +50,9 @@ import dawgie.pl.schedule import dawgie.pl.snapshot import enum +import importlib.metadata import json import logging; log = logging.getLogger(__name__) # fmt: skip # noqa: E702 # pylint: disable=multiple-statements -import pkg_resources import os import sys @@ -281,20 +281,18 @@ def start_state(): def versions(): - # it is iterable so pylint: disable=not-an-iterable - dl = { - p.key: p.parsed_version.base_version for p in pkg_resources.working_set - } - # pylint: enable=not-an-iterable fn = os.path.join(os.path.dirname(__file__), 'requirements.txt') vers = {'dawgie': dawgie.__version__, 'python': sys.version} with open(fn, 'rt', encoding="utf-8") as f: for name in f.readlines(): - if name in dl: - vers[name] = dl[name] - else: + name = name.strip() + for c in ['=', '<', '>']: + if c in name: + name = name[: name.find(c)] + try: + vers[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: vers[name] = 'indeterminate' - pass pass return json.dumps(vers).encode() diff --git a/Python/dawgie/pl/farm.py b/Python/dawgie/pl/farm.py index 8ce215a..abbefb6 100644 --- a/Python/dawgie/pl/farm.py +++ b/Python/dawgie/pl/farm.py @@ -246,7 +246,7 @@ def _put(job, runid: int, target: str, where: dawgie.Distribution): ) pass - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.UTC) fac = job.get('factory') msg = dawgie.pl.message.make( ctxt=dawgie.context.dumps(), diff --git a/Python/dawgie/pl/worker/__init__.py b/Python/dawgie/pl/worker/__init__.py index 6aadf50..eed14da 100644 --- a/Python/dawgie/pl/worker/__init__.py +++ b/Python/dawgie/pl/worker/__init__.py @@ -82,7 +82,7 @@ def run(self, factory, ps_hint, jobid, runid, target, timing) -> [str]: ) setattr(task, 'abort', self.abort) dawgie.pl.version.record(task, only=jobid.split('.')[1]) - timing['started'] = datetime.datetime.utcnow() + timing['started'] = datetime.datetime.now(datetime.UTC) task.do(goto=jobid.split('.')[1]) timing.update(task.timing()) return task.new_values() diff --git a/Python/dawgie/pl/worker/aws.py b/Python/dawgie/pl/worker/aws.py index 116e2ef..69ce8f9 100644 --- a/Python/dawgie/pl/worker/aws.py +++ b/Python/dawgie/pl/worker/aws.py @@ -191,7 +191,7 @@ def advertise(self): pass message = { 'id': os.path.basename(self._did), - 'now': datetime.datetime.utcnow(), + 'now': datetime.datetime.now(datetime.UTC), 'payload': '', 'step': 'advertise', } @@ -218,7 +218,7 @@ def advertise(self): def interview(self): message = { 'id': os.path.basename(self._did), - 'now': datetime.datetime.utcnow(), + 'now': datetime.datetime.now(datetime.UTC), 'payload': '', 'step': 'interview', } @@ -257,7 +257,7 @@ def interview(self): def hire(self): message = { 'id': os.path.basename(self._did), - 'now': datetime.datetime.utcnow(), + 'now': datetime.datetime.now(datetime.UTC), 'payload': dawgie.pl.message.dumps(self._job), 'step': 'hire', } @@ -414,7 +414,7 @@ def exchange(message): # AWS lambda function step = message['step'] if ( - datetime.datetime.utcnow() - message['now'] + datetime.datetime.now(datetime.UTC) - message['now'] ).total_seconds() < 50: print('exchange() - within time bounds') print('exchange() - do', step) diff --git a/Python/dawgie/security.py b/Python/dawgie/security.py index a7503df..e71cf7e 100755 --- a/Python/dawgie/security.py +++ b/Python/dawgie/security.py @@ -113,7 +113,9 @@ def _p3(self, hid: bytes) -> bool: if response.valid: hid = _PGP.decrypt(hid).data.decode() log.debug('Received handshake identification:\n%s', hid) - self.__msg = 'timestamp: ' + str(datetime.datetime.utcnow()) + self.__msg = 'timestamp: ' + str( + datetime.datetime.now(datetime.UTC) + ) self.__msg += '\nunique id: ' + str(random.random()) msg = struct.pack('>I', len(self.__msg)) + self.__msg.encode() @@ -241,7 +243,7 @@ def connect(address: (str, int)) -> socket.socket: s.connect(address) message = ' machine: ' + _my_ip() + '\n' - message += 'temporal: ' + str(datetime.datetime.utcnow()) + '\n' + message += 'temporal: ' + str(datetime.datetime.now(datetime.UTC)) + '\n' message += 'username: ' + getpass.getuser() + '\n' _send(s, message) _send(s, _recv(s))