Skip to content

Commit

Permalink
moved part of BBIDaily to neuro-forge
Browse files Browse the repository at this point in the history
  • Loading branch information
@denisri
denisri committed Jul 18, 2024
1 parent 1f489e2 commit 10f93a9
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 211 deletions.
6 changes: 3 additions & 3 deletions python/casa_distro/admin_commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@
from casa_distro.command import command, check_boolean
from casa_distro.defaults import (default_base_directory,
publish_url, default_download_url)
from casa_distro.environment import (BBIDaily,
casa_distro_directory,
from casa_distro.environment import (CasaDistroBBIDaily, casa_distro_directory,
iter_environments,
run_container,
select_environment,
Expand Down Expand Up @@ -1191,7 +1190,8 @@ def bbi_daily(distro=None, branch=None, system=None,
jenkins_password)
else:
jenkins = None
bbi_daily = BBIDaily(base_directory, jenkins=jenkins)

bbi_daily = CasaDistroBBIDaily(base_directory, jenkins=jenkins)

if update_casa_distro:
# Update casa_distro with git and restart with update_casa_distro=no
Expand Down
242 changes: 34 additions & 208 deletions python/casa_distro/environment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,54 @@

import errno
import fnmatch
import getpass
from glob import glob
import json
import locale
import os
import os.path as osp
import re
import shutil
import socket
import stat
import subprocess
import sys
import time
import tempfile

from casa_distro import six
from casa_distro.six.moves import shlex_quote

from casa_distro import share_directories
from casa_distro import singularity
from casa_distro.web import url_listdir
from casa_distro import downloader


def add_soma_forge_path():
try:
import neuro_forge # noqa: F401
return
except ImportError:
pass

dn = osp.dirname(osp.dirname(osp.dirname(__file__)))
branch = None
if osp.basename(dn) != 'casa_distro':
branch = osp.basename(dn)
dn = osp.dirname(dn)
dn = osp.dirname(dn)
dn = osp.join(dn, 'neuro-forge')
if not osp.exists(osp.join(dn, 'neuro_forge')):
if branch:
dn = osp.join(dn, branch)
else:
d = os.listdir(dn)[0]
dn = osp.join(dn, d)
sys.path.append(dn)


add_soma_forge_path()
try:
from neuro_forge.soma_forge import bbi_daily
except ImportError:
bbi_daily = None


bv_maker_branches = {
'latest_release': 'latest_release',
'master': 'master',
Expand Down Expand Up @@ -854,61 +878,21 @@ def run_container(config, command, gui, opengl, root, cwd, env, image,
verbose=verbose)


class BBIDaily:
class CasaDistroBBIDaily(bbi_daily.BBIDaily if bbi_daily else object):
def __init__(self, base_directory, jenkins=None):
# This environment variable must be set by the caller of BBIDaily, to
# ensure that all recursively called instances of casa_distro will use
# the correct base_directory.
assert os.environ['CASA_BASE_DIRECTORY'] == base_directory
self.bbe_name = 'BBE-{0}-{1}'.format(getpass.getuser(),
socket.gethostname())
super().__init__(base_directory=base_directory, jenkins=jenkins)
self.casa_distro_src = osp.dirname(osp.dirname(
osp.dirname(__file__)))
casa_distro = osp.join(self.casa_distro_src, 'bin',
'casa_distro')
casa_distro_admin = osp.join(self.casa_distro_src, 'bin',
'casa_distro_admin')
self.casa_distro_cmd = [sys.executable, casa_distro]
self.casa_distro_cmd_env = None
self.casa_distro_admin_cmd = [sys.executable, casa_distro_admin]
self.jenkins = jenkins
if self.jenkins:
if not self.jenkins.job_exists(self.bbe_name):
self.jenkins.create_job(self.bbe_name)

def log(self, environment, task_name, result, log,
duration=None):
if self.jenkins:
self.jenkins.create_build(environment=environment,
task=task_name,
result=result,
log=log+'\n',
duration=duration)
else:
name = '{0}:{1}'.format(environment, task_name)
print()
print(' /-' + '-' * len(name) + '-/')
print(' / ' + name + ' /')
print('/-' + '-' * len(name) + '-/')
print()
print(log)

def call_output(self, args, **kwargs):
p = subprocess.Popen(args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, bufsize=-1,
**kwargs)
output, nothing = p.communicate()
output = six.ensure_str(output,
encoding=locale.getpreferredencoding(),
errors='backslashreplace')
log = ['-'*40,
'$ ' + ' '.join(shlex_quote(six.ensure_str(arg))
for arg in args),
'-'*40,
output]

return p.returncode, '\n'.join(log)

def update_casa_distro(self):
super().update_casa_distro()
start = time.time()
result, log = self.call_output(['git',
'-C', self.casa_distro_src,
Expand Down Expand Up @@ -936,164 +920,6 @@ def update_base_images(self, images):
duration=duration)
return result == 0

def bv_maker(self, config, steps):
environment = config['name']
if self.jenkins:
if not self.jenkins.job_exists(environment):
self.jenkins.create_job(environment,
**config)
done = []
failed = []
for step in steps:
start = time.time()
result, log = self.call_output(self.casa_distro_cmd + [
'bv_maker',
'name={0}'.format(config['name']),
'--',
step,
])
duration = int(1000 * (time.time() - start))
self.log(environment, step, result, log, duration=duration)
if result:
failed.append(step)
break # stop on the first failed step
else:
done.append(step)
return (done, failed)

def tests(self, test_config, dev_config):
environment = test_config['name']
if self.jenkins:
if not self.jenkins.job_exists(environment):
self.jenkins.create_job(environment,
**test_config)
# get test commands dict, and log it in the test config log (which may
# be the dev log or the user image log)
tests = self.get_test_commands(dev_config,
log_config_name=test_config['name'])
successful_tests = []
failed_tests = []
srccmdre = re.compile('/casa/host/src/.*/bin/')
for test, commands in tests.items():
log = []
start = time.time()
success = True
for command in commands:
if test_config['type'] in ('run', 'user'):
# replace paths in build dir with install ones
command = command.replace('/casa/host/build',
'/casa/install')
# replace paths in sources with install ones
command = srccmdre.sub('/casa/install/bin/', command)
result, output = self.call_output(self.casa_distro_cmd + [
'run',
'name={0}'.format(test_config['name']),
'env=BRAINVISA_TEST_RUN_DATA_DIR=/casa/host/tests/test,'
'BRAINVISA_TEST_REF_DATA_DIR=/casa/host/tests/ref',
'--',
'sh', '-c', command
])
log.append('=' * 80)
log.append(output)
log.append('=' * 80)
if result:
success = False
if result in (124, 128+9):
log.append('TIMED OUT (exit code {0})'.format(result))
else:
log.append('FAILED with exit code {0}'
.format(result))
else:
log.append('SUCCESS (exit code {0})'.format(result))
duration = int(1000 * (time.time() - start))
self.log(environment, test, (0 if success else 1),
'\n'.join(log), duration=duration)
if success:
successful_tests.append(test)
else:
failed_tests.append(test)
if failed_tests:
self.log(environment, 'tests failed', 1,
'The following tests failed: {0}'.format(
', '.join(failed_tests)))
return (successful_tests, failed_tests)

def get_test_commands(self, config, log_config_name=None):
'''
Given the config of a dev environment, return a dictionary
whose keys are name of a test (i.e. 'axon', 'soma', etc.) and
values are a list of commands to run to perform the test.
'''
cmd = self.casa_distro_cmd + [
'run',
'name={0}'.format(config['name']),
'cwd=/casa/host/build',
'--',
'ctest', '--print-labels'
]
# universal_newlines is the old name to request text-mode (text=True)
o = subprocess.check_output(cmd, bufsize=-1,
universal_newlines=True)
labels = [i.strip() for i in o.split('\n')[2:] if i.strip()]
log_lines = ['$ ' + ' '.join(shlex_quote(arg) for arg in cmd),
o, '\n']
tests = {}
for label in labels:
cmd = self.casa_distro_cmd + [
'run',
'name={0}'.format(config['name']),
'cwd=/casa/host/build',
'env=BRAINVISA_TEST_REMOTE_COMMAND=echo',
'--',
'ctest', '-V', '-L',
'^{0}$'.format(label)
] + config.get('ctest_options', [])
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=-1,
universal_newlines=True)
o, stderr = p.communicate()
log_lines += ['$ ' + ' '.join(shlex_quote(arg) for arg in cmd),
o, '\n']
if p.returncode != 0:
# We want to hide stderr unless ctest returns a nonzero exit
# code. In the case of test filtering where no tests are
# matched (e.g. with ctest_options=['-R', 'dummyfilter']), the
# annoying message 'No tests were found!!!' is printed to
# stderr by ctest, but it exits with return code 0.
sys.stderr.write(stderr)
log_lines.append('Error in get_test_commands:')
log_lines.append('get_test_commands command:')
log_lines.append("'" + "' '".join(cmd) + "'")
log_lines.append('Error:')
log_lines.append(stderr)
self.log(log_config_name, 'get test commands', 0,
'\n'.join(log_lines))
raise RuntimeError('ctest failed with the above error')
o = o.split('\n')
# Extract the third line that follows each line containing ': Test
# command:'
commands = [o[i+2][o[i+2].find(':')+2:].strip()
for i in range(len(o))
if ': Test command:' in o[i]]
timeouts = [o[i+1][o[i+1].find(':')+2:].strip()
for i in range(len(o))
if ': Test command:' in o[i]]
timeouts = [x[x.find(':')+2:] for x in timeouts]
if commands: # skip empty command lists
for i, command in enumerate(commands):
if float(timeouts[i]) < 9.999e+06:
command = 'timeout -k 10 %s %s' % (timeouts[i],
command)
commands[i] = command
tests[label] = commands
log_lines += ['Final test dictionary:',
json.dumps(tests, indent=4, separators=(',', ': '))]

if log_config_name is None:
log_config_name = config['name']
self.log(log_config_name, 'get test commands', 0, '\n'.join(log_lines))
return tests

def recreate_user_env(self, user_config, dev_config):
environment = user_config['name']
if self.jenkins:
Expand Down

0 comments on commit 10f93a9

Please sign in to comment.