From 67c6a8aba33a47b594617c570cc0d94bb103db8a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 28 Jul 2017 10:31:40 +0200 Subject: [PATCH] Reframe 2.5 public release --- ci-scripts/ci-runner.bash | 10 +- reframe/core/fields.py | 19 ++++ reframe/core/launchers.py | 100 +++++++++++------ reframe/core/pipeline.py | 53 +++++++-- reframe/frontend/argparse.py | 144 +++++++++++++++++++++++++ reframe/frontend/cli.py | 110 +++++++++++-------- reframe/frontend/executors/policies.py | 11 ++ reframe/frontend/loader.py | 15 ++- reframe/frontend/printer.py | 10 +- reframe/frontend/resources.py | 5 +- reframe/settings.py | 2 +- reframe/utility/os.py | 26 +++++ unittests/resources/frontend_checks.py | 11 +- unittests/test_argparser.py | 59 ++++++++++ unittests/test_cli.py | 31 +++++- unittests/test_fields.py | 22 ++++ unittests/test_launchers.py | 143 ++++++++++++++++++++++++ unittests/test_pipeline.py | 52 ++++++++- unittests/test_policies.py | 105 +++++++++++++----- unittests/test_schedulers.py | 24 ----- unittests/test_utility.py | 37 ++++++- 21 files changed, 837 insertions(+), 152 deletions(-) create mode 100644 reframe/frontend/argparse.py create mode 100644 unittests/test_argparser.py create mode 100644 unittests/test_launchers.py diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 18c912d367..b30b340e3a 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -60,7 +60,14 @@ checked_exec() run_user_checks() { - cmd="python reframe.py --prefix . --notimestamp -r -t production $@" + cmd="./bin/reframe --exec-policy=async -r -t production $@" + echo "Running user checks with \`$cmd'" + checked_exec $cmd +} + +run_serial_user_checks() +{ + cmd="./bin/reframe --exec-policy=serial -r -t production-serial $@" echo "Running user checks with \`$cmd'" checked_exec $cmd } @@ -194,6 +201,7 @@ if [ ${#userchecks[@]} -ne 0 ]; then # for i in ${!invocations[@]}; do run_user_checks ${userchecks_path} ${invocations[i]} + run_serial_user_checks ${userchecks_path} ${invocations[i]} done fi diff --git a/reframe/core/fields.py b/reframe/core/fields.py index 09b84ec571..33f49febcd 100644 --- a/reframe/core/fields.py +++ b/reframe/core/fields.py @@ -21,6 +21,20 @@ def __set__(self, obj, value): obj.__dict__[self.name] = value +class ForwardField(object): + """Simple field that forwards set/get to a target object.""" + def __init__(self, obj, attr): + self.target = obj + self.attr = attr + + def __get__(self, obj, objtype): + return self.target.__dict__[self.attr] + + + def __set__(self, obj, value): + self.target.__dict__[self.attr] = value + + class TypedField(Field): """Stores a field of predefined type""" def __init__(self, fieldname, fieldtype, allow_none = False): @@ -354,6 +368,11 @@ def __init__(self, mapping={}, scope_sep=':', global_scope='*'): self.global_scope = global_scope + def __str__(self): + # just print the internal dictionary + return str(self.scopes) + + def _check_scope_type(self, key, value): if not isinstance(key, str): raise TypeError('scope keys in a scoped dict must be strings') diff --git a/reframe/core/launchers.py b/reframe/core/launchers.py index 5b5fd692d9..d1ba5c3647 100644 --- a/reframe/core/launchers.py +++ b/reframe/core/launchers.py @@ -1,54 +1,86 @@ +from math import ceil + + class JobLauncher: - def __init__(self, job, options): - self.job = job + def __init__(self, job, options=[]): + self.job = job self.options = options - def emit_run_command(self, cmd, builder, **builder_opts): + @property + def executable(self): raise NotImplementedError('Attempt to call an abstract method') + @property + def fixed_options(self): + return [] -class LocalLauncher(JobLauncher): - def __init__(self, job, options = []): - super().__init__(job, options) - - def emit_run_command(self, cmd, builder, **builder_opts): - # Just emit the command - return builder.verbatim(cmd, **builder_opts) + def emit_run_command(self, target_executable, builder, **builder_opts): + options = ' '.join(self.fixed_options + self.options) + return builder.verbatim('%s %s %s' % \ + (self.executable, options, target_executable), + **builder_opts) class NativeSlurmLauncher(JobLauncher): - def __init__(self, job, options = []): - super().__init__(job, options) - self.launcher = 'srun %s' % (' '.join(self.options)) - - - def emit_run_command(self, cmd, builder, **builder_opts): - return builder.verbatim('%s %s' % (self.launcher, cmd), **builder_opts) + @property + def executable(self): + return 'srun' class AlpsLauncher(JobLauncher): - def __init__(self, job, options = []): - super().__init__(job, options) - self.launcher = 'aprun -B %s' % (' '.join(self.options)) + @property + def executable(self): + return 'aprun' - def emit_run_command(self, cmd, builder, **builder_opts): - return builder.verbatim('%s %s' % (self.launcher, cmd), **builder_opts) + @property + def fixed_options(self): + return [ '-B' ] class LauncherWrapper(JobLauncher): - """ - Wraps a launcher object so that you can modify the launcher's invocation - """ - def __init__(self, launcher, wrapper_cmd, wrapper_options = []): - self.launcher = launcher - self.wrapper = wrapper_cmd + """Wrap a launcher object so that its invocation may be modified.""" + def __init__(self, target_launcher, wrapper_command, wrapper_options=[]): + super().__init__(target_launcher.job, target_launcher.options) + self.target_launcher = target_launcher + self.wrapper_command = wrapper_command self.wrapper_options = wrapper_options + @property + def executable(self): + return self.wrapper_command + + @property + def fixed_options(self): + return self.wrapper_options + [ self.target_launcher.executable ] + \ + self.target_launcher.fixed_options + +class LocalLauncher(JobLauncher): def emit_run_command(self, cmd, builder, **builder_opts): - # Suppress the output of the wrapped launcher in the builder - launcher_cmd = self.launcher.emit_run_command(cmd, builder, - suppress=True) - return builder.verbatim( - '%s %s %s' % (self.wrapper, ' '.join(self.wrapper_options), - launcher_cmd), **builder_opts) + # Just emit the command + return builder.verbatim(cmd, **builder_opts) + + +class VisitLauncher(JobLauncher): + def __init__(self, job, options=[]): + super().__init__(job, options) + if self.job: + # The self.job.launcher must be stored at the moment of the + # VisitLauncher construction, because the user will afterwards set + # the newly created VisitLauncher as new self.job.launcher! + self.target_launcher = self.job.launcher + + @property + def executable(self): + return 'visit' + + @property + def fixed_options(self): + options = [] + if self.target_launcher and \ + not isinstance(self.target_launcher, LocalLauncher): + num_nodes = ceil(self.job.num_tasks/self.job.num_tasks_per_node) + options.append('-np %s' % self.job.num_tasks) + options.append('-nn %s' % num_nodes) + options.append('-l %s' % self.target_launcher.executable) + return options diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index c718a430e5..ea63aaa1c9 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -56,7 +56,7 @@ class RegressionTest(object): use_multithreading = BooleanField('use_multithreading', allow_none=True) local = BooleanField('local') prefix = StringField('prefix') - sourcesdir = StringField('sourcesdir') + sourcesdir = StringField('sourcesdir', allow_none=True) stagedir = StringField('stagedir', allow_none=True) stdout = StringField('stdout', allow_none=True) stderr = StringField('stderr', allow_none=True) @@ -171,6 +171,13 @@ def is_local(self): return self.local or self.current_partition.scheduler == 'local' + def _sanitize_basename(self, name): + """Create a basename safe to be used as path component + + Replace all path separator characters in `name` with underscores.""" + return name.replace(os.sep, '_') + + def _setup_environ(self, environ): """Setup the current environment and load it.""" @@ -196,10 +203,17 @@ def _setup_environ(self, environ): def _setup_paths(self): """Setup the check's dynamic paths.""" self.logger.debug('setting up paths') + self.stagedir = self._resources.stagedir( - self.current_partition.name, self.name, self.current_environ.name) + self._sanitize_basename(self.current_partition.name), + self.name, + self._sanitize_basename(self.current_environ.name) + ) self.outputdir = self._resources.outputdir( - self.current_partition.name, self.name, self.current_environ.name) + self._sanitize_basename(self.current_partition.name), + self.name, + self._sanitize_basename(self.current_environ.name) + ) self.stdout = os.path.join(self.stagedir, '%s.out' % self.name) self.stderr = os.path.join(self.stagedir, '%s.err' % self.name) @@ -230,10 +244,13 @@ def _setup_job(self, **job_opts): raise ReframeFatalError('Oops: unsupported launcher: %s' % self.current_partition.scheduler) - job_name = '%s_%s_%s_%s' % (self.name, - self.current_system.name, - self.current_partition.name, - self.current_environ.name) + job_name = '%s_%s_%s_%s' % ( + self.name, + self._sanitize_basename(self.current_system.name), + self._sanitize_basename(self.current_partition.name), + self._sanitize_basename(self.current_environ.name) + ) + if self.is_local(): self.job = LocalJob( job_name=job_name, @@ -342,6 +359,9 @@ def compile(self, **compile_opts): if not self.current_environ: raise ReframeError('no programming environment set') + if not self.sourcesdir: + raise ReframeError('sourcesdir is not set') + # if self.sourcepath refers to a directory, stage it first target_sourcepath = os.path.join(self.sourcesdir, self.sourcepath) if os.path.isdir(target_sourcepath): @@ -420,8 +440,8 @@ def check_performance(self): return self._match_patterns(self.perf_patterns, self.reference) - def cleanup(self, remove_files=False, unload_env=True): - # Copy stdout/stderr and job script + def _copy_to_outputdir(self): + """Copy checks interesting files to the output directory.""" self.logger.debug('copying interesting files to output directory') shutil.copy(self.stdout, self.outputdir) shutil.copy(self.stderr, self.outputdir) @@ -434,6 +454,15 @@ def cleanup(self, remove_files=False, unload_env=True): f = os.path.join(self.stagedir, f) shutil.copy(f, self.outputdir) + + def cleanup(self, remove_files=False, unload_env=True): + aliased = os.path.samefile(self.stagedir, self.outputdir) + if aliased: + self.logger.debug('skipping copy to output dir ' + 'since they alias each other') + else: + self._copy_to_outputdir() + if remove_files: self.logger.debug('removing stage directory') shutil.rmtree(self.stagedir) @@ -550,7 +579,11 @@ def compile(self, **compile_opts): def run(self): - self._copy_to_stagedir(os.path.join(self.sourcesdir, self.sourcepath)) + # The sourcesdir can be set to None by the user; then we don't copy. + if self.sourcesdir: + self._copy_to_stagedir(os.path.join(self.sourcesdir, + self.sourcepath)) + super().run() diff --git a/reframe/frontend/argparse.py b/reframe/frontend/argparse.py new file mode 100644 index 0000000000..7895d2c0c7 --- /dev/null +++ b/reframe/frontend/argparse.py @@ -0,0 +1,144 @@ +import argparse + +from reframe.core.fields import ForwardField + +# +# Notes on the ArgumentParser design +# +# An obvious design for the Reframe's `ArgumentParser` would be to directly +# inherit from `argparse.ArgumentParser`. However, this would not allow us to +# intercept the call to `add_argument()` of an argument group. Argument groups +# are of an "unknown" type to the users of the `argparse` module, since they +# inherit from an internal private class. +# +# For this reason, we base our design on composition by implementing wrappers of +# both the argument group and the argument parser. These wrappers provide the +# same public interface as their `argparse` counterparts (currently we only +# implement the part of the interface that matters for Reframe), delegating the +# parsing work to them. For these "shadow" data structures for argument groups +# and the parser, we follow a similar design as in the `argparse` module: both +# the argument group and the parser inherit from a base class implementing the +# functionality of `add_argument()`. +# +# A final trick we had to do in order to avoid repeating all the public fields +# of the internal argument holders (`argparse`'s argument group or argument +# parser) was to programmaticallly export them by creating special descriptor +# fields that forward the set/get actions to the internal argument holder. +# + + +class _ArgumentHolder(object): + def __init__(self, holder): + self._holder = holder + self._defaults = argparse.Namespace() + + # Create forward descriptors to all public members of _holder + for m in self._holder.__dict__.keys(): + if m[0] != '_': + setattr(self.__class__, m, ForwardField(self._holder, m)) + + + def _attr_from_flag(self, *flags): + if not flags: + raise ValueError('could not infer a dest name: no flags defined') + + return flags[-1].lstrip('-').replace('-', '_') + + + def _extract_default(self, *flags, **kwargs): + attr = kwargs.get('dest', self._attr_from_flag(*flags)) + action = kwargs.get('action', None) + if action == 'store_true' or action == 'store_false': + # These actions imply a default; we will convert them to their + # 'const' action equivalent and add an explicit default value + kwargs['action'] = 'store_const' + kwargs['const'] = True if action == 'store_true' else False + kwargs['default'] = False if action == 'store_true' else True + + try: + self._defaults.__dict__[attr] = kwargs['default'] + del kwargs['default'] + except KeyError: + self._defaults.__dict__[attr] = None + finally: + return kwargs + + + def add_argument(self, *flags, **kwargs): + return self._holder.add_argument( + *flags, **self._extract_default(*flags, **kwargs) + ) + + +class _ArgumentGroup(_ArgumentHolder): + pass + + +class ArgumentParser(_ArgumentHolder): + """Reframe's extended argument parser. + + This argument parser behaves almost identical to the original + `argparse.ArgumenParser`. In fact, it uses such a parser internally, + delegating all the calls to it. The key difference is how newly parsed + options are combined with existing namespaces in `parse_args()`.""" + def __init__(self, **kwargs): + super().__init__(argparse.ArgumentParser(**kwargs)) + self._groups = [] + + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self._holder.add_argument_group(*args, **kwargs)) + self._groups.append(group) + return group + + def _resolve_attr(self, attr, namespaces): + for ns in namespaces: + if ns == None: + continue + + val = ns.__dict__.setdefault(attr, None) + if val != None: + return val + + return None + + + def _update_defaults(self): + for g in self._groups: + self._defaults.__dict__.update(g._defaults.__dict__) + + + def print_help(self): + self._holder.print_help() + + + def parse_args(self, args=None, namespace=None): + """Convert argument strings to objects and return them as attributes of a + namespace. + + If `namespace` is `None`, this method is equivalent to + `argparse.ArgumentParser.parse_args()`. + + If `namespace` is not `None` and an attribute has not been assigned a + value during the parsing process of argument strings `args`, a value for + it will be looked up first in `namespace` and if not found there, it + will be assigned the default value as specified in its corresponding + `add_argument()` call. If no default value was specified either, the + attribute will be set to `None`.""" + + # We always pass an empty namespace to our internal argparser and we do + # the namespace resolution ourselves. We do this, because we want the + # newly parsed options to completely override any options defined in + # namespace. The implementation of `argparse.ArgumentParser` does not + # do this in options with an 'append' action. + options = self._holder.parse_args(args, None) + + # Update parser's defaults with groups' defaults + self._update_defaults() + for attr, val in options.__dict__.items(): + if val == None: + options.__dict__[attr] = self._resolve_attr( + attr, [ namespace, self._defaults ] + ) + + return options diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index cc4c017969..e728f21857 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1,13 +1,15 @@ -import argparse import os import socket import sys import traceback import reframe.core.logging as logging +import reframe.utility.os as os_ext from reframe.core.exceptions import ModuleError from reframe.core.modules import module_force_load, module_unload +from reframe.core.logging import getlogger +from reframe.frontend.argparse import ArgumentParser from reframe.frontend.executors import Runner from reframe.frontend.executors.policies import SerialExecutionPolicy, \ AsynchronousExecutionPolicy @@ -21,7 +23,7 @@ def list_supported_systems(systems, printer): printer.info('List of supported systems:') for s in systems: - printer.info(' ', s) + printer.info(' %s' % s) def list_checks(checks, printer): @@ -35,7 +37,7 @@ def list_checks(checks, printer): def main(): # Setup command line options - argparser = argparse.ArgumentParser() + argparser = ArgumentParser() output_options = argparser.add_argument_group( 'Options controlling regression directories') locate_options = argparser.add_argument_group('Options for locating checks') @@ -63,6 +65,10 @@ def main(): output_options.add_argument( '--keep-stage-files', action='store_true', help='Keep stage directory even if check is successful') + output_options.add_argument( + '--save-log-files', action='store_true', default=False, + help='Copy the log file from the work dir to the output dir at the ' + 'end of the program') # Check discovery options locate_options.add_argument( @@ -74,7 +80,7 @@ def main(): # Select options select_options.add_argument( - '-t', '--tag', action='append', default=[], + '-t', '--tag', action='append', dest='tags', default=[], help='Select checks matching TAG') select_options.add_argument( '-n', '--name', action='append', dest='names', default=[], @@ -143,6 +149,9 @@ def main(): choices=[ 'serial', 'async' ], default='serial', help='Specify the execution policy for running the regression tests. ' 'Available policies: "serial" (default), "async"') + run_options.add_argument( + '--mode', action='store', help='Execution mode to use' + ) misc_options.add_argument( '-m', '--module', action='append', default=[], @@ -152,19 +161,14 @@ def main(): '--nocolor', action='store_false', dest='colorize', default=True, help='Disable coloring of output') misc_options.add_argument( - '--notimestamp', action='store_false', dest='timestamp', default=True, - help='Disable timestamping when creating regression directories') - misc_options.add_argument( - '--timefmt', action='store', default='%FT%T', - help='Set timestamp format (default "%%FT%%T")') + '--timestamp', action='store', nargs='?', + const='%FT%T', metavar='TIMEFMT', + help='Append a timestamp component to the regression directories' + '(default format "%%FT%%T")' + ) misc_options.add_argument( '--system', action='store', help='Load SYSTEM configuration explicitly') - misc_options.add_argument( - '--save-log-files', action='store_true', dest='save_log_files', - default=False, - help='Copy the log file from the work dir to the output dir at the ' - 'end of the program') misc_options.add_argument('-V', '--version', action='version', version=settings.version) @@ -186,27 +190,6 @@ def main(): site_config = SiteConfiguration() site_config.load_from_dict(settings.site_configuration) - # Setup the check loader - if options.checkpath: - load_path = [] - for d in options.checkpath: - if not os.path.exists(d): - printer.info("%s: path `%s' does not exist. Skipping...\n" % - (argparser.prog, d)) - continue - - load_path.append(d) - - loader = RegressionCheckLoader(load_path, recurse=options.recursive) - else: - loader = RegressionCheckLoader( - load_path=settings.checks_path, - prefix=os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', '..') - ), - recurse=settings.checks_path_recurse, - ) - if options.system: try: sysname, sep, partname = options.system.partition(':') @@ -236,29 +219,72 @@ def main(): list_supported_systems(site_config.systems.values(), printer) sys.exit(1) + + if options.mode: + try: + mode_key = '%s:%s' % (system.name, options.mode) + mode_args = site_config.modes[options.mode] + + # Parse the mode's options and reparse the command-line + options = argparser.parse_args(mode_args) + options = argparser.parse_args(namespace=options) + except KeyError: + printer.error("no such execution mode: `%s'" % (options.mode)) + sys.exit(1) + + + # Setup the check loader + if options.checkpath: + load_path = [] + for d in options.checkpath: + if not os.path.exists(d): + printer.info("%s: path `%s' does not exist. Skipping...\n" % + (argparser.prog, d)) + continue + + load_path.append(d) + + loader = RegressionCheckLoader(load_path, recurse=options.recursive) + else: + loader = RegressionCheckLoader( + load_path=settings.checks_path, + prefix=os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..') + ), + recurse=settings.checks_path_recurse, + ) + # Adjust system directories if options.prefix: # if prefix is set, reset all other directories - system.prefix = options.prefix + system.prefix = os.path.expandvars(options.prefix) system.outputdir = None system.stagedir = None system.logdir = None if options.output: - system.outputdir = options.output + system.outputdir = os.path.expandvars(options.output) if options.stage: - system.stagedir = options.stage + system.stagedir = os.path.expandvars(options.stage) if options.logdir: - system.logdir = options.logdir + system.logdir = os.path.expandvars(options.logdir) resources = ResourcesManager(prefix=system.prefix, output_prefix=system.outputdir, stage_prefix=system.stagedir, log_prefix=system.logdir, - timestamp=options.timestamp, - timefmt=options.timefmt) + timestamp=options.timestamp) + if os_ext.samefile(resources.stage_prefix, resources.output_prefix) and \ + not options.keep_stage_files: + printer.error('stage and output refer to the same directory. ' + 'If this is on purpose, please use also the ' + "`--keep-stage-files' option.") + sys.exit(1) + + + printer.log_config(options) # Print command line printer.info('Command line: %s' % ' '.join(sys.argv)) @@ -293,7 +319,7 @@ def main(): ) # Filter checks by tags - user_tags = set(options.tag) + user_tags = set(options.tags) checks_matched = filter( lambda c: c if user_tags.issubset(c.tags) else None, checks_matched diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py index fdf315a491..935af30942 100644 --- a/reframe/frontend/executors/policies.py +++ b/reframe/frontend/executors/policies.py @@ -372,3 +372,14 @@ def exit(self): self.printer.separator( 'short single line', 'all spawned checks finished' ) + + +class DebugAsynchronousExecutionPolicy(AsynchronousExecutionPolicy): + def __init__(self): + super().__init__() + self.keep_stage_files = True + self.checks = [] + + def exit_environ(self, c, p, e): + super().exit_environ(c, p, e) + self.checks.append(c) diff --git a/reframe/frontend/loader.py b/reframe/frontend/loader.py index d777d16174..a64e5dc917 100644 --- a/reframe/frontend/loader.py +++ b/reframe/frontend/loader.py @@ -12,7 +12,7 @@ from reframe.core.environments import Environment, ProgEnvironment from reframe.core.exceptions import ConfigurationError, ReframeError from reframe.core.systems import System, SystemPartition -from reframe.core.fields import ScopedDict +from reframe.core.fields import ScopedDict, ScopedDictField from reframe.settings import settings @@ -134,8 +134,11 @@ def load_all(self, **check_args): class SiteConfiguration: """Holds the configuration of systems and environments""" + modes = ScopedDictField('modes', (list, str)) + def __init__(self): self.systems = {} + self.modes = ScopedDict({}) def load_from_dict(self, site_config): @@ -144,6 +147,7 @@ def load_from_dict(self, site_config): sysconfig = site_config.get('systems', None) envconfig = site_config.get('environments', None) + modes = site_config.get('modes', {}) if not sysconfig: raise ConfigurationError('no entry for systems was found') @@ -158,6 +162,15 @@ def load_from_dict(self, site_config): raise ConfigurationError('environments configuration ' 'is not properly formatted') + # Convert modes to a `ScopedDict`; note that `modes` will implicitly + # converted to a scoped dict here, since `self.modes` is a + # `ScopedDictField`. + try: + self.modes = modes + except TypeError: + raise ConfigurationError('modes configuration ' + 'is not properly formatted') + def create_env(system, partition, name): # Create an environment instance try: diff --git a/reframe/frontend/printer.py b/reframe/frontend/printer.py index d3ce88869b..ac00ff7493 100644 --- a/reframe/frontend/printer.py +++ b/reframe/frontend/printer.py @@ -38,7 +38,7 @@ class PrettyPrinter(object): """Pretty printing facility for the framework. Final printing is delegated to an internal logger, which is responsible for - printing both to standard output and in a specfial output file.""" + printing both to standard output and in a special output file.""" def __init__(self): self.colorize = True @@ -102,3 +102,11 @@ def error(self, msg): def info(self, msg = ''): self._logger.info(msg) + + + def log_config(self, options): + config_str = 'configuration\n' + for attr, val in sorted(options.__dict__.items()): + config_str += ' %s=%s\n' % (attr, str(val)) + + self._logger.debug(config_str) diff --git a/reframe/frontend/resources.py b/reframe/frontend/resources.py index e53f60afa3..fdf0eb7e06 100644 --- a/reframe/frontend/resources.py +++ b/reframe/frontend/resources.py @@ -5,14 +5,13 @@ import os from datetime import datetime -from reframe.settings import settings class ResourcesManager: def __init__(self, prefix = '.', output_prefix = None, stage_prefix = None, - log_prefix = None, timestamp = False, timefmt = '%FT%T'): + log_prefix = None, timestamp = None): # Get the timestamp - time = datetime.now().strftime(timefmt) if timestamp else '' + time = datetime.now().strftime(timestamp) if timestamp else '' self.prefix = os.path.abspath(prefix) if output_prefix: diff --git a/reframe/settings.py b/reframe/settings.py index 6e2f2f6c14..584d8aa3c3 100644 --- a/reframe/settings.py +++ b/reframe/settings.py @@ -8,7 +8,7 @@ from reframe.core.fields import ReadOnlyField class RegressionSettings: - version = ReadOnlyField('2.4') + version = ReadOnlyField('2.5') module_name = ReadOnlyField('reframe') job_state_poll_intervals = ReadOnlyField([ 1, 2, 3 ]) job_init_poll_intervals = ReadOnlyField([ 1 ]) diff --git a/reframe/utility/os.py b/reframe/utility/os.py index 7582b1dc80..8815fa3d8e 100644 --- a/reframe/utility/os.py +++ b/reframe/utility/os.py @@ -142,3 +142,29 @@ def subdirs(dirname, recurse=False): dirs.extend(subdirs(entry.path, recurse)) return dirs + + +def follow_link(path): + """Return the final target of a symlink chain""" + while os.path.islink(path): + path = os.readlink(path) + + return path + + +def samefile(path1, path2): + """Check if paths refer to the same file. + + If paths exist, this is equivalent to `os.path.samefile()`. If only one of + the paths exists, it will be followed if it is a symbolic link and its final + target will be compared to the other path. If both paths do not exist, a + simple string comparison will be performed (after they have been + normalized).""" + + # normalise the paths first + path1 = os.path.normpath(path1) + path2 = os.path.normpath(path2) + if os.path.exists(path1) and os.path.exists(path2): + return os.path.samefile(path1, path2) + + return follow_link(path1) == follow_link(path2) diff --git a/unittests/resources/frontend_checks.py b/unittests/resources/frontend_checks.py index 30712ed10f..fb9d320b35 100644 --- a/unittests/resources/frontend_checks.py +++ b/unittests/resources/frontend_checks.py @@ -145,11 +145,20 @@ def __init__(self, sleep_time, **kwargs): super().__init__(type(self).__name__, **kwargs) self.name += str(id(self)) self.sleep_time = sleep_time - self.executable = 'sleep %s' % self.sleep_time + self.executable = 'python3' + self.executable_opts = [ + '-c "from time import sleep; sleep(%s)"' % sleep_time + ] self.sanity_patterns = None self.valid_systems = [ '*' ] self.valid_prog_environs = [ '*' ] + def setup(self, system, environ, **job_opts): + super().setup(system, environ, **job_opts) + print_timestamp = "python3 -c \"from datetime import datetime; " \ + "print(datetime.today().strftime('%s.%f'))\"" + self.job.pre_run = [ print_timestamp ] + self.job.post_run = [ print_timestamp ] def _get_checks(**kwargs): return [ BadSetupCheck(**kwargs), diff --git a/unittests/test_argparser.py b/unittests/test_argparser.py new file mode 100644 index 0000000000..209b8c0daa --- /dev/null +++ b/unittests/test_argparser.py @@ -0,0 +1,59 @@ +import unittest + +from reframe.frontend.argparse import ArgumentParser + + +class TestArgumentParser(unittest.TestCase): + def setUp(self): + self.parser = ArgumentParser() + self.foo_options = self.parser.add_argument_group('foo options') + self.bar_options = self.parser.add_argument_group('bar options') + self.foo_options.add_argument('-f', '--foo', dest='foo', + action='store', default='FOO') + self.foo_options.add_argument('--foolist', dest='foolist', + action='append', default=[]) + self.foo_options.add_argument('--foobar', action='store_true') + self.foo_options.add_argument('--unfoo', action='store_false') + + self.bar_options.add_argument('-b', '--bar', dest='bar', + action='store', default='BAR') + self.bar_options.add_argument('--barlist', dest='barlist', + action='append', default=[]) + self.foo_options.add_argument('--barfoo', action='store_true') + + + def test_arguments(self): + self.assertRaises(ValueError, self.foo_options.add_argument, + action='store', default='FOO') + self.foo_options.add_argument('--foo-bar', action='store_true') + self.foo_options.add_argument('--alist', action='append', default=[]) + options = self.parser.parse_args([ '--foobar', '--foo-bar']) + self.assertTrue(options.foobar) + self.assertTrue(options.foo_bar) + + + def test_parsing(self): + options = self.parser.parse_args( + '--foo name --foolist gag --barfoo --unfoo'.split() + ) + self.assertEqual('name', options.foo) + self.assertEqual(['gag'], options.foolist) + self.assertTrue(options.barfoo) + self.assertFalse(options.unfoo) + + # Check the defaults now + self.assertFalse(options.foobar) + self.assertEqual('BAR', options.bar) + self.assertEqual([], options.barlist) + + # Reparse based on the already loaded options + options = self.parser.parse_args( + '--bar beer --foolist any'.split(), options + ) + self.assertEqual('name', options.foo) + self.assertEqual(['any'], options.foolist) + self.assertFalse(options.foobar) + self.assertFalse(options.unfoo) + self.assertEqual('beer', options.bar) + self.assertEqual([], options.barlist) + self.assertTrue(options.barfoo) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index 25a3556f0c..f3c3ce2b38 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -11,6 +11,7 @@ from contextlib import redirect_stdout, redirect_stderr from io import StringIO +from reframe.core.environments import EnvironmentSnapshot from reframe.frontend.loader import SiteConfiguration, autodetect_system from reframe.settings import settings from unittests.fixtures import guess_system, system_with_scheduler @@ -18,6 +19,7 @@ def run_command_inline(argv, funct, *args, **kwargs): argv_save = sys.argv + environ_save = EnvironmentSnapshot() captured_stdout = StringIO() captured_stderr = StringIO() sys.argv = argv @@ -28,6 +30,8 @@ def run_command_inline(argv, funct, *args, **kwargs): except SystemExit as e: exitcode = e.code finally: + # restore environment and command-line arguments + environ_save.load() sys.argv = argv_save return (exitcode, captured_stdout.getvalue(), @@ -43,9 +47,9 @@ def setUp(self): self.prgenv = 'builtin-gcc' self.cmdstr = '{executable} {checkopt} ' \ '--prefix {prefix} {prgenvopt} ' \ - '--notimestamp --nocolor {action} ' \ + '--nocolor {action} ' \ '{sysopt} {local} {options}' - self.options = '' + self.options = [] self.action = '-r' self.local = True @@ -85,6 +89,7 @@ def _run_reframe(self): sysopt = ('--system %s' % self.sysopt) if self.sysopt else '' ).split() + print(argv) return run_command_inline(argv, cli.main) @@ -227,6 +232,16 @@ def test_sanity_of_checks(self): self.assert_log_file_is_saved() + def test_unknown_system(self): + self.action = '-l' + self.sysopt = 'foo' + self.checkfile = None + returncode, stdout, stderr = self._run_reframe() + self.assertNotIn('Traceback', stdout) + self.assertNotIn('Traceback', stderr) + self.assertEqual(1, returncode) + + def test_sanity_of_optconfig(self): # Test the sanity of the command line options configuration self.action = '-h' @@ -255,5 +270,17 @@ def test_checkpath_recursion(self): self.assertEqual('0', num_checks_in_checkdir) + def test_same_output_stage_dir(self): + output_dir = os.path.join(self.prefix, 'foo') + self.options = ('-o %s -s %s' % (output_dir, output_dir)).split() + returncode, stdout, stderr = self._run_reframe() + self.assertEqual(1, returncode) + + # retry with --keep-stage-files + self.options.append('--keep-stage-files') + returncode, stdout, stderr = self._run_reframe() + self.assertEqual(0, returncode) + self.assertTrue(os.path.exists(output_dir)) + def tearDown(self): shutil.rmtree(self.prefix) diff --git a/unittests/test_fields.py b/unittests/test_fields.py index 0d2ad5e878..23a755fcac 100644 --- a/unittests/test_fields.py +++ b/unittests/test_fields.py @@ -418,6 +418,28 @@ def test_sandbox(self): self.assertEqual(system.name, 'mysystem') + def test_proxy_field(self): + class Target: + def __init__(self): + self.a = 1 + self.b = 2 + + t = Target() + + class Proxy: + a = ForwardField(t, 'a') + b = ForwardField(t, 'b') + + proxy = Proxy() + self.assertEqual(1, proxy.a) + self.assertEqual(2, proxy.b) + + proxy.a = 3 + proxy.b = 4 + self.assertEqual(3, t.a) + self.assertEqual(4, t.b) + + def test_settings(self): from reframe.settings import settings diff --git a/unittests/test_launchers.py b/unittests/test_launchers.py new file mode 100644 index 0000000000..f6afb3469c --- /dev/null +++ b/unittests/test_launchers.py @@ -0,0 +1,143 @@ +import re +import unittest + +from reframe.core.launchers import * +from reframe.core.schedulers import * +from reframe.core.shell import BashScriptBuilder + + +# The classes that inherit from _TestLauncher only test the launcher commands; +# nothing is actually launched (this is done in test_schedulers.py). +class _TestLauncher(unittest.TestCase): + def setUp(self): + self.builder = BashScriptBuilder() + # Pattern to match: must include only horizontal spaces [ \t] + # (\h in perl; in python \h might be introduced in future) + self.expected_launcher_patt = None + self.launcher_options = [ '--foo' ] + self.target_executable = 'hostname' + + @property + def launcher_command(self): + return ' '.join([ self.launcher.executable ] + + self.launcher.fixed_options) + + @property + def expected_shell_script_patt(self): + return '^[ \t]*%s[ \t]+--foo[ \t]+hostname[ \t]*$' % \ + self.launcher_command + + def test_launcher(self): + self.assertIsNotNone(self.launcher) + self.assertIsNotNone( + # No MULTILINE mode here; a launcher must not contain new lines. + re.search(self.expected_launcher_patt, + self.launcher_command) + ) + + def test_launcher_emit_command(self): + self.launcher.options = self.launcher_options + self.launcher.emit_run_command(self.target_executable, self.builder) + shell_script_text = self.builder.finalise() + self.assertIsNotNone(self.launcher) + self.assertIsNotNone( + re.search(self.expected_shell_script_patt, shell_script_text, + re.MULTILINE) + ) + + +class TestNativeSlurmLauncher(_TestLauncher): + def setUp(self): + super().setUp() + self.launcher = NativeSlurmLauncher(None) + self.expected_launcher_patt = '^[ \t]*srun[ \t]*$' + + +class TestAlpsLauncher(_TestLauncher): + def setUp(self): + super().setUp() + self.launcher = AlpsLauncher(None) + self.expected_launcher_patt = '^[ \t]*aprun[ \t]+-B[ \t]*$' + + +class TestLauncherWrapperAlps(_TestLauncher): + def setUp(self): + super().setUp() + self.launcher = LauncherWrapper(AlpsLauncher(None), + 'ddt', '-o foo.out'.split()) + self.expected_launcher_patt = '^[ \t]*ddt[ \t]+-o[ \t]+foo.out' \ + '[ \t]+aprun[ \t]+-B[ \t]*$' + + +class TestLauncherWrapperNativeSlurm(_TestLauncher): + def setUp(self): + super().setUp() + self.launcher = LauncherWrapper(NativeSlurmLauncher(None), + 'ddt', '-o foo.out'.split()) + self.expected_launcher_patt = '^[ \t]*ddt[ \t]+-o[ \t]+foo.out' \ + '[ \t]+srun[ \t]*$' + + +class TestLocalLauncher(_TestLauncher): + def setUp(self): + super().setUp() + self.launcher = LocalLauncher(None) + + def test_launcher(self): + self.assertRaises(NotImplementedError, + exec, 'self.launcher_command', + globals(), locals()) + + @property + def expected_shell_script_patt(self): + return '^[ \t]*hostname[ \t]*$' + + +class TestAbstractLauncher(_TestLauncher): + def setUp(self): + super().setUp() + self.launcher = JobLauncher(None) + + def test_launcher(self): + # This is implicitly tested in test_launcher_emit_command(). + pass + + def test_launcher_emit_command(self): + self.assertRaises(NotImplementedError, + super().test_launcher_emit_command) + + +class TestVisitLauncherNativeSlurm(_TestLauncher): + def setUp(self): + super().setUp() + self.job = SlurmJob(job_name='visittest', + job_environ_list=[], + job_script_builder=self.builder, + num_tasks=5, + num_tasks_per_node=2, + launcher=NativeSlurmLauncher) + self.launcher = VisitLauncher(self.job) + self.expected_launcher_patt = '^[ \t]*visit[ \t]+-np[ \t]+5[ \t]+' \ + '-nn[ \t]+3[ \t]+-l[ \t]+srun[ \t]*$' + self.launcher_options = [ '-o data.nc' ] + self.target_executable = '' + + @property + def expected_shell_script_patt(self): + return '^[ \t]*%s[ \t]+-o[ \t]+data.nc[ \t]*$' % self.launcher_command + + +class TestVisitLauncherLocal(_TestLauncher): + def setUp(self): + super().setUp() + self.job = LocalJob(job_name='visittest', + job_environ_list=[], + job_script_builder=self.builder) + self.launcher = VisitLauncher(self.job) + self.expected_launcher_patt = '^[ \t]*visit[ \t]*$' + self.launcher_options = [ '-o data.nc' ] + self.target_executable = '' + + @property + def expected_shell_script_patt(self): + return '^[ \t]*%s[ \t]+-o[ \t]+data.nc[ \t]*$' % self.launcher_command \ No newline at end of file diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index 81716a5a5b..3d3c37a271 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -146,6 +146,15 @@ def test_hellocheck_local(self): self._run_test(test) + def test_hellocheck_local_slashes(self): + # Try to fool path creation by adding slashes to environment partitions + # names + self.system.name += os.sep + 'bad' + self.progenv.name += os.sep + 'bad' + self.partition.name += os.sep + 'bad' + self.test_hellocheck_local() + + def test_run_only(self): test = RunOnlyRegressionTest('runonlycheck', 'unittests/resources', @@ -154,8 +163,8 @@ def test_run_only(self): test.executable = './hello.sh' test.executable_opts = ['Hello, World!'] test.local = True - test.valid_prog_environs = [ self.progenv.name ] - test.valid_systems = [ self.system.name ] + test.valid_prog_environs = [ '*' ] + test.valid_systems = [ '*' ] test.sanity_patterns = { '-' : { 'Hello, World\!' : [] } } @@ -227,6 +236,45 @@ def test_supports_system(self): self.assertFalse(test.supports_system('testsys:login')) + def test_sourcesdir_none(self): + test = RegressionTest('hellocheck', + 'unittests/resources', + resources=self.resources, + system=self.system) + test.sourcesdir = None + test.valid_prog_environs = [ '*' ] + test.valid_systems = [ '*' ] + self.assertRaises(ReframeError, self._run_test, test) + + + def test_sourcesdir_none_compile_only(self): + test = CompileOnlyRegressionTest('hellocheck', + 'unittests/resources', + resources=self.resources, + system=self.system) + test.sourcesdir = None + test.valid_prog_environs = [ '*' ] + test.valid_systems = [ '*' ] + self.assertRaises(ReframeError, self._run_test, test) + + + def test_sourcesdir_none_run_only(self): + test = RunOnlyRegressionTest('hellocheck', + 'unittests/resources', + resources=self.resources, + system=self.system) + test.sourcesdir = None + test.executable = 'echo' + test.executable_opts = [ "Hello, World!" ] + test.local = True + test.valid_prog_environs = [ '*' ] + test.valid_systems = [ '*' ] + test.sanity_patterns = { + '-' : { 'Hello, World\!' : [] } + } + self._run_test(test) + + class TestRegressionOutputScan(unittest.TestCase): def setUp(self): self.system = System('testsys') diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 22531034f3..3532ce5d83 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -126,7 +126,8 @@ def test_system_exit_within_test(self): class TestAsynchronousExecutionPolicy(TestSerialExecutionPolicy): def setUp(self): super().setUp() - self.runner = Runner(AsynchronousExecutionPolicy()) + self.debug_policy = DebugAsynchronousExecutionPolicy() + self.runner = Runner(self.debug_policy) def set_max_jobs(self, value): @@ -134,57 +135,105 @@ def set_max_jobs(self, value): p.max_jobs = value + def read_timestamps_sorted(self): + self.begin_stamps = [] + self.end_stamps = [] + for c in self.debug_policy.checks: + with open(c.stdout, 'r') as f: + self.begin_stamps.append(float(f.readline().strip())) + self.end_stamps.append(float(f.readline().strip())) + + self.begin_stamps.sort() + self.end_stamps.sort() + + def test_concurrency_unlimited(self): from unittests.resources.frontend_checks import SleepCheck - checks = [ SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources) ] - self.set_max_jobs(3) - - t_run = datetime.now() + checks = [ + SleepCheck(0.5, system=self.system, resources=self.resources), + SleepCheck(0.5, system=self.system, resources=self.resources), + SleepCheck(0.5, system=self.system, resources=self.resources) + ] + num_checks = len(checks) + self.set_max_jobs(num_checks) self.runner.runall(checks, self.system) - t_run = datetime.now() - t_run - self.assertLess(t_run.seconds, 2) - self.assertEqual(3, self.runner.stats.num_cases()) + # Assure that all tests were run and without failures + self.assertEqual(num_checks, self.runner.stats.num_cases()) self.assertEqual(0, self.runner.stats.num_failures()) + # Read the timestamps sorted to permit simple concurrency tests + self.read_timestamps_sorted() + + # Assure that all tests were run in parallel + self.assertTrue(self.begin_stamps[-1] < self.end_stamps[0]) + def test_concurrency_limited(self): from unittests.resources.frontend_checks import SleepCheck - checks = [ SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources) ] - self.set_max_jobs(2) - - t_run = datetime.now() + # The number of checks must be <= 2*max_jobs + t = 0.5 + checks = [ SleepCheck(t, system=self.system, resources=self.resources), + SleepCheck(t, system=self.system, resources=self.resources), + SleepCheck(t, system=self.system, resources=self.resources), + SleepCheck(t, system=self.system, resources=self.resources), + SleepCheck(t, system=self.system, resources=self.resources) ] + num_checks = len(checks) + max_jobs = num_checks - 2 + self.set_max_jobs(max_jobs) self.runner.runall(checks, self.system) - t_run = datetime.now() - t_run - self.assertGreaterEqual(t_run.seconds, 2) - self.assertLess(t_run.seconds, 3) - self.assertEqual(3, self.runner.stats.num_cases()) + # Assure that all tests were run and without failures + self.assertEqual(num_checks, self.runner.stats.num_cases()) self.assertEqual(0, self.runner.stats.num_failures()) + # Read the timestamps sorted to permit simple concurrency tests + self.read_timestamps_sorted() + + # Assure that the first #max_jobs jobs were run in parallel + self.assertTrue(self.begin_stamps[max_jobs-1] < self.end_stamps[0]) + + # Assure that the remaining jobs were each run after one of the + # previous #max_jobs jobs had finished (e.g. begin[max_jobs] > end[0]) + begin_after_end = [b > e for b, e in zip(self.begin_stamps[max_jobs:], + self.end_stamps[:-max_jobs])] + self.assertTrue(all(begin_after_end)) + + # NOTE: to assure that these remaining jobs were also run + # in parallel one could do the command hereafter; however, it would + # require to substantially increase the sleep time (in SleepCheck), + # because of the delays in rescheduling (1s, 2s, 3s, 1s, 2s,...). + # We currently prefer not to do this last concurrency test to avoid an + # important prolongation of the unit test execution time. + # self.assertTrue(self.begin_stamps[-1] < self.end_stamps[max_jobs]) + def test_concurrency_none(self): from unittests.resources.frontend_checks import SleepCheck - checks = [ SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources), - SleepCheck(1, system=self.system, resources=self.resources) ] + t = 0.5 + checks = [ SleepCheck(t, system=self.system, resources=self.resources), + SleepCheck(t, system=self.system, resources=self.resources), + SleepCheck(t, system=self.system, resources=self.resources) ] + num_checks = len(checks) self.set_max_jobs(1) - - t_run = datetime.now() self.runner.runall(checks, self.system) - t_run = datetime.now() - t_run - self.assertGreaterEqual(t_run.seconds, 3) - self.assertEqual(3, self.runner.stats.num_cases()) + # Assure that all tests were run and without failures + self.assertEqual(num_checks, self.runner.stats.num_cases()) self.assertEqual(0, self.runner.stats.num_failures()) + # Read the timestamps sorted to permit simple concurrency tests + self.read_timestamps_sorted() + + # Assure that the jobs were run after the previous job had finished + # (e.g. begin[1] > end[0]) + begin_after_end = [ b > e for b, e in zip(self.begin_stamps[1:], + self.end_stamps[:-1]) ] + self.assertTrue(all(begin_after_end)) + def _run_checks(self, checks, max_jobs): self.set_max_jobs(max_jobs) diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py index c615e697f8..54cd64f307 100644 --- a/unittests/test_schedulers.py +++ b/unittests/test_schedulers.py @@ -181,27 +181,3 @@ def test_local_job_timelimit(self): self.assertGreaterEqual(t_job.seconds, 2) self.assertLess(t_job.seconds, 10) - - - def test_launcher_wrapper_native_slurm(self): - builder = BashScriptBuilder() - ddt_launcher = LauncherWrapper(NativeSlurmLauncher(None), - 'ddt', '-o foo.out'.split()) - ddt_launcher.emit_run_command('hostname', builder) - script_text = builder.finalise() - self.assertIsNone(re.search('^\s*srun', script_text, re.MULTILINE)) - self.assertIsNotNone(re.search('^ddt\s+-o\s+foo\.out\s+srun\s+hostname', - script_text, re.MULTILINE)) - - - def test_launcher_wrapper_alps(self): - builder = BashScriptBuilder() - ddt_launcher = LauncherWrapper(AlpsLauncher(None), - 'ddt', '-o foo.out'.split()) - ddt_launcher.emit_run_command('hostname', builder) - script_text = builder.finalise() - self.assertIsNone(re.search('^\s*aprun', script_text, re.MULTILINE)) - self.assertIsNotNone( - re.search('^ddt\s+-o\s+foo\.out\s+aprun\s+-B\s+hostname', - script_text, re.MULTILINE) - ) diff --git a/unittests/test_utility.py b/unittests/test_utility.py index 8e881ba334..bbef165e53 100644 --- a/unittests/test_utility.py +++ b/unittests/test_utility.py @@ -74,7 +74,7 @@ def test_inpath(self): self.assertFalse(os_ext.inpath('/foo/bin', '/bin:/usr/local/bin')) - def test_subdirs(self): + def _make_testdirs(self, prefix): # Create a temporary directory structure # foo/ # bar/ @@ -82,12 +82,16 @@ def test_subdirs(self): # goo/ # loo/ # bar/ - prefix = tempfile.mkdtemp() os.makedirs(os.path.join(prefix, 'foo', 'bar'), exist_ok=True) os.makedirs(os.path.join(prefix, 'foo', 'bar', 'boo'), exist_ok=True) os.makedirs(os.path.join(prefix, 'foo', 'goo'), exist_ok=True) os.makedirs(os.path.join(prefix, 'loo', 'bar'), exist_ok=True) + + def test_subdirs(self): + prefix = tempfile.mkdtemp() + self._make_testdirs(prefix) + # Try to fool the algorithm by adding normal files open(os.path.join(prefix, 'foo', 'bar', 'file.txt'), 'w').close() open(os.path.join(prefix, 'loo', 'file.txt'), 'w').close() @@ -108,6 +112,35 @@ def test_subdirs(self): shutil.rmtree(prefix) + def test_samefile(self): + # Create a temporary directory structure + prefix = tempfile.mkdtemp() + self._make_testdirs(prefix) + + # Try to fool the algorithm by adding symlinks + os.symlink(os.path.join(prefix, 'foo'), + os.path.join(prefix, 'foolnk')) + os.symlink(os.path.join(prefix, 'foolnk'), + os.path.join(prefix, 'foolnk1')) + + # Create a broken link on purpose + os.symlink('/foo', os.path.join(prefix, 'broken')) + os.symlink(os.path.join(prefix, 'broken'), + os.path.join(prefix, 'broken1')) + + self.assertTrue(os_ext.samefile('/foo', '/foo')) + self.assertTrue(os_ext.samefile('/foo', '/foo/')) + self.assertTrue(os_ext.samefile('/foo/bar', '/foo//bar/')) + self.assertTrue(os_ext.samefile(os.path.join(prefix, 'foo'), + os.path.join(prefix, 'foolnk'))) + self.assertTrue(os_ext.samefile(os.path.join(prefix, 'foo'), + os.path.join(prefix, 'foolnk1'))) + self.assertFalse(os_ext.samefile('/foo', '/bar')) + self.assertTrue(os_ext.samefile('/foo', os.path.join(prefix, 'broken'))) + self.assertTrue(os_ext.samefile(os.path.join(prefix, 'broken'), + os.path.join(prefix, 'broken1'))) + + class TestCopyTree(unittest.TestCase): def setUp(self): # Create a test directory structure