From 5d1b2b7379e043272e523907bdd15ee234785400 Mon Sep 17 00:00:00 2001 From: Liran BG Date: Sun, 17 Nov 2024 10:39:45 +0200 Subject: [PATCH] [Python] Bump python to 3.9 (#62) --- .github/workflows/CI.yaml | 24 +- .gitignore | 1 + Makefile | 12 +- clients/logging/__init__.py | 88 +++---- clients/logging/formatter/helpers.py | 30 +-- clients/logging/formatter/human_readable.py | 64 ++--- clients/logging/formatter/json.py | 20 +- core/__init__.py | 54 ++-- core/update_manager.py | 24 +- examples/basic/manofest.py | 58 ++--- install | 2 +- manof.py | 198 +++++++-------- manof/image.py | 238 +++++++++--------- manof/target.py | 30 +-- manof/utils/__init__.py | 56 ++--- manof/volume.py | 34 +-- requirements/common.txt | 16 +- requirements/dev.txt | 4 +- tests/integration/__init__.py | 24 +- .../cases/basic/artifacts/manofest.py | 4 +- .../cases/basic/test_basic_commands.py | 26 +- .../shell_commands/artifacts/manofest.py | 18 +- .../shell_commands/test_shell_commands.py | 78 +++--- tests/unit/test_manof.py | 52 ++-- 24 files changed, 573 insertions(+), 582 deletions(-) diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 8dc768f..7831c15 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -13,15 +13,15 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies - run: make install-ci + run: make install - name: Run lint run: make lint @@ -29,15 +29,15 @@ jobs: unit_test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies - run: make install-ci + run: make install - name: Run unit test run: make test-unit @@ -45,15 +45,15 @@ jobs: integration_test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: 3.9 - name: Install dependencies - run: make install-ci + run: make install - name: Run integration test run: make test-integ diff --git a/.gitignore b/.gitignore index f4b8f74..9e719ab 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ pip-selfcheck.json *.egg-info .coverage .ziggy +.python-version .idea/* !.idea/runConfigurations/ diff --git a/Makefile b/Makefile index 018182e..9fec1da 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,12 @@ lint: venv fmt-check .PHONY: fmt fmt: @echo "Running black fmt..." - $(VENV_PYTHON) -m black --skip-string-normalization --exclude='.*venv.*' . + $(VENV_PYTHON) -m black --exclude='.*venv.*' . .PHONY: fmt-check fmt-check: @echo "Running black fmt check..." - $(VENV_PYTHON) -m black --skip-string-normalization --check --diff . + $(VENV_PYTHON) -m black --check --diff . .PHONY: test test: test-unit test-integ @@ -33,13 +33,5 @@ test-integ: venv install: venv @echo Installed -.PHONY: install-ci -install-ci: install-venv install - venv: python ./install --dev - -.PHONY: install-venv -install-venv: - python -m pip install --upgrade pip - python -m pip install virtualenv diff --git a/clients/logging/__init__.py b/clients/logging/__init__.py index 2700090..899990c 100644 --- a/clients/logging/__init__.py +++ b/clients/logging/__init__.py @@ -26,25 +26,25 @@ def __init__(self, name, level=logging.NOTSET): # a new Logger instance and adds it to his list # so we need to add the first error to the manager attributes # so we can keep the first error in the whole application - if not hasattr(self.manager, 'first_error'): - setattr(self.manager, 'first_error', None) + if not hasattr(self.manager, "first_error"): + setattr(self.manager, "first_error", None) @property def first_error(self): return self.manager.first_error def clear_first_error(self): - if hasattr(self.manager, 'first_error'): + if hasattr(self.manager, "first_error"): self.manager.first_error = None def _check_and_log(self, level, msg, args, kw_args): if self.isEnabledFor(level): kw_args.update(self._bound_variables) - self._log(level, msg, args, extra={'vars': kw_args}) + self._log(level, msg, args, extra={"vars": kw_args}) def error(self, msg, *args, **kw_args): if self.manager.first_error is None: - self.manager.first_error = {'msg': msg, 'args': args, 'kw_args': kw_args} + self.manager.first_error = {"msg": msg, "args": args, "kw_args": kw_args} self._check_and_log(helpers.Severity.Error, msg, args, kw_args) @@ -70,22 +70,22 @@ def __init__(self, logger_instance): def __call__(self, event_info): try: - if event_info['isError'] == 1: + if event_info["isError"] == 1: try: self.logger_instance.error( - 'Unhandled exception in deferred', - failure=str(event_info['failure']).replace('\n', '\n\r'), + "Unhandled exception in deferred", + failure=str(event_info["failure"]).replace("\n", "\n\r"), traceback=str( - event_info['failure'].getBriefTraceback() - ).replace('\n', '\n\r'), + event_info["failure"].getBriefTraceback() + ).replace("\n", "\n\r"), ) except Exception: pass try: - if len(event_info['message']) > 0: + if len(event_info["message"]) > 0: self.logger_instance.error( - str(event_info['message']).replace('\n', '\n\r') + str(event_info["message"]).replace("\n", "\n\r") ) except Exception: pass @@ -105,7 +105,7 @@ def __init__( max_log_size_mb=5, max_num_log_files=3, log_file_name=None, - log_colors='on', + log_colors="on", ): # disabled - this hijacks stdout and adds RESETCOLOR at the end regardless if we are on atty or not @@ -134,9 +134,9 @@ def __init__( # on - disable colors if stdout is not a tty # always - never disable colors # off - always disable colors - if log_colors == 'off': + if log_colors == "off": enable_colors = False - elif log_colors == 'always': + elif log_colors == "always": enable_colors = True else: # on - colors when stdout is a tty enable_colors = sys.stdout.isatty() @@ -154,9 +154,9 @@ def __init__( if output_dir is not None: log_file_name = ( - name.replace('-', '.') + name.replace("-", ".") if log_file_name is None - else log_file_name.replace('.log', '') + else log_file_name.replace(".log", "") ) self.enable_log_file_writing( output_dir, @@ -192,12 +192,12 @@ def enable_log_file_writing( for h in self.logger.handlers ): helpers.make_dir_recursively(output_dir) - log_path = os.path.join(output_dir, '{0}.log'.format(log_file_name)) + log_path = os.path.join(output_dir, "{0}.log".format(log_file_name)) # Creates the log file if it doesn't already exist. rotating_file_handler = logging.handlers.RotatingFileHandler( log_path, - mode='a+', + mode="a+", maxBytes=max_log_size_mb * 1024 * 1024, backupCount=max_num_log_files, ) @@ -217,55 +217,55 @@ def register_arguments(parser): :param parser: The argparser """ parser.add_argument( - '--log-severity', - help='Set log severity', + "--log-severity", + help="Set log severity", choices=helpers.Severity.string_enum_dict.keys(), - default='debug', + default="debug", ) # old-style abbreviation log-level for backwards compatibility parser.add_argument( - '--log-console-severity', - help='Defines severity of logs printed to console', + "--log-console-severity", + help="Defines severity of logs printed to console", choices=helpers.Severity.string_enum_dict.keys(), - default='debug', + default="debug", ) # old-style abbreviation log-level for backwards compatibility parser.add_argument( - '--log-file-severity', - help='Defines severity of logs printed to file', + "--log-file-severity", + help="Defines severity of logs printed to file", choices=helpers.Severity.string_enum_dict.keys(), - default='debug', + default="debug", ) parser.add_argument( - '--log-disable-stdout', - help='Disable logging to stdout', - action='store_true', + "--log-disable-stdout", + help="Disable logging to stdout", + action="store_true", ) - parser.add_argument('--log-output-dir', help='Log files directory path') + parser.add_argument("--log-output-dir", help="Log files directory path") parser.add_argument( - '--log-file-rotate-max-file-size', help='Max log file size', default=5 + "--log-file-rotate-max-file-size", help="Max log file size", default=5 ) parser.add_argument( - '--log-file-rotate-num-files', help='Num of log files to keep', default=5 + "--log-file-rotate-num-files", help="Num of log files to keep", default=5 ) parser.add_argument( - '--log-file-name', + "--log-file-name", help=( - 'Override to filename (instead of deriving it from the logger name. ' - 'e.g. [node_name].[service_name].[service_instance].log' + "Override to filename (instead of deriving it from the logger name. " + "e.g. [node_name].[service_name].[service_instance].log" ), ) parser.add_argument( - '--log-colors', + "--log-colors", help=( - 'CLI friendly color control. default is on (color when stdout+tty). ' - 'You can also force always/off.' + "CLI friendly color control. default is on (color when stdout+tty). " + "You can also force always/off." ), - choices=['on', 'off', 'always'], - default='on', + choices=["on", "off", "always"], + default="on", ) @@ -274,5 +274,5 @@ class TestingClient(Client): An override of the logging client with defaults suitable for testing """ - def __init__(self, name='test', initial_severity='debug'): - super(TestingClient, self).__init__(name, initial_severity, log_colors='always') + def __init__(self, name="test", initial_severity="debug"): + super(TestingClient, self).__init__(name, initial_severity, log_colors="always") diff --git a/clients/logging/formatter/helpers.py b/clients/logging/formatter/helpers.py index bb3c1fe..a2273fa 100644 --- a/clients/logging/formatter/helpers.py +++ b/clients/logging/formatter/helpers.py @@ -13,18 +13,18 @@ class Severity(object): Error = logging.ERROR string_enum_dict = { - 'verbose': Verbose, - 'debug': Debug, - 'info': Info, - 'warn': Warning, - 'warning': Warning, + "verbose": Verbose, + "debug": Debug, + "info": Info, + "warn": Warning, + "warning": Warning, # Allow abbreviations # Also provides backwards compatibility with log-console/file-severity syntax - 'V': Verbose, - 'D': Debug, - 'I': Info, - 'W': Warning, - 'E': Error, + "V": Verbose, + "D": Debug, + "I": Info, + "W": Warning, + "E": Error, } @staticmethod @@ -51,15 +51,15 @@ def format_to_json_str(params): # this is the widest complementary encoding found return simplejson.dumps( - params, cls=ObjectEncoder, encoding='raw_unicode_escape' + params, cls=ObjectEncoder, encoding="raw_unicode_escape" ) def format(self, record): params = { - 'datetime': self.formatTime(record, self.datefmt), - 'name': record.name, - 'level': record.levelname.lower(), - 'message': record.getMessage(), + "datetime": self.formatTime(record, self.datefmt), + "name": record.name, + "level": record.levelname.lower(), + "message": record.getMessage(), } params.update(record.vars) diff --git a/clients/logging/formatter/human_readable.py b/clients/logging/formatter/human_readable.py index 239b76c..29d02c5 100644 --- a/clients/logging/formatter/human_readable.py +++ b/clients/logging/formatter/human_readable.py @@ -17,11 +17,11 @@ def __init__(self, enable_colors, *args, **kwargs): # Maps severity to its letter representation _level_to_short_name = { - helpers.Severity.Verbose: 'V', - helpers.Severity.Debug: 'D', - helpers.Severity.Info: 'I', - helpers.Severity.Warning: 'W', - helpers.Severity.Error: 'E', + helpers.Severity.Verbose: "V", + helpers.Severity.Debug: "D", + helpers.Severity.Info: "I", + helpers.Severity.Warning: "W", + helpers.Severity.Error: "E", } # Maps severity to its color representation @@ -42,44 +42,44 @@ def _get_what_color(): # coloured using pygments if self._enable_colors: - more = self._prettify_output(record.vars) if len(record.vars) else '' + more = self._prettify_output(record.vars) if len(record.vars) else "" else: - more = simplejson.dumps(record.vars) if len(record.vars) else '' + more = simplejson.dumps(record.vars) if len(record.vars) else "" output = { - 'reset_color': colorama.Fore.RESET, - 'when': datetime.datetime.fromtimestamp(record.created).strftime( - '%d.%m.%y %H:%M:%S.%f' + "reset_color": colorama.Fore.RESET, + "when": datetime.datetime.fromtimestamp(record.created).strftime( + "%d.%m.%y %H:%M:%S.%f" ), - 'when_color': colorama.Fore.WHITE, - 'who': record.name[-30:], - 'who_color': colorama.Fore.WHITE, - 'severity': HumanReadableFormatter._level_to_short_name[record.levelno], - 'severity_color': HumanReadableFormatter._level_to_color.get( + "when_color": colorama.Fore.WHITE, + "who": record.name[-30:], + "who_color": colorama.Fore.WHITE, + "severity": HumanReadableFormatter._level_to_short_name[record.levelno], + "severity_color": HumanReadableFormatter._level_to_color.get( record.levelno, colorama.Fore.RESET ), - 'what': record.getMessage(), - 'what_color': _get_what_color(), - 'more': more, + "what": record.getMessage(), + "what_color": _get_what_color(), + "more": more, } # Slice ms to be at maximum of 3 digits try: - time_parts = output['when'].split('.') + time_parts = output["when"].split(".") time_parts[-1] = time_parts[-1][:-3] - output['when'] = '.'.join(time_parts) + output["when"] = ".".join(time_parts) except Exception: pass # Disable coloring if requested if not self._enable_colors: - for ansi_color in [f for f in output.keys() if 'color' in f]: - output[ansi_color] = '' + for ansi_color in [f for f in output.keys() if "color" in f]: + output[ansi_color] = "" return ( - '{when_color}{when}{reset_color} {who_color}{who:>10}:{reset_color} ' - '{severity_color}({severity}){reset_color} {what_color}{what}{reset_color} ' - '{more}'.format(**output) + "{when_color}{when}{reset_color} {who_color}{who:>10}:{reset_color} " + "{severity_color}({severity}){reset_color} {what_color}{what}{reset_color} " + "{more}".format(**output) ) def _prettify_output(self, vars_dict): @@ -96,7 +96,7 @@ def _prettify_output(self, vars_dict): # some params for the long texts long_values = [] - content_indent = ' ' + content_indent = " " wrap_width = 80 for var_name, var_value in vars_dict.items(): @@ -133,21 +133,21 @@ def _prettify_output(self, vars_dict): # The long text is not a full json string, but a raw string (not escaped), as to keep it human readable, # but it is surrounded by double-quotes so the coloring lexer will eat it up - values_str = '' + values_str = "" if short_values: values_str = helpers.JsonFormatter.format_to_json_str( {k: v for k, v in short_values} ) if long_values: - values_str += '\n' + values_str += "\n" for lv_name, lv_value in long_values: - values_str += '{{{0}:\n{1}}}\n'.format( + values_str += "{{{0}:\n{1}}}\n".format( helpers.JsonFormatter.format_to_json_str(lv_name), - lv_value.rstrip('\n'), + lv_value.rstrip("\n"), ) - json_lexer = pygments.lexers.get_lexer_by_name('Json') + json_lexer = pygments.lexers.get_lexer_by_name("Json") formatter = pygments.formatters.get_formatter_by_name( - 'terminal16m', style='paraiso-dark' + "terminal16m", style="paraiso-dark" ) return pygments.highlight(values_str, json_lexer, formatter) diff --git a/clients/logging/formatter/json.py b/clients/logging/formatter/json.py index 443c0a8..47d196a 100644 --- a/clients/logging/formatter/json.py +++ b/clients/logging/formatter/json.py @@ -13,25 +13,25 @@ def format(self, record): # we can't delete from record.vars because of other handlers more = dict(record.vars) if len(record.vars) else {} try: - del more['ctx'] + del more["ctx"] except Exception: pass except Exception as exc: - more = f'Record vars are not parsable: {str(exc)}' + more = f"Record vars are not parsable: {str(exc)}" try: what = record.getMessage() except Exception as exc: - what = f'Log message is not parsable: {str(exc)}' + what = f"Log message is not parsable: {str(exc)}" output = { - 'when': datetime.datetime.fromtimestamp(record.created).isoformat(), - 'who': record.name, - 'severity': logging.getLevelName(record.levelno), - 'what': what, - 'more': more, - 'ctx': record.vars.get('ctx', ''), - 'lang': 'py', + "when": datetime.datetime.fromtimestamp(record.created).isoformat(), + "who": record.name, + "severity": logging.getLevelName(record.levelno), + "what": what, + "more": more, + "ctx": record.vars.get("ctx", ""), + "lang": "py", } return clients.logging.formatter.helpers.JsonFormatter.format_to_json_str( diff --git a/core/__init__.py b/core/__init__.py index 73fb752..3393a3d 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -17,7 +17,7 @@ class RootTarget(manof.Target): def name(self): # override default behavior of "root_target" - return 'root' + return "root" class Manof(object): @@ -25,18 +25,18 @@ def __init__(self, logger, args, known_arg_options): self._logger = logger self._args = self._ungreedify_targets(args, known_arg_options) - if hasattr(self._args, 'print_command_only') and self._args.print_command_only: + if hasattr(self._args, "print_command_only") and self._args.print_command_only: self._args.dry_run = True self._logger.setLevel(0) elif ( - hasattr(self._args, 'print_run_md5_only') and self._args.print_run_md5_only + hasattr(self._args, "print_run_md5_only") and self._args.print_run_md5_only ): self._args.dry_run = True self._logger.setLevel(0) # Set number of tries according to args (only effects pull and push) self._number_of_tries = ( - self._args.num_retries + 1 if self._args.command in ['pull', 'push'] else 1 + self._args.num_retries + 1 if self._args.command in ["pull", "push"] else 1 ) manof_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -61,17 +61,17 @@ def _ungreedify_targets(self, parsed_args, known_arg_options): value = args[idx + 1] if ( - arg.startswith('-') - and not value.startswith('-') - and '=' not in arg + arg.startswith("-") + and not value.startswith("-") + and "=" not in arg and arg not in known_arg_options ): if value in parsed_args.targets: parsed_args.targets.remove(value) if not len(parsed_args.targets): raise IOError( - 'No targets arguments found. You must have entered a bad' - ' argument combination' + "No targets arguments found. You must have entered a bad" + " argument combination" ) return parsed_args @@ -82,7 +82,7 @@ def execute_command(self): # run the command def _log_tracebacks(failure): self._logger.error( - 'Unhandled exception running command', + "Unhandled exception running command", command=self._args.command, error=failure.getErrorMessage(), traceback=failure.getTraceback(), @@ -97,25 +97,25 @@ def _log_tracebacks(failure): yield d def provision(self): - return self._run_command_on_target_tree('provision') + return self._run_command_on_target_tree("provision") def run(self): - return self._run_command_on_target_tree('run') + return self._run_command_on_target_tree("run") def stop(self): - return self._run_command_on_target_tree('stop') + return self._run_command_on_target_tree("stop") def lift(self): - return self._run_command_on_target_tree('lift') + return self._run_command_on_target_tree("lift") def rm(self): - return self._run_command_on_target_tree('rm') + return self._run_command_on_target_tree("rm") def push(self): - return self._run_command_on_target_tree('push') + return self._run_command_on_target_tree("push") def pull(self): - return self._run_command_on_target_tree('pull') + return self._run_command_on_target_tree("pull") def update(self): return self._update_manager.update() @@ -171,7 +171,7 @@ def _load_manofest(self): targets = self._load_targets_from_manofest(self._args.manofest_path) # create a new argparser - secondary_ap = argparse.ArgumentParser(conflict_handler='resolve') + secondary_ap = argparse.ArgumentParser(conflict_handler="resolve") # pass I # iterate over targets and register class level arguments @@ -202,11 +202,11 @@ def _load_manofest(self): def _load_targets_from_manofest(self, manofest_path): target_instances = [] excluded_targets = ( - self._args.exclude.split(',') if 'exclude' in self._args else [] + self._args.exclude.split(",") if "exclude" in self._args else [] ) # start by loading the manofest module - self._logger.debug('Loading manofest', manofest_path=manofest_path) + self._logger.debug("Loading manofest", manofest_path=manofest_path) manofest_module = self._load_manofest_module(manofest_path) # normalize to cls names @@ -221,7 +221,7 @@ def _load_targets_from_manofest(self, manofest_path): for target in targets: if target in excluded_targets: self._logger.debug( - 'Exclusion requested. Skipping target', + "Exclusion requested. Skipping target", target=target, excluded_targets=excluded_targets, ) @@ -238,7 +238,7 @@ def _load_targets_from_manofest(self, manofest_path): for member in members: if member in excluded_targets: self._logger.debug( - 'Exclusion requested. Skipping target', + "Exclusion requested. Skipping target", member=member, excluded_targets=excluded_targets, ) @@ -257,7 +257,7 @@ def _load_targets_from_manofest(self, manofest_path): def _load_manofest_module(self, manofest_path): manofest_module = importlib.machinery.SourceFileLoader( - 'manofest', manofest_path + "manofest", manofest_path ).load_module() return manofest_module @@ -291,11 +291,11 @@ def _normalize_target_names_to_cls_names( if skip_missing: self._logger.info( - 'Failed to find target in manofest module. Skipping', target=target + "Failed to find target in manofest module. Skipping", target=target ) else: raise RuntimeError( - 'Failed to find target in manofest module: {0}'.format(target) + "Failed to find target in manofest module: {0}".format(target) ) return cls_names @@ -397,7 +397,7 @@ def _enforce_no_store_true_args(parser): for action in parser._actions: if isinstance(action, argparse._StoreTrueAction): error_msg = ( - 'manofest.py doens\'t support argument registration of' - ' type=\'store_true\' \noffending action={0}'.format(action) + "manofest.py doens't support argument registration of" + " type='store_true' \noffending action={0}".format(action) ) raise SyntaxError(error_msg) diff --git a/core/update_manager.py b/core/update_manager.py index 91f919f..3fce449 100644 --- a/core/update_manager.py +++ b/core/update_manager.py @@ -9,13 +9,13 @@ class UpdateManager(object): def __init__(self, logger, manof_path): - self._logger = logger.get_child('update_manager') + self._logger = logger.get_child("update_manager") self._manof_path = manof_path - self._requirements_path = os.path.join(self._manof_path, 'requirements.txt') + self._requirements_path = os.path.join(self._manof_path, "requirements.txt") @defer.inlineCallbacks def update(self): - sys.stdout.write('Checking for manof updates ... ') + sys.stdout.write("Checking for manof updates ... ") sys.stdout.flush() # try to update by simply pulling whatever branch / remote we're on @@ -23,36 +23,36 @@ def update(self): self._logger, self._manof_path, quiet=True ) if exit_code: - if 'You are not currently on a branch' in err: + if "You are not currently on a branch" in err: sys.stdout.write( - 'Skipping updating manof (checkout to a specific branch first)' + "Skipping updating manof (checkout to a specific branch first)" ) return self._logger.error( - 'Failed to update manof', exit_code=exit_code, err=err, out=out + "Failed to update manof", exit_code=exit_code, err=err, out=out ) - raise RuntimeError('Failed to update manof') + raise RuntimeError("Failed to update manof") # if "up-to-date" was not outputted, this means that we updated - return True in this case - updated = 'up-to-date' not in out + updated = "up-to-date" not in out # if we pulled in new code, make sure our venv has all the packages required by that code if updated: yield self._update_venv() sys.stdout.write( - ('Updated!' if updated else 'Everything up to date') + os.linesep + ("Updated!" if updated else "Everything up to date") + os.linesep ) defer.returnValue(updated) @defer.inlineCallbacks def _update_venv(self): - venv_path = os.path.join(self._manof_path, 'venv') - requirements_path = os.path.join(self._manof_path, 'requirements.txt') + venv_path = os.path.join(self._manof_path, "venv") + requirements_path = os.path.join(self._manof_path, "requirements.txt") self._logger.debug( - 'Updating virtual env', + "Updating virtual env", venv_path=venv_path, requirements_path=requirements_path, ) diff --git a/examples/basic/manofest.py b/examples/basic/manofest.py index f04dcb6..fb75bba 100644 --- a/examples/basic/manofest.py +++ b/examples/basic/manofest.py @@ -21,22 +21,22 @@ def rm_on_run(self): @property def labels(self): return { - 'manofest-class': self.name, + "manofest-class": self.name, } @property def env(self): return [ - {'VERSE': 'Plain talking. Take us so far.'}, + {"VERSE": "Plain talking. Take us so far."}, ] @property def command(self): - return '{0} "echo \'{1}\'"'.format(self.shell_cmd, self.chorus_line) + return "{0} \"echo '{1}'\"".format(self.shell_cmd, self.chorus_line) @property def shell_cmd(self): - raise RuntimeError('Unknown shell') + raise RuntimeError("Unknown shell") @property def chorus_line(self): @@ -46,15 +46,15 @@ def chorus_line(self): class MobyUbuntu(MobyBase): @property def image_name(self): - return 'ubuntu:16.04' + return "ubuntu:16.04" @property def shell_cmd(self): - return '/bin/bash -c' + return "/bin/bash -c" @property def chorus_line(self): - return 'Lift me up, lift me up' + return "Lift me up, lift me up" @property def exposed_ports(self): @@ -65,22 +65,22 @@ def exposed_ports(self): @property def env(self): return super(MobyUbuntu, self).env + [ - {'MY_CUSTOM_ENV': 'VALUE_A'}, + {"MY_CUSTOM_ENV": "VALUE_A"}, ] class MobyAlpine(MobyBase): @property def image_name(self): - return 'alpine:3.7' + return "alpine:3.7" @property def shell_cmd(self): - return '/bin/sh -c' + return "/bin/sh -c" @property def chorus_line(self): - return 'Higher now ama' + return "Higher now ama" @property def exposed_ports(self): @@ -91,14 +91,14 @@ def exposed_ports(self): @property def env(self): return super(MobyAlpine, self).env + [ - {'MY_CUSTOM_ENV': 'VALUE_B'}, + {"MY_CUSTOM_ENV": "VALUE_B"}, ] class ImageA(manof.Image): @property def image_name(self): - return 'ubuntu:16.04' + return "ubuntu:16.04" @property def detach(self): @@ -107,7 +107,7 @@ def detach(self): @property def labels(self): return { - 'my-project': 'custom_image_1', + "my-project": "custom_image_1", } @property @@ -120,25 +120,25 @@ def exposed_ports(self): @property def env(self): return [ - 'MY_ENV_1', - {'MY_ENV_2': 'TARGET_VALUE_1'}, + "MY_ENV_1", + {"MY_ENV_2": "TARGET_VALUE_1"}, ] @property def command(self): - return '/bin/bash -c "echo \'hello manof user\'"' + return "/bin/bash -c \"echo 'hello manof user'\"" class ImageB(ImageA): @classmethod def alias(cls): - return 'imageb' + return "imageb" @property def env(self): return [ - 'MY_ENV_1', - {'MY_ENV_2': 'TARGET_VALUE_2'}, + "MY_ENV_1", + {"MY_ENV_2": "TARGET_VALUE_2"}, ] @@ -149,7 +149,7 @@ def env(self): class VolumeA(manof.NamedVolume): def register_args(self, parser): - parser.add_argument('--node-name', type=str, default='node0') + parser.add_argument("--node-name", type=str, default="node0") @property def prefix(self): @@ -157,20 +157,20 @@ def prefix(self): Here we use the argument --node-name to affect a prefix. This will prefix the actual named-volume name as can be seen using 'docker volume ls' """ - return 'proj_a_{0}_'.format(self._args.node_name) + return "proj_a_{0}_".format(self._args.node_name) @property def labels(self): return { - 'creation_datetime': datetime.datetime.now(pytz.utc).isoformat(), - 'volume_image': self.name, + "creation_datetime": datetime.datetime.now(pytz.utc).isoformat(), + "volume_image": self.name, } class VolumeB(VolumeA): @classmethod def alias(cls): - return 'volb' + return "volb" # @@ -182,8 +182,8 @@ class MyImages(manof.Group): @property def members(self): return [ - 'ImageA', - 'ImageB', + "ImageA", + "ImageB", ] @@ -196,6 +196,6 @@ class MyVolumes(manof.Group): @property def members(self): return [ - 'VolumeA', - 'VolumeB', + "VolumeA", + "VolumeB", ] diff --git a/install b/install index 93a9e10..656d11c 100755 --- a/install +++ b/install @@ -53,7 +53,7 @@ def main(): # (from a local dir if it exists) # "python -m pip install" instead of "pip install" handles a pip # issue where it fails in a long-named dir - run('virtualenv --python=python3 venv && ' + run('python3 -m venv venv && ' 'source venv/bin/activate && ' 'python -m pip install {0} incremental && '.format(local_pip_packages) + 'python -m pip install {0} -r {1}'.format(local_pip_packages, requirements_file)) diff --git a/manof.py b/manof.py index f254eb6..d41688e 100644 --- a/manof.py +++ b/manof.py @@ -11,7 +11,7 @@ def _run(args, known_arg_options): retval = 1 logger = clients.logging.Client( - 'manof', + "manof", initial_severity=args.log_severity, initial_console_severity=args.log_console_severity, initial_file_severity=args.log_file_severity, @@ -43,10 +43,10 @@ def _register_arguments(parser): # main command subparser, to which we'll add subparsers below subparsers = parser.add_subparsers( - dest='command', - title='subcommands', + dest="command", + title="subcommands", description=( - 'To print additional help on a subcommand, run manof --help' + "To print additional help on a subcommand, run manof --help" ), ) @@ -54,213 +54,213 @@ def _register_arguments(parser): clients.logging.Client.register_arguments(parser) parser.add_argument( - '-mp', '--manofest-path', help='Location of manofest.py', default='manofest.py' + "-mp", "--manofest-path", help="Location of manofest.py", default="manofest.py" ) parser.add_argument( - '--num-retries', - help='Set number of retires for push and pull operations after first try fails', + "--num-retries", + help="Set number of retires for push and pull operations after first try fails", type=int, default=2, ) # don't actually run any commands parser.add_argument( - '-dr', - '--dry-run', - help='Don\'t actually run any commands, just log', - action='store_true', + "-dr", + "--dry-run", + help="Don't actually run any commands, just log", + action="store_true", ) parser.add_argument( - '-p', - '--parallel', - action='store', - help='Set how many commands to run in parallel', + "-p", + "--parallel", + action="store", + help="Set how many commands to run in parallel", type=int, ) # update - subparsers.add_parser('update', help='Updates Manof') + subparsers.add_parser("update", help="Updates Manof") # base sub parser base_command_parent_parser = argparse.ArgumentParser(add_help=False) - base_command_parent_parser.add_argument('targets', nargs='+') + base_command_parent_parser.add_argument("targets", nargs="+") base_command_parent_parser.add_argument( - '-e', - '--exclude', - help='Exclude targets when running manof cmd (comma-delimited, no spaces)', - default='', + "-e", + "--exclude", + help="Exclude targets when running manof cmd (comma-delimited, no spaces)", + default="", ) # TODO: Change default to 'docker.io'. Currently default is None for backwards compatibility base_command_parent_parser.add_argument( - '-r', - '--repository', - help='The repository from which images shall be taken from or pushed to', + "-r", + "--repository", + help="The repository from which images shall be taken from or pushed to", default=None, ) # image based commands provision_parent_command = argparse.ArgumentParser(add_help=False) provision_parent_command.add_argument( - '-n', '--no-cache', help='Don\'t use cache images on build', action='store_true' + "-n", "--no-cache", help="Don't use cache images on build", action="store_true" ) provision_parent_command.add_argument( - '--force-rm', + "--force-rm", help=( - 'Image: Always remove intermediate containers. ' - 'NamedVolume: Delete existing before creation' + "Image: Always remove intermediate containers. " + "NamedVolume: Delete existing before creation" ), - action='store_true', + action="store_true", ) provision_parent_command.add_argument( - '-tl', - '--skip-tag-local', + "-tl", + "--skip-tag-local", help=( - 'If no context is given, provision will perform pull and ' - 'skip tagging the image with its local repository (default: False)' + "If no context is given, provision will perform pull and " + "skip tagging the image with its local repository (default: False)" ), - dest='tag_local', - action='store_false', + dest="tag_local", + action="store_false", ) # provision subparsers.add_parser( - 'provision', - help='Build or pull target images', + "provision", + help="Build or pull target images", parents=[base_command_parent_parser, provision_parent_command], ) run_parent_parser = argparse.ArgumentParser(add_help=False) run_parent_parser.add_argument( - '--privileged', - action='store_true', - help='Give extended privileges to these containers', + "--privileged", + action="store_true", + help="Give extended privileges to these containers", ) run_parent_parser.add_argument( - '--device', - help='Add a host device to the containers (can be used multiple times)', - action='append', - dest='devices', + "--device", + help="Add a host device to the containers (can be used multiple times)", + action="append", + dest="devices", ) run_parent_parser.add_argument( - '--device-cgroup-rule', - help='Add a rule to the cgroup allowed devices list (e.g. c 42:* rmw)', + "--device-cgroup-rule", + help="Add a rule to the cgroup allowed devices list (e.g. c 42:* rmw)", ) run_parent_parser.add_argument( - '--device-read-bps', - help='Limit read rate (bytes per second) from a device (e.g. /dev/sda:50mb)', + "--device-read-bps", + help="Limit read rate (bytes per second) from a device (e.g. /dev/sda:50mb)", ) run_parent_parser.add_argument( - '--device-read-iops', - help='Limit read rate (IO per second) from a device (e.g. /dev/sda:50)', + "--device-read-iops", + help="Limit read rate (IO per second) from a device (e.g. /dev/sda:50)", ) run_parent_parser.add_argument( - '--device-write-bps', - help='Limit write rate (bytes per second) to a device (e.g. /dev/sda:50mb)', + "--device-write-bps", + help="Limit write rate (bytes per second) to a device (e.g. /dev/sda:50mb)", ) run_parent_parser.add_argument( - '--device-write-iops', - help='Limit write rate (IO per second) to a device (e.g. /dev/sda:50)', + "--device-write-iops", + help="Limit write rate (IO per second) to a device (e.g. /dev/sda:50)", ) run_parent_parser.add_argument( - '--cap-add', help='Add capability to the container', action='append' + "--cap-add", help="Add capability to the container", action="append" ) run_parent_parser.add_argument( - '--cap-drop', help='Drop capability from the container', action='append' + "--cap-drop", help="Drop capability from the container", action="append" ) run_parent_parser.add_argument( - '-dv', - '--delete-volumes', - help='Image: Delete named_volumes that are used by this image', - action='store_true', + "-dv", + "--delete-volumes", + help="Image: Delete named_volumes that are used by this image", + action="store_true", ) run_parent_parser.add_argument( - '-pco', - '--print-command-only', - help='Will enforce dry run and print the run command only, no logs at all', - action='store_true', + "-pco", + "--print-command-only", + help="Will enforce dry run and print the run command only, no logs at all", + action="store_true", ) run_parent_parser.add_argument( - '-prmd5o', - '--print-run-md5-only', - help='Will print the run command md5 only. no logs at all', - action='store_true', + "-prmd5o", + "--print-run-md5-only", + help="Will print the run command md5 only. no logs at all", + action="store_true", ) # run subparsers.add_parser( - 'run', - help='Run target containers', + "run", + help="Run target containers", parents=[base_command_parent_parser, run_parent_parser], ) # stop stop_command = subparsers.add_parser( - 'stop', help='Stop target containers', parents=[base_command_parent_parser] + "stop", help="Stop target containers", parents=[base_command_parent_parser] ) stop_command.add_argument( - '-t', - '--time', - help='Seconds to wait for stop before killing it (default=10)', + "-t", + "--time", + help="Seconds to wait for stop before killing it (default=10)", type=int, default=10, ) # rm rm_command = subparsers.add_parser( - 'rm', help='Remove targets', parents=[base_command_parent_parser] + "rm", help="Remove targets", parents=[base_command_parent_parser] ) rm_command.add_argument( - '-f', - '--force', - help='Kill targets even if they are running', - action='store_true', + "-f", + "--force", + help="Kill targets even if they are running", + action="store_true", ) rm_command.add_argument( - '-v', - '--volumes', - help='Remove the volumes associated with the container', - action='store_true', + "-v", + "--volumes", + help="Remove the volumes associated with the container", + action="store_true", ) # serialize subparsers.add_parser( - 'serialize', - help='Get a JSON representation of the targets', + "serialize", + help="Get a JSON representation of the targets", parents=[base_command_parent_parser], ) # push push_command = subparsers.add_parser( - 'push', help='Push targets', parents=[base_command_parent_parser] + "push", help="Push targets", parents=[base_command_parent_parser] ) push_command.add_argument( - '-nc', - '--no-cleanup', - help='After pushing, delete the tagged image created to push', - action='store_true', + "-nc", + "--no-cleanup", + help="After pushing, delete the tagged image created to push", + action="store_true", ) # pull pull_parent_parser = argparse.ArgumentParser(add_help=False) pull_parent_parser.add_argument( - '-tl', - '--tag-local', - help='After pulling, tag the image with its local repository', - action='store_true', + "-tl", + "--tag-local", + help="After pulling, tag the image with its local repository", + action="store_true", ) subparsers.add_parser( - 'pull', - help='Pull targets', + "pull", + help="Pull targets", parents=[base_command_parent_parser, pull_parent_parser], ) # lift subparsers.add_parser( - 'lift', - help='Provision and run targets', + "lift", + help="Provision and run targets", parents=[ base_command_parent_parser, provision_parent_command, @@ -292,5 +292,5 @@ def run(): return retval -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(run()) diff --git a/manof/image.py b/manof/image.py index 4324d13..5f5ab05 100644 --- a/manof/image.py +++ b/manof/image.py @@ -13,9 +13,9 @@ class Constants(object): - RUN_COMMAND_MD5_HASH_LABEL_NAME = 'manof.runCommandMD5Hash' + RUN_COMMAND_MD5_HASH_LABEL_NAME = "manof.runCommandMD5Hash" RUN_COMMAND_MD5_HASH_LABEL_VALUE_PLACEHOLDER = ( - '' + "" ) @@ -27,34 +27,34 @@ def provision(self): """ provision_args = [] - if 'no_cache' in self._args and self._args.no_cache: - provision_args.append('--no-cache') + if "no_cache" in self._args and self._args.no_cache: + provision_args.append("--no-cache") - if 'force_rm' in self._args and self._args.force_rm: - provision_args.append('--force-rm') + if "force_rm" in self._args and self._args.force_rm: + provision_args.append("--force-rm") daemon_supports_multiplatform_build = ( yield self._daemon_supports_multiplatform_build() ) if self.platform_architecture and daemon_supports_multiplatform_build: - provision_args.append('--platform={0}'.format(self.platform_architecture)) + provision_args.append("--platform={0}".format(self.platform_architecture)) # if there is a context, do a build if self.context is not None: - command = 'docker build --rm {0} --tag={1} -f {2} {3}'.format( - ' '.join(provision_args), self.image_name, self.dockerfile, self.context + command = "docker build --rm {0} --tag={1} -f {2} {3}".format( + " ".join(provision_args), self.image_name, self.dockerfile, self.context ) # if image provides a programmatic docker ignore, we need to create a temporary # file at the context and remove it when we're done if self.dockerignore is not None: - dockerignore_path = os.path.join(self.context, '.dockerignore') + dockerignore_path = os.path.join(self.context, ".dockerignore") try: # write the docker ignore - with open(dockerignore_path, 'w') as dockerignore_file: - dockerignore_file.write('\n'.join(self.dockerignore)) + with open(dockerignore_path, "w") as dockerignore_file: + dockerignore_file.write("\n".join(self.dockerignore)) # do the build yield self._run_command(command) @@ -78,66 +78,66 @@ def provision(self): @defer.inlineCallbacks def run(self): - self._logger.debug('Running') + self._logger.debug("Running") # remove if self.container_name: yield self.rm(True) - command = 'docker run ' + command = "docker run " # add detach if needed if self.detach: - command += '--detach ' + command += "--detach " # make it interactive if self.interactive: - command += '--interactive ' + command += "--interactive " # allocate a pseudo-tty if self.tty: - command += '--tty ' + command += "--tty " # add rm if needed if self.rm_on_run: - command += '--rm ' + command += "--rm " # add privileged if needed if self.privileged: - command += '--privileged ' + command += "--privileged " if self.pid: - command += '--pid {0} '.format(self.pid) + command += "--pid {0} ".format(self.pid) command = self._add_resource_limit_arguments(command) # add devices for device in self.devices: if device: - command += '--device={0} '.format(device) + command += "--device={0} ".format(device) # add dns if needed, this allowes for the container to resolve addresses using custom dns resolvers for dns_ip in self.dns: - command += '--dns {0} '.format(dns_ip) + command += "--dns {0} ".format(dns_ip) # add net - command += '--net {0} '.format(self.net) + command += "--net {0} ".format(self.net) # add custom hosts to /etc/hosts if self.add_hosts is not None: for hostname, address in self.add_hosts.items(): - command += '--add-host {0}:{1} '.format(hostname, address) + command += "--add-host {0}:{1} ".format(hostname, address) # add log driver if self.log_driver is not None: - command += '--log-driver {0} '.format(self.log_driver) + command += "--log-driver {0} ".format(self.log_driver) # add labels if len(self.labels): for k, v in self.labels.items(): - command += '--label {0}={1} '.format(k, v) + command += "--label {0}={1} ".format(k, v) - command += '--label {0}={1} '.format( + command += "--label {0}={1} ".format( Constants.RUN_COMMAND_MD5_HASH_LABEL_NAME, Constants.RUN_COMMAND_MD5_HASH_LABEL_VALUE_PLACEHOLDER, ) @@ -145,7 +145,7 @@ def run(self): # add user/group if self.user_and_group is not None: user, group = self.user_and_group - command += '--user {0}:{1} '.format(user, group) + command += "--user {0}:{1} ".format(user, group) # add health check related args command += self._generate_healthcheck_args() @@ -153,10 +153,10 @@ def run(self): # add published ports for exposed_port in self.exposed_ports: if isinstance(exposed_port, int): - command += '--publish {0}:{0} '.format(exposed_port) + command += "--publish {0}:{0} ".format(exposed_port) elif isinstance(exposed_port, dict): host_port, container_port = list(exposed_port.items())[0] - command += '--publish {0}:{1} '.format(host_port, container_port) + command += "--publish {0}:{1} ".format(host_port, container_port) # add volumes for volume in self.volumes: @@ -171,10 +171,10 @@ def run(self): else: # if user passed relative path, prefix it with host manofest directory - if not host_path.startswith('/'): + if not host_path.startswith("/"): host_path = os.path.join(self.host_manofest_dir, host_path) - command += '--volume {0}:{1} '.format(host_path, container_path) + command += "--volume {0}:{1} ".format(host_path, container_path) # replace env vars with argument-given ones: for env in self._update_env_override(): @@ -192,34 +192,34 @@ def run(self): elif isinstance(env, dict): lvalue, rvalue = list(env.items())[0] else: - raise RuntimeError('Invalid env') + raise RuntimeError("Invalid env") - command += '--env {0}={1} '.format(lvalue, pipes.quote(str(rvalue))) + command += "--env {0}={1} ".format(lvalue, pipes.quote(str(rvalue))) # set hostname if self.hostname is not None: - command += '--hostname={0} '.format(self.hostname) + command += "--hostname={0} ".format(self.hostname) # set name if self.container_name: - command += '--name {0} '.format(self.container_name) + command += "--name {0} ".format(self.container_name) for cap in self.cap_add: if cap: - command += '--cap-add={0} '.format(cap) + command += "--cap-add={0} ".format(cap) for cap in self.cap_drop: if cap: - command += '--cap-drop={0} '.format(cap) + command += "--cap-drop={0} ".format(cap) # set restart policy if self.restart: - command += '--restart={0} '.format(self.restart) + command += "--restart={0} ".format(self.restart) command = self._add_device_arguments(command) # set tag - command += self.image_name + ' ' + command += self.image_name + " " # if there's a command, append it if self.command is not None: @@ -230,16 +230,16 @@ def run(self): # update command md5 md5 = hashlib.md5() - md5.update(command.encode('utf-8')) + md5.update(command.encode("utf-8")) command_sha = md5.hexdigest() command = command.replace( Constants.RUN_COMMAND_MD5_HASH_LABEL_VALUE_PLACEHOLDER, command_sha ) - if hasattr(self._args, 'print_command_only') and self._args.print_command_only: + if hasattr(self._args, "print_command_only") and self._args.print_command_only: print(command) elif ( - hasattr(self._args, 'print_run_md5_only') and self._args.print_run_md5_only + hasattr(self._args, "print_run_md5_only") and self._args.print_run_md5_only ): print(command_sha) @@ -251,8 +251,8 @@ def run(self): except Exception as exc: dangling_container_error = re.search( - 'endpoint with name (?P.*) already exists in network' - ' (?P.*).', + "endpoint with name (?P.*) already exists in network" + " (?P.*).", str(exc), ) @@ -260,11 +260,11 @@ def run(self): dangling_container_error is not None and self.force_run_with_disconnection ): - container_name = dangling_container_error.group('container_name') - network = dangling_container_error.group('network') + container_name = dangling_container_error.group("container_name") + network = dangling_container_error.group("network") yield self._disconnect_container_from_network(container_name, network) - self._logger.debug('Re-running container', command=command) + self._logger.debug("Re-running container", command=command) yield self._run_command(command) else: @@ -279,9 +279,9 @@ def run(self): @defer.inlineCallbacks def stop(self): - self._logger.debug('Stopping') + self._logger.debug("Stopping") - command = 'docker stop --time={0} '.format(self._args.time) + command = "docker stop --time={0} ".format(self._args.time) command += self.container_name # stop container @@ -290,12 +290,12 @@ def stop(self): @defer.inlineCallbacks def rm(self, force=False): - self._logger.debug('Removing') + self._logger.debug("Removing") - command = 'docker rm ' + command = "docker rm " - if force or (hasattr(self._args, 'force') and self._args.force): - command += '--force ' + if force or (hasattr(self._args, "force") and self._args.force): + command += "--force " command += self.container_name @@ -303,21 +303,21 @@ def rm(self, force=False): yield self._run_command(command, raise_on_error=False) # delete named volumes if asked (After removing containers, because a named_volume in use can't be removed) - if 'volumes' in self._args and self._args.volumes: + if "volumes" in self._args and self._args.volumes: yield self._delete_all_named_volumes() @defer.inlineCallbacks def push(self): if self.skip_push: self._logger.debug( - 'Skipping push', + "Skipping push", image_name=self.image_name, remote_image_name=self.remote_image_name, ) defer.returnValue(None) self._logger.debug( - 'Pushing', + "Pushing", image_name=self.image_name, remote_image_name=self.remote_image_name, skip_push=self.skip_push, @@ -326,36 +326,36 @@ def push(self): # tag and push yield self._run_command( [ - 'docker tag {0} {1}'.format(self.image_name, self.remote_image_name), - 'docker push {0}'.format(self.remote_image_name), + "docker tag {0} {1}".format(self.image_name, self.remote_image_name), + "docker push {0}".format(self.remote_image_name), ] ) if not self._args.no_cleanup: self._logger.debug( - 'Cleaning after push', + "Cleaning after push", image_name=self.image_name, remote_image_name=self.remote_image_name, ) - yield self._run_command('docker rmi {0}'.format(self.remote_image_name)) + yield self._run_command("docker rmi {0}".format(self.remote_image_name)) self.pprint_json( { - 'image_name': self.image_name, - 'remote_image_name': self.remote_image_name, + "image_name": self.image_name, + "remote_image_name": self.remote_image_name, } ) @defer.inlineCallbacks def pull(self): self._logger.debug( - 'Pulling', + "Pulling", remote_image_name=self.remote_image_name, tag_local=self._args.tag_local, ) # first, pull the image - yield self._run_command('docker pull {0}'.format(self.remote_image_name)) + yield self._run_command("docker pull {0}".format(self.remote_image_name)) # tag pulled images with its local repository + name if self._args.tag_local: @@ -363,7 +363,7 @@ def pull(self): @defer.inlineCallbacks def lift(self): - self._logger.debug('Lifting') + self._logger.debug("Lifting") # remove containers and ignore errors (since docker returns error if the container doesn't exist) yield self.provision() @@ -377,38 +377,38 @@ def _add_resource_limit_arguments(self, command): # add memory limit args if self.memory: - command += '--memory {0} '.format(self.memory) + command += "--memory {0} ".format(self.memory) if self.memory_reservation: - command += '--memory-reservation {0} '.format(self.memory_reservation) + command += "--memory-reservation {0} ".format(self.memory_reservation) if self.kernel_memory: - command += '--kernel-memory {0} '.format(self.kernel_memory) + command += "--kernel-memory {0} ".format(self.kernel_memory) if self.memory_swap: - command += '--memory-swap {0} '.format(self.memory_swap) + command += "--memory-swap {0} ".format(self.memory_swap) if self.memory_swappiness: - command += '--memory-swappiness {0} '.format(self.memory_swappiness) + command += "--memory-swappiness {0} ".format(self.memory_swappiness) if self.oom_kill_disable: - command += '--oom-kill-disable ' + command += "--oom-kill-disable " # add cpus limit args if self.cpus: - command += '--cpus {0} '.format(self.cpus) + command += "--cpus {0} ".format(self.cpus) if self.cpu_period: - command += '--cpu-period {0} '.format(self.cpu_period) + command += "--cpu-period {0} ".format(self.cpu_period) if self.cpu_quota: - command += '--cpu-quota {0} '.format(self.cpu_quota) + command += "--cpu-quota {0} ".format(self.cpu_quota) if self.cpuset_cpus: - command += '--cpuset-cpus {0} '.format(self.cpuset_cpus) + command += "--cpuset-cpus {0} ".format(self.cpuset_cpus) if self.cpu_shares: - command += '--cpu-shares {0} '.format(self.cpu_shares) + command += "--cpu-shares {0} ".format(self.cpu_shares) return command @@ -416,23 +416,23 @@ def _add_device_arguments(self, command): # set device cgroup rule if self.device_cgroup_rule: - command += '--device-cgroup-rule={0} '.format(self.device_cgroup_rule) + command += "--device-cgroup-rule={0} ".format(self.device_cgroup_rule) # set device read bps if self.device_read_bps: - command += '--device-read-bps={0} '.format(self.device_read_bps) + command += "--device-read-bps={0} ".format(self.device_read_bps) # set device read iops if self.device_read_iops: - command += '--device-read-iops={0} '.format(self.device_read_iops) + command += "--device-read-iops={0} ".format(self.device_read_iops) # set device write bps if self.device_write_bps: - command += '--device-write-bps={0} '.format(self.device_write_bps) + command += "--device-write-bps={0} ".format(self.device_write_bps) # set device write iops if self.device_write_iops: - command += '--device-write-iops={0} '.format(self.device_write_iops) + command += "--device-write-iops={0} ".format(self.device_write_iops) return command @@ -455,16 +455,16 @@ def context(self): @property def dockerfile(self): if isinstance(self.context, str): - return os.path.join(self.context, 'Dockerfile') + return os.path.join(self.context, "Dockerfile") return None @property def image_name(self): - raise ValueError('{0}: Image name not set'.format(self.name)) + raise ValueError("{0}: Image name not set".format(self.name)) @property def local_repository(self): - raise ValueError('{0}: Local repository not set'.format(self.name)) + raise ValueError("{0}: Local repository not set".format(self.name)) @property def container_name(self): @@ -600,7 +600,7 @@ def rm_on_run(self): @property def privileged(self): - if 'privileged' in self._args: + if "privileged" in self._args: return self._args.privileged is True return False @@ -611,14 +611,14 @@ def pid(self): @property def devices(self): - if 'devices' in self._args and self._args.devices: + if "devices" in self._args and self._args.devices: return self._args.devices return [] @property def net(self): - return 'host' + return "host" @property def force_run_with_disconnection(self): @@ -684,49 +684,49 @@ def dns(self): @property def cap_add(self): - if 'cap_add' in self._args and self._args.cap_add: + if "cap_add" in self._args and self._args.cap_add: return self._args.cap_add return [] @property def cap_drop(self): - if 'cap_drop' in self._args and self._args.cap_drop: + if "cap_drop" in self._args and self._args.cap_drop: return self._args.cap_drop return [] @property def device_cgroup_rule(self): - if 'device_cgroup_rule' in self._args and self._args.device_cgroup_rule: + if "device_cgroup_rule" in self._args and self._args.device_cgroup_rule: return self._args.device_cgroup_rule return None @property def device_read_bps(self): - if 'device_read_bps' in self._args and self._args.device_read_bps: + if "device_read_bps" in self._args and self._args.device_read_bps: return self._args.device_read_bps return None @property def device_read_iops(self): - if 'device_read_iops' in self._args and self._args.device_read_iops: + if "device_read_iops" in self._args and self._args.device_read_iops: return self._args.device_read_iops return None @property def device_write_bps(self): - if 'device_write_bps' in self._args and self._args.device_write_bps: + if "device_write_bps" in self._args and self._args.device_write_bps: return self._args.device_write_bps return None @property def device_write_iops(self): - if 'device_write_iops' in self._args and self._args.device_write_iops: + if "device_write_iops" in self._args and self._args.device_write_iops: return self._args.device_write_iops return None @@ -737,14 +737,14 @@ def restart(self): def to_dict(self): d = super(Image, self).to_dict() - for idx, item in enumerate(d['volumes']): + for idx, item in enumerate(d["volumes"]): if issubclass(type(item), dict): volume = list(item.keys())[0] if self._classname_is_subclass(volume, manof.Volume): # instantiate named_volume = volume(self._logger, self._args) - d['volumes'][idx] = { + d["volumes"][idx] = { named_volume.volume_name: list(item.values())[0] } return d @@ -763,7 +763,7 @@ def _update_env_override(self): value = vars(self._args)[argument] if self.allow_env_args and value: self._logger.debug( - 'Replacing env var from argument', envvar=envvar, value=value + "Replacing env var from argument", envvar=envvar, value=value ) env[idx] = {envvar: value} @@ -772,28 +772,28 @@ def _update_env_override(self): @defer.inlineCallbacks def _tag_local(self): repository = self._determine_repository() - if repository == 'docker.io': + if repository == "docker.io": # docker.io is omitted by default self._logger.debug( - 'Image is already tagged with its local repository', + "Image is already tagged with its local repository", repository=repository, ) defer.returnValue(None) self._logger.debug( - 'Tagging image with local repository', + "Tagging image with local repository", repository=repository, image_name=self.image_name, remote_image_name=self.remote_image_name, ) yield self._run_command( - 'docker tag {0} {1}'.format(self.remote_image_name, self.image_name) + "docker tag {0} {1}".format(self.remote_image_name, self.image_name) ) # Clean repository from image name if provided if self.image_name != self.remote_image_name: - yield self._run_command('docker rmi {0}'.format(self.remote_image_name)) + yield self._run_command("docker rmi {0}".format(self.remote_image_name)) @defer.inlineCallbacks def _ensure_named_volume_exists(self, volume_name): @@ -801,7 +801,7 @@ def _ensure_named_volume_exists(self, volume_name): # instantiate named_volume = volume_name(self._logger, self._args) - if 'delete_volumes' in self._args and self._args.delete_volumes: + if "delete_volumes" in self._args and self._args.delete_volumes: yield named_volume.provision(rm=True) else: yield named_volume.provision(rm=False) @@ -809,7 +809,7 @@ def _ensure_named_volume_exists(self, volume_name): @defer.inlineCallbacks def _delete_all_named_volumes(self): - self._logger.debug('Removing named-volumes') + self._logger.debug("Removing named-volumes") for volume in self.volumes: host_path, container_path = list(volume.items())[0] if self._classname_is_subclass(host_path, manof.NamedVolume): @@ -823,22 +823,22 @@ def _classname_is_subclass(class_name, cls): return inspect.isclass(class_name) and issubclass(class_name, cls) def _generate_healthcheck_args(self): - arg_string = '' + arg_string = "" if self.health_cmd is not None: - arg_string += '--health-cmd=\"{0}\" '.format(self.health_cmd) + arg_string += '--health-cmd="{0}" '.format(self.health_cmd) if self.health_interval is not None: - arg_string += '--health-interval={0} '.format(self.health_interval) + arg_string += "--health-interval={0} ".format(self.health_interval) if self.health_retries is not None: - arg_string += '--health-retries={0} '.format(self.health_retries) + arg_string += "--health-retries={0} ".format(self.health_retries) if self.health_timeout is not None: - arg_string += '--health-timeout={0} '.format(self.health_timeout) + arg_string += "--health-timeout={0} ".format(self.health_timeout) if self.no_healthcheck: - arg_string += '--no-healthcheck ' + arg_string += "--no-healthcheck " return arg_string @@ -852,18 +852,16 @@ def _determine_repository(self): # no repository was determined, use docker's default if repository is None: # TODO: Remove once "default_repository" is set to 'docker.io' - self._logger.debug( - 'No remote repository was given, setting to \"docker.io\"' - ) - repository = 'docker.io' + self._logger.debug('No remote repository was given, setting to "docker.io"') + repository = "docker.io" return repository @defer.inlineCallbacks def _disconnect_container_from_network(self, container_name, network): - self._logger.debug('Disconnecting container from net') + self._logger.debug("Disconnecting container from net") yield self._run_command( - 'docker network disconnect -f {0} {1}'.format(network, container_name), + "docker network disconnect -f {0} {1}".format(network, container_name), raise_on_error=False, ) @@ -872,10 +870,10 @@ def _daemon_supports_multiplatform_build(self): # multiplatform build is not experimental from 20.10.21 out, _, _ = yield self._run_command( - 'docker version --format \'{{.Client.Version}}\'' + "docker version --format '{{.Client.Version}}'" ) try: - if out and semver.Version.parse(out) >= semver.Version.parse('20.10.21'): + if out and semver.Version.parse(out) >= semver.Version.parse("20.10.21"): defer.returnValue(True) except ValueError: @@ -883,6 +881,6 @@ def _daemon_supports_multiplatform_build(self): # There are 2 lines with the key Experimental - one for the server and one for the client. # They both need to be true for the multiplatform build to be supported - out, _, _ = yield self._run_command('docker version | grep Experimental') + out, _, _ = yield self._run_command("docker version | grep Experimental") defer.returnValue("false" not in out) diff --git a/manof/target.py b/manof/target.py index 792f411..eb56f62 100644 --- a/manof/target.py +++ b/manof/target.py @@ -16,7 +16,7 @@ def __init__(self, logger, args): self._manofest_dir = os.path.dirname(self._manofest_path) def add_dependent_target(self, target): - self._logger.debug('Adding dependent target', target=target.name) + self._logger.debug("Adding dependent target", target=target.name) self._dependent_targets.append(target) def register_args(self, parser): @@ -39,14 +39,14 @@ def register_env_args(self, parser): envvar_name = list(env.keys())[0] else: raise RuntimeError( - 'env var not defined as string or dict: {0}'.format(env) + "env var not defined as string or dict: {0}".format(env) ) # register new arg that will override this env var argument = self._to_argument(envvar_name) - self._logger.debug('Registering env arg', argument=argument) + self._logger.debug("Registering env arg", argument=argument) parser.add_argument( - argument, required=False, help='Environment variable population option' + argument, required=False, help="Environment variable population option" ) def update_args(self, args): @@ -71,7 +71,7 @@ def depends_on(self): def to_dict(self): d = {} for attr in dir(self): - if attr.startswith('_'): + if attr.startswith("_"): continue value = getattr(self, attr) @@ -82,7 +82,7 @@ def to_dict(self): ): continue - if attr == 'dependent_targets': + if attr == "dependent_targets": value = [t.name for t in value] d[attr] = value @@ -90,8 +90,8 @@ def to_dict(self): def pprint_json(self, some_object): self._logger.debug( - 'Calling Target.pprint_json is deprecated, use `manof.utils.pprint_json`' - ' instead' + "Calling Target.pprint_json is deprecated, use `manof.utils.pprint_json`" + " instead" ) return manof.utils.pprint_json(some_object) @@ -101,7 +101,7 @@ def env(self): @property def env_prefix(self): - return '' + return "" @property def allow_env_args(self): @@ -110,7 +110,7 @@ def allow_env_args(self): @defer.inlineCallbacks def _run_command(self, command, cwd=None, raise_on_error=True, env=None): self._logger.debug( - 'Running command', + "Running command", command=command, cwd=cwd, raise_on_error=raise_on_error, @@ -119,7 +119,7 @@ def _run_command(self, command, cwd=None, raise_on_error=True, env=None): # combine commands if list if isinstance(command, list): - command = ' && '.join(command) + command = " && ".join(command) # if dry run, do nothing if not self._args.dry_run: @@ -127,7 +127,7 @@ def _run_command(self, command, cwd=None, raise_on_error=True, env=None): command, cwd=cwd, quiet=not raise_on_error, env=env, logger=self._logger ) else: - result = yield '', '', 0 + result = yield "", "", 0 defer.returnValue(result) @@ -138,12 +138,12 @@ def _to_argument(self, envvar, hyphenate=True, arg_prefix=True): if envvar.startswith(self.env_prefix): argument = argument[len(self.env_prefix) :] - argument = '{0}_{1}'.format(self.name, argument).lower() + argument = "{0}_{1}".format(self.name, argument).lower() if hyphenate: - argument = argument.replace('_', '-') + argument = argument.replace("_", "-") if arg_prefix: - argument = '--{0}'.format(argument) + argument = "--{0}".format(argument) return argument diff --git a/manof/utils/__init__.py b/manof/utils/__init__.py index 578e34c..e928599 100644 --- a/manof/utils/__init__.py +++ b/manof/utils/__init__.py @@ -34,18 +34,18 @@ def __init__( self._err = err if code is not None: - message = '\'{0}\' exited with code {1}'.format(command, code) + message = "'{0}' exited with code {1}".format(command, code) else: - message = '\'{0}\' received signal {1}'.format(command, signal) + message = "'{0}' received signal {1}".format(command, signal) if cwd: - message += '\n (cwd: {0})'.format(cwd) + message += "\n (cwd: {0})".format(cwd) if err: - message += '\n (stderr: {0})'.format(err) + message += "\n (stderr: {0})".format(err) if out: - message += '\n (stdout: {0})'.format(out) + message += "\n (stdout: {0})".format(out) super(CommandFailedError, self).__init__(message) @@ -63,35 +63,35 @@ def err(self): def git_pull(logger, path, quiet=False): - logger.debug('Pulling', **locals()) - return shell_run(logger, 'git pull', cwd=path, quiet=quiet) + logger.debug("Pulling", **locals()) + return shell_run(logger, "git pull", cwd=path, quiet=quiet) def shell_run(logger, command, cwd=None, quiet=False, env=None): - logger.debug('Running command', **locals()) + logger.debug("Running command", **locals()) # combine commands if list if isinstance(command, list): - command = ' && '.join(command) + command = " && ".join(command) return execute(command, cwd, quiet, env=env, logger=logger) def ensure_pip_requirements_exist(logger, venv_path, requirement_file_path): - logger.debug('Ensuring pip requirements exist', **locals()) + logger.debug("Ensuring pip requirements exist", **locals()) return venv_run( - logger, venv_path, 'pip install -r {0}'.format(requirement_file_path) + logger, venv_path, "pip install -r {0}".format(requirement_file_path) ) def venv_run(logger, venv_path, command, cwd=None, quiet=False): - logger.debug('Running command in virtualenv', **locals()) + logger.debug("Running command in virtualenv", **locals()) commands = [ - 'source {0}'.format(os.path.join(venv_path, 'bin', 'activate')), + "source {0}".format(os.path.join(venv_path, "bin", "activate")), command, - 'deactivate', + "deactivate", ] return shell_run(logger, commands, cwd, quiet) @@ -170,7 +170,7 @@ def _get_error(failure): _signal = failure.value[2] if logger: logger.warn( - 'Command killed by signal', + "Command killed by signal", command=command, cwd=cwd, out=_out, @@ -180,7 +180,7 @@ def _get_error(failure): if not quiet: if logger: - logger.warn('Command failed') + logger.warn("Command failed") raise CommandFailedError( command=command, cwd=cwd, out=_out, err=_err, signal=_signal ) @@ -188,7 +188,7 @@ def _get_error(failure): return _out, _err, _signal d = getProcessOutputAndValue( - '/bin/bash', args=['-c', command], path=cwd, env=env or os.environ + "/bin/bash", args=["-c", command], path=cwd, env=env or os.environ ) # errback chain is fired if a signal is raised in the process @@ -200,7 +200,7 @@ def _get_error(failure): if code: if quiet and logger: logger.debug( - 'Command failed quietly', + "Command failed quietly", command=command, cwd=cwd, code_or_signal=code, @@ -210,7 +210,7 @@ def _get_error(failure): else: if logger: logger.warn( - 'Command failed', + "Command failed", command=command, cwd=cwd, code_or_signal=code, @@ -222,7 +222,7 @@ def _get_error(failure): ) else: if logger: - logger.info('Command succeeded', command=command, cwd=cwd, out=out, err=err) + logger.info("Command succeeded", command=command, cwd=cwd, out=out, err=err) defer.returnValue((out, err, code)) @@ -230,7 +230,7 @@ def _get_error(failure): @defer.inlineCallbacks def get_running_container_label(target_name, label, logger=None): sha, _, _ = yield execute( - 'docker inspect --format \'{{{{ index .Config.Labels "{0}"}}}}\' {1}'.format( + "docker inspect --format '{{{{ index .Config.Labels \"{0}\"}}}}' {1}".format( label, target_name ), logger=logger, @@ -241,7 +241,7 @@ def get_running_container_label(target_name, label, logger=None): def store_boolean(value): - return True if value == 'true' else False + return True if value == "true" else False @defer.inlineCallbacks @@ -259,7 +259,7 @@ def retry_until_successful(num_of_tries, logger, function, *args, **kwargs): def _on_operation_callback_error(failure): logger.debug( - 'Exception during operation execution', + "Exception during operation execution", function=function.__name__, tb=failure.getBriefTraceback(), ) @@ -278,7 +278,7 @@ def _on_operation_callback_error(failure): except Exception as exc: last_exc = exc logger.warn( - 'Operation failed', + "Operation failed", function=function.__name__, exc=repr(exc), current_try_number=tries, @@ -289,8 +289,8 @@ def _on_operation_callback_error(failure): else: defer.returnValue(result) - last_exc.message = 'Failed to execute command with given retries:\n {0}'.format( - getattr(last_exc, 'message', str(last_exc)) + last_exc.message = "Failed to execute command with given retries:\n {0}".format( + getattr(last_exc, "message", str(last_exc)) ) raise last_exc @@ -298,9 +298,9 @@ def _on_operation_callback_error(failure): def pprint_json(obj: typing.Union[typing.List, typing.Dict]): formatted_json = simplejson.dumps(obj, indent=2) if sys.stdout.isatty(): - json_lexer = pygments.lexers.get_lexer_by_name('Json') + json_lexer = pygments.lexers.get_lexer_by_name("Json") formatter = pygments.formatters.get_formatter_by_name( - 'terminal16m', style='paraiso-dark' + "terminal16m", style="paraiso-dark" ) colorful_json = pygments.highlight(formatted_json, json_lexer, formatter) print(colorful_json) diff --git a/manof/volume.py b/manof/volume.py index 0957181..9cba270 100644 --- a/manof/volume.py +++ b/manof/volume.py @@ -24,7 +24,7 @@ def exists(self): @property def prefix(self): - return '' + return "" @property def volume_name(self): @@ -44,49 +44,49 @@ def provision(self, rm=None): """ if rm is None: - rm = 'force_rm' in self._args and self._args.force_rm + rm = "force_rm" in self._args and self._args.force_rm if rm: yield self.rm(safe=True) - self._logger.info('Creating named-volume', name=self.name) + self._logger.info("Creating named-volume", name=self.name) creation_args = [] if len(self.labels): for k, v in self.labels.items(): - creation_args.append('--label {0}={1}'.format(k, v)) + creation_args.append("--label {0}={1}".format(k, v)) if len(self.options): for k, v in self.options.items(): - creation_args.append('--opt {0}={1}'.format(k, v)) + creation_args.append("--opt {0}={1}".format(k, v)) - command = 'docker volume create {0} --driver={1} --name={2}'.format( - ' '.join(creation_args), self.driver, self.volume_name + command = "docker volume create {0} --driver={1} --name={2}".format( + " ".join(creation_args), self.driver, self.volume_name ) # don't count on idempotency (labels): exists = yield self.exists() if exists: self._logger.debug( - 'Named volume exists. Doing nothing.', named_volume=self.volume_name + "Named volume exists. Doing nothing.", named_volume=self.volume_name ) else: self._logger.debug( - 'Named volume doesn\'t exist. Creating.', named_volume=self.volume_name + "Named volume doesn't exist. Creating.", named_volume=self.volume_name ) yield self._run_command(command) def run(self): - self._logger.info('Running a named-volume is meaningless', name=self.name) + self._logger.info("Running a named-volume is meaningless", name=self.name) def stop(self): - self._logger.info('Stopping a named-volume is meaningless', name=self.name) + self._logger.info("Stopping a named-volume is meaningless", name=self.name) @defer.inlineCallbacks def rm(self, safe=True): """ De facto "docker volume rm" """ - self._logger.info('Removing named-volume') + self._logger.info("Removing named-volume") yield self._lock.acquire() try: @@ -95,7 +95,7 @@ def rm(self, safe=True): if not exists: defer.returnValue(None) - command = 'docker volume rm {0}'.format(self.volume_name) + command = "docker volume rm {0}".format(self.volume_name) # remove volume (fail if doesn't exist) yield self._run_command(command) @@ -104,14 +104,14 @@ def rm(self, safe=True): @defer.inlineCallbacks def lift(self): - self._logger.debug('Lifting') + self._logger.debug("Lifting") # just provision yield self.provision() @defer.inlineCallbacks def exists(self): - command = 'docker volume inspect {0}'.format(self.volume_name) + command = "docker volume inspect {0}".format(self.volume_name) # retcode=0 -> volume exists _, _, retcode = yield self._run_command(command, raise_on_error=False) @@ -120,11 +120,11 @@ def exists(self): @property def prefix(self): - return '' + return "" @property def driver(self): - return 'local' + return "local" @property def options(self): diff --git a/requirements/common.txt b/requirements/common.txt index 9ee18ac..df2d263 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,9 +1,9 @@ inflection==0.5.1 -simplejson==3.17.6 -Twisted==22.10.0 -pytz==2022.5 -colorama==0.4.5 -pygments==2.15.0 -ruamel.yaml==0.17.21 -setuptools==67.3.3 -semver==3.0.0 +simplejson==3.19.3 +Twisted==24.10.0 +pytz==2024.2 +colorama==0.4.6 +pygments==2.18.0 +ruamel.yaml==0.18.6 +setuptools==75.5.0 +semver==3.0.2 diff --git a/requirements/dev.txt b/requirements/dev.txt index 6483f28..184d8c7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,3 +1,3 @@ -mock==4.0.3 +mock==5.1.0 nose==1.3.7 -black==22.10.0 +black==24.10.0 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 3a0ab81..a973829 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -10,7 +10,7 @@ import core import clients.logging -logger = clients.logging.TestingClient('integration_test').logger +logger = clients.logging.TestingClient("integration_test").logger class IntegrationTestCase(unittest.TestCase): @@ -23,14 +23,14 @@ def __init__(self, *args, **kwargs): os.path.dirname( os.path.realpath(sys.modules[self.__class__.__module__].__file__) ), - 'artifacts', + "artifacts", ) - self._manofest_file_name = 'manofest.py' + self._manofest_file_name = "manofest.py" @defer.inlineCallbacks def setUp(self): self._logger = logger.get_child(self.name) - self._logger.info('Setting up integration test') + self._logger.info("Setting up integration test") yield defer.maybeDeferred(self.set_up) @@ -51,10 +51,10 @@ def name(self): @defer.inlineCallbacks def _remove_docker_container(self, docker_container, quiet=True, cwd=None): self._logger.debug( - 'Removing docker container', docker_container=docker_container + "Removing docker container", docker_container=docker_container ) yield manof.utils.execute( - 'docker rm -f {0}'.format(docker_container), + "docker rm -f {0}".format(docker_container), cwd=cwd, quiet=quiet, logger=self._logger, @@ -62,18 +62,18 @@ def _remove_docker_container(self, docker_container, quiet=True, cwd=None): @defer.inlineCallbacks def _remove_docker_image(self, docker_image, quiet=True, cwd=None): - self._logger.debug('Removing docker image', docker_image=docker_image) + self._logger.debug("Removing docker image", docker_image=docker_image) yield manof.utils.execute( - 'docker rmi -f {0}'.format(docker_image), + "docker rmi -f {0}".format(docker_image), cwd=cwd, quiet=quiet, logger=self._logger, ) def _get_manof_image(self, image_name): - manofest_path = os.path.join(self._working_dir, 'manofest.py') + manofest_path = os.path.join(self._working_dir, "manofest.py") manofest_module = importlib.machinery.SourceFileLoader( - 'manofest', manofest_path + "manofest", manofest_path ).load_module() return getattr(manofest_module, image_name)(self._logger, mock.MagicMock()) @@ -82,7 +82,7 @@ class ManofIntegrationTestCase(IntegrationTestCase): @defer.inlineCallbacks def setUp(self): self._logger = logger.get_child(self.name) - self._logger.info('Setting up integration test') + self._logger.info("Setting up integration test") self._manof_args = mock.MagicMock() self._manof_known_args = mock.MagicMock() @@ -99,7 +99,7 @@ def manof_args(self): def _load_manofest_targets(self, *targets): self._logger.debug( - 'Loading test manofest', + "Loading test manofest", manofest_file_name=self._manofest_file_name, targets=targets, ) diff --git a/tests/integration/cases/basic/artifacts/manofest.py b/tests/integration/cases/basic/artifacts/manofest.py index 1ce237f..3d38621 100644 --- a/tests/integration/cases/basic/artifacts/manofest.py +++ b/tests/integration/cases/basic/artifacts/manofest.py @@ -8,8 +8,8 @@ class LoadTestImage(manof.Image): class PullTestImage(manof.Image): @property def image_name(self): - return 'busybox:1' + return "busybox:1" @property def command(self): - return '/bin/sh -c "echo \'hello manof user\'"' + return "/bin/sh -c \"echo 'hello manof user'\"" diff --git a/tests/integration/cases/basic/test_basic_commands.py b/tests/integration/cases/basic/test_basic_commands.py index 1af5231..4e73c5a 100644 --- a/tests/integration/cases/basic/test_basic_commands.py +++ b/tests/integration/cases/basic/test_basic_commands.py @@ -8,36 +8,36 @@ class BasicCommandsTestCase(tests.integration.ManofIntegrationTestCase): def manof_args(self): return { - 'parallel': None, - 'repository': None, - 'tag_local': None, - 'dry_run': False, + "parallel": None, + "repository": None, + "tag_local": None, + "dry_run": False, } def test_load_manofest(self): - self._logger.info('Testing load manofest happy flow') - expected_dependent_targets = ['load_test_image'] + self._logger.info("Testing load manofest happy flow") + expected_dependent_targets = ["load_test_image"] # load manofest - manofest = self._load_manofest_targets('LoadTestImage') + manofest = self._load_manofest_targets("LoadTestImage") self.assertEqual(type(manofest), core.RootTarget) self.assertEqual( - manofest.to_dict()['dependent_targets'], expected_dependent_targets + manofest.to_dict()["dependent_targets"], expected_dependent_targets ) @defer.inlineCallbacks def test_pull_images(self): - self._logger.info('Testing pull images happy flow') + self._logger.info("Testing pull images happy flow") # load the integration test manofest - manofest = self._load_manofest_targets('PullTestImage') + manofest = self._load_manofest_targets("PullTestImage") docker_image = manofest.dependent_targets[0].image_name # sanity - removing image if exists - self._logger.debug('Removing docker image', docker_image=docker_image) + self._logger.debug("Removing docker image", docker_image=docker_image) yield manof.utils.execute( - 'docker rmi -f {0}'.format(docker_image), + "docker rmi -f {0}".format(docker_image), cwd=None, quiet=True, logger=self._logger, @@ -48,7 +48,7 @@ def test_pull_images(self): # check the image exists yield manof.utils.execute( - 'docker image history -Hq {0}'.format(docker_image), + "docker image history -Hq {0}".format(docker_image), cwd=None, quiet=False, logger=self._logger, diff --git a/tests/integration/cases/shell_commands/artifacts/manofest.py b/tests/integration/cases/shell_commands/artifacts/manofest.py index bf30ed4..58e8b6d 100644 --- a/tests/integration/cases/shell_commands/artifacts/manofest.py +++ b/tests/integration/cases/shell_commands/artifacts/manofest.py @@ -18,39 +18,39 @@ def env(self): @property def local_repository(self): - return '' + return "" @property def image_name(self): - return 'busybox:1' + return "busybox:1" @property def command(self): - return '/bin/sh -c "echo \'{0}\' && sleep infinity"'.format(self.name) + return "/bin/sh -c \"echo '{0}' && sleep infinity\"".format(self.name) class TestImage2(TestImage): @property def restart(self): - return 'on-failure:5' + return "on-failure:5" @property def memory(self): - return '6Mib' + return "6Mib" @property def cpus(self): - return '1' + return "1" @property def cap_add(self): - return ['SYS_ADMIN'] + return ["SYS_ADMIN"] class SomeGroup(manof.Group): @property def members(self): return [ - 'TestImage', - 'TestImage2', + "TestImage", + "TestImage2", ] diff --git a/tests/integration/cases/shell_commands/test_shell_commands.py b/tests/integration/cases/shell_commands/test_shell_commands.py index 06f0186..cc03b96 100644 --- a/tests/integration/cases/shell_commands/test_shell_commands.py +++ b/tests/integration/cases/shell_commands/test_shell_commands.py @@ -10,99 +10,99 @@ class BasicCommandsTestCase(tests.integration.IntegrationTestCase): @defer.inlineCallbacks def test_serialize(self): serialized_group_contents, _, _ = yield self._execute_manof_command( - '--log-console-severity E serialize', + "--log-console-severity E serialize", [ - 'SomeGroup', + "SomeGroup", ], ) serialized_group = simplejson.loads(serialized_group_contents) - self.assertEqual('test_image', serialized_group[0]['name']) + self.assertEqual("test_image", serialized_group[0]["name"]) @defer.inlineCallbacks def test_run_verify_md5(self): - self._logger.info('Testing run verify md5') - target_name = 'test_image' + self._logger.info("Testing run verify md5") + target_name = "test_image" label_name = manof.image.Constants.RUN_COMMAND_MD5_HASH_LABEL_NAME # run twice to ensure md5 won't change between runs for _ in range(2): - yield self._execute_manof_command('run', ['--dummy', 'do', target_name]) + yield self._execute_manof_command("run", ["--dummy", "do", target_name]) command_sha = yield manof.utils.get_running_container_label( target_name, label_name, self._logger ) - self.assertEqual('4a738101122b28baae05fac7a5dc6b32', command_sha) + self.assertEqual("4a738101122b28baae05fac7a5dc6b32", command_sha) run_md5, _, _ = yield self._execute_manof_command( - 'run', ['--print-run-md5-only', '--dummy', 'do', target_name] + "run", ["--print-run-md5-only", "--dummy", "do", target_name] ) - self.assertEqual('4a738101122b28baae05fac7a5dc6b32', run_md5) + self.assertEqual("4a738101122b28baae05fac7a5dc6b32", run_md5) # run again and make ensure md5 has changed due to "--dummy" value change - yield self._execute_manof_command('run', ['--dummy', 'value', target_name]) + yield self._execute_manof_command("run", ["--dummy", "value", target_name]) command_sha = yield manof.utils.get_running_container_label( target_name, label_name, self._logger ) run_md5, _, _ = yield self._execute_manof_command( - 'run', ['--print-run-md5-only', '--dummy', 'value', target_name] + "run", ["--print-run-md5-only", "--dummy", "value", target_name] ) - self.assertEqual('a3ada1db9e167a8a747c8ddd4de63757', command_sha) - self.assertEqual('a3ada1db9e167a8a747c8ddd4de63757', run_md5) + self.assertEqual("a3ada1db9e167a8a747c8ddd4de63757", command_sha) + self.assertEqual("a3ada1db9e167a8a747c8ddd4de63757", run_md5) # different dummy data yields different run md5 run_md5, _, _ = yield self._execute_manof_command( - 'run', ['--print-run-md5-only', '--dummy', 'value2', target_name] + "run", ["--print-run-md5-only", "--dummy", "value2", target_name] ) self.assertNotEqual(run_md5, command_sha) @defer.inlineCallbacks def test_run_verify_md5(self): - self._logger.info('Testing run verify md5') - target_name = 'test_image' + self._logger.info("Testing run verify md5") + target_name = "test_image" label_name = manof.image.Constants.RUN_COMMAND_MD5_HASH_LABEL_NAME # run twice to ensure md5 won't change between runs for _ in range(2): - yield self._execute_manof_command('run', ['--dummy', 'do', target_name]) + yield self._execute_manof_command("run", ["--dummy", "do", target_name]) command_sha = yield manof.utils.get_running_container_label( target_name, label_name, self._logger ) run_md5, _, _ = yield self._execute_manof_command( - 'run', ['--print-run-md5-only', '--dummy', 'do', target_name] + "run", ["--print-run-md5-only", "--dummy", "do", target_name] ) self.assertEqual(command_sha, run_md5) # run again and make ensure md5 has changed due to "--dummy" value change - yield self._execute_manof_command('run', ['--dummy', 'else', target_name]) + yield self._execute_manof_command("run", ["--dummy", "else", target_name]) command_sha = yield manof.utils.get_running_container_label( target_name, label_name, self._logger ) run_md5, _, _ = yield self._execute_manof_command( - 'run', ['--print-run-md5-only', '--dummy', 'else', target_name] + "run", ["--print-run-md5-only", "--dummy", "else", target_name] ) self.assertEqual(command_sha, run_md5) # different dummy data yields different run md5 run_md5, _, _ = yield self._execute_manof_command( - 'run', ['--print-run-md5-only', '--dummy', 'else2', target_name] + "run", ["--print-run-md5-only", "--dummy", "else2", target_name] ) self.assertNotEqual(run_md5, command_sha) @defer.inlineCallbacks def test_image_run_and_rm(self): - self._logger.info('Testing run command happy flow') + self._logger.info("Testing run command happy flow") - for image_name in ['test_image', 'test_image2']: + for image_name in ["test_image", "test_image2"]: # sanity - removing containers if exists yield self._remove_docker_container(image_name) # run the image using manof - yield self._execute_manof_command('run', [image_name]) + yield self._execute_manof_command("run", [image_name]) # check the container exists docker_log_output, _, _ = yield manof.utils.execute( - 'docker logs {0}'.format(image_name), + "docker logs {0}".format(image_name), cwd=None, quiet=False, logger=self._logger, @@ -111,12 +111,12 @@ def test_image_run_and_rm(self): self.assertEqual(docker_log_output, image_name) # remove the container using manof - yield self._execute_manof_command('rm', [image_name, '--force']) + yield self._execute_manof_command("rm", [image_name, "--force"]) # check the container doesn't exist exists yield self.assertFailure( manof.utils.execute( - 'docker logs {0}'.format(image_name), + "docker logs {0}".format(image_name), cwd=None, quiet=False, logger=self._logger, @@ -126,11 +126,11 @@ def test_image_run_and_rm(self): @defer.inlineCallbacks def test_image_provision(self): - self._logger.info('Testing provision images happy flow') + self._logger.info("Testing provision images happy flow") for image_name, class_name in [ - ('test_image', 'TestImage'), - ('test_image2', 'TestImage2'), + ("test_image", "TestImage"), + ("test_image2", "TestImage2"), ]: manof_image = self._get_manof_image(class_name) docker_image = manof_image.image_name @@ -139,11 +139,11 @@ def test_image_provision(self): yield self._remove_docker_image(docker_image) # provision the image using manof - yield self._execute_manof_command('provision', [image_name]) + yield self._execute_manof_command("provision", [image_name]) # check the image exists yield manof.utils.execute( - 'docker image history -Hq {0}'.format(docker_image), + "docker image history -Hq {0}".format(docker_image), cwd=None, quiet=False, logger=self._logger, @@ -151,11 +151,11 @@ def test_image_provision(self): @defer.inlineCallbacks def test_image_lift(self): - self._logger.info('Testing provision images happy flow') + self._logger.info("Testing provision images happy flow") for image_name, class_name in [ - ('test_image', 'TestImage'), - ('test_image2', 'TestImage2'), + ("test_image", "TestImage"), + ("test_image2", "TestImage2"), ]: manof_image = self._get_manof_image(class_name) docker_image = manof_image.image_name @@ -165,11 +165,11 @@ def test_image_lift(self): yield self._remove_docker_image(docker_image) # provision the image using manof - yield self._execute_manof_command('lift', [image_name]) + yield self._execute_manof_command("lift", [image_name]) # check the image exists yield manof.utils.execute( - 'docker image history -Hq {0}'.format(docker_image), + "docker image history -Hq {0}".format(docker_image), cwd=None, quiet=False, logger=self._logger, @@ -177,7 +177,7 @@ def test_image_lift(self): # check the container exists docker_log_output, _, _ = yield manof.utils.execute( - 'docker logs {0}'.format(image_name), + "docker logs {0}".format(image_name), cwd=None, quiet=False, logger=self._logger, @@ -188,7 +188,7 @@ def test_image_lift(self): @defer.inlineCallbacks def _execute_manof_command(self, command, args): out, err, signal = yield manof.utils.execute( - 'manof {command} {args}'.format(command=command, args=' '.join(args)), + "manof {command} {args}".format(command=command, args=" ".join(args)), cwd=self._working_dir, quiet=False, logger=self._logger, diff --git a/tests/unit/test_manof.py b/tests/unit/test_manof.py index 6ca468a..cba9411 100644 --- a/tests/unit/test_manof.py +++ b/tests/unit/test_manof.py @@ -7,13 +7,13 @@ import manof.image import clients.logging -logger = clients.logging.TestingClient('unit_test').logger +logger = clients.logging.TestingClient("unit_test").logger class ManofUnitTestCase(unittest.TestCase): def setUp(self): self._logger = logger - self._logger.info('Setting up unit test') + self._logger.info("Setting up unit test") @property def name(self): @@ -21,14 +21,14 @@ def name(self): @defer.inlineCallbacks def test_lift(self): - self._logger.info('Testing manof lift') + self._logger.info("Testing manof lift") image = self._create_manof_image( image_properties={ - 'image_name': 'test_image', - 'dockerignore': None, - 'context': None, + "image_name": "test_image", + "dockerignore": None, + "context": None, }, - image_args={'repository': None, 'tag_local': None}, + image_args={"repository": None, "tag_local": None}, ) yield manof.Image.lift(image) @@ -39,54 +39,54 @@ def test_lift(self): @defer.inlineCallbacks def test_provision_pull(self): - self._logger.info('Testing manof provision with pull') + self._logger.info("Testing manof provision with pull") image = self._create_manof_image( image_properties={ - 'image_name': 'test_image', - 'dockerignore': None, - 'context': None, + "image_name": "test_image", + "dockerignore": None, + "context": None, }, - image_args={'repository': None, 'tag_local': None}, + image_args={"repository": None, "tag_local": None}, ) - self._logger.debug('Calling image provisioning') + self._logger.debug("Calling image provisioning") yield manof.Image.provision(image) - self._logger.debug('Checking pull method has been called') + self._logger.debug("Checking pull method has been called") image.pull.assert_called_once() @defer.inlineCallbacks def test_provision_build(self): - self._logger.info('Testing manof provision with build') + self._logger.info("Testing manof provision with build") image = self._create_manof_image( image_properties={ - 'image_name': 'test_image', - 'dockerignore': None, - 'context': 'test_image', - 'dockerfile': 'test_image/Dockerfile', + "image_name": "test_image", + "dockerignore": None, + "context": "test_image", + "dockerfile": "test_image/Dockerfile", } ) - self._logger.debug('Calling image provisioning') + self._logger.debug("Calling image provisioning") yield manof.Image.provision(image) - self._logger.debug('Checking pull method has\'nt been called') + self._logger.debug("Checking pull method has'nt been called") self.assertFalse(image.pull.called) command = image._run_command.call_args.args[0] self._logger.debug( - 'Checking _run_command method has been called with a docker build command', + "Checking _run_command method has been called with a docker build command", command=command, ) - self.assertSubstring('docker build', command) + self.assertSubstring("docker build", command) def _create_manof_image(self, image_properties, image_args=None): - self._logger.debug('Creating test image mock') + self._logger.debug("Creating test image mock") image = mock.Mock(manof.Image) image._logger = self._logger - self._logger.debug('Setting mocked image args', args=image_args) + self._logger.debug("Setting mocked image args", args=image_args) manof_args = mock.MagicMock() if image_args is not None: for attr, val in image_args.items(): @@ -97,7 +97,7 @@ def _create_manof_image(self, image_properties, image_args=None): image._manofest_dir = os.path.dirname(image._manofest_path) self._logger.debug( - 'Setting mocked image properties', properties=image_properties + "Setting mocked image properties", properties=image_properties ) for property_name, property_val in image_properties.items(): setattr(image, property_name, property_val)