From a35001558eec432c5d86418d28e971122633b331 Mon Sep 17 00:00:00 2001 From: omesser Date: Fri, 29 Jan 2021 04:20:05 +0200 Subject: [PATCH] Initial --- .flake8 | 22 + .github/workflows/CI.yaml | 28 + .gitignore | 11 + .idea/.gitignore | 8 + .idea/dockerregistrypusher.iml | 11 + .idea/inspectionProfiles/Project_Default.xml | 42 ++ .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + LICENSE | 21 + Makefile | 52 ++ README.md | 43 ++ clients/logging/__init__.py | 517 ++++++++++++++++++ core/__init__.py | 2 + core/manifest_creator.py | 43 ++ core/registry.py | 287 ++++++++++ dockerregistrypusher | 32 ++ dockerregistrypusher.py | 128 +++++ install | 72 +++ pyproject.toml | 10 + requirements.txt | 1 + requirements/common.txt | 6 + requirements/dev.txt | 5 + requirements_all.txt | 2 + tests/pusher_tests.py | 95 ++++ .../flake8_igz.egg-info/PKG-INFO | 11 + .../flake8_igz.egg-info/SOURCES.txt | 8 + .../flake8_igz.egg-info/dependency_links.txt | 1 + .../flake8_igz.egg-info/entry_points.txt | 8 + .../flake8_igz.egg-info/requires.txt | 1 + .../flake8_igz.egg-info/top_level.txt | 1 + tools/flake8_plugin/flake8_igz.py | 99 ++++ tools/flake8_plugin/setup.py | 35 ++ upload.sh | 1 + 34 files changed, 1621 insertions(+) create mode 100644 .flake8 create mode 100644 .github/workflows/CI.yaml create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/dockerregistrypusher.iml create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 clients/logging/__init__.py create mode 100755 core/__init__.py create mode 100644 core/manifest_creator.py create mode 100644 core/registry.py create mode 100755 dockerregistrypusher create mode 100644 dockerregistrypusher.py create mode 100755 install create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 requirements/common.txt create mode 100644 requirements/dev.txt create mode 100644 requirements_all.txt create mode 100644 tests/pusher_tests.py create mode 100644 tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO create mode 100644 tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt create mode 100644 tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt create mode 100644 tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt create mode 100644 tools/flake8_plugin/flake8_igz.egg-info/requires.txt create mode 100644 tools/flake8_plugin/flake8_igz.egg-info/top_level.txt create mode 100644 tools/flake8_plugin/flake8_igz.py create mode 100644 tools/flake8_plugin/setup.py create mode 100644 upload.sh diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..40e8d31 --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] + +# no lines over 120 allowed +max-line-length = 120 + +# exclude these dirs +exclude = .git,venv + +# ignore the following rules +ignore = + + # whitespace before ':' + E203 + + # line break before binary operator + W503 + + # F401 unused imports (TODO: unignore and use flake8-putty to allow this for 'models' when it supports flake8 v3) + F401 + + # bare except + E722 diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml new file mode 100644 index 0000000..ca90839 --- /dev/null +++ b/.github/workflows/CI.yaml @@ -0,0 +1,28 @@ +name: CI + +on: + push: + branches: + - development + - master + pull_request: + branches: + - development + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - name: Install dependencies + run: make install-ci + + - name: Run lint + run: make lint diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d014a0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# virtualenv +.venv +venv/ +ENV/ + +# pycharm proj +.idea diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..73f69e0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/dockerregistrypusher.iml b/.idea/dockerregistrypusher.iml new file mode 100644 index 0000000..757717e --- /dev/null +++ b/.idea/dockerregistrypusher.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..11fe9bf --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,42 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..5509b4d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..c29154e --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7ee6e46 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Adam Raźniewski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..710deb5 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +VENV_PYTHON = ./venv/bin/python + +.PHONY: all +all: venv lint + @echo Done. + +.PHONY: lint +lint: flake8 fmt-check + +.PHONY: flake8 +flake8: + @echo "Running flake8 lint..." + $(VENV_PYTHON) -m flake8 . + +.PHONY: fmt +fmt: + @echo "Running black fmt..." + $(VENV_PYTHON) -m black --skip-string-normalization . + +.PHONY: fmt-check +fmt-check: + @echo "Running black fmt check..." + $(VENV_PYTHON) -m black --skip-string-normalization --check --diff . + +#.PHONY: test +#test: test-unit test-integ +# @echo Finished running Tests +# +#.PHONY: test-unit +#test-unit: venv +# $(VENV_PYTHON) -m nose tests/unit +# +#.PHONY: test-integ +#test-integ: venv +# $(VENV_PYTHON) -m nose tests/integration/cases/ + +.PHONY: install +install: venv + @echo Installed + +.PHONY: install-ci +install-ci: install-venv install + +.PHONY: venv +venv: + python ./install --dev + $(VENV_PYTHON) -m pip install -e ./tools/flake8_plugin + +.PHONY: install-venv +install-venv: + python -m pip install --upgrade pip + python -m pip install virtualenv diff --git a/README.md b/README.md new file mode 100644 index 0000000..9d0134a --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# What is this? +This package contains dockerregistrypusher CLI that allows to push image packed as tar (usually from docker save command) to a docker registry +This project was forked from [Adam Raźniewski's dockerregistrypusher](https://github.com/Razikus/dockerregistrypusher) but with changes and adjustments to iguazio's needs as a CLI
+All rights reserved to the original author [Adam](https://github.com/Razikus) + +# Why? +To push tar-packed image archives (created by `docker save`) to registries without going through (and taxing) docker-daemon + +Usage of CLI: + +# installation + +Install and create a symlink at `/usr/local/bin/dockerregistrypusher` (requires sudo) +```shell +./install +``` + +Or, install without symlink creation (no elevated permissions needed) +```shell +./install --no-link +``` + +# Running the CLI + +CLI structure +```shell +dockerregistrypusher [options] {TAR_PATH} {REGISTRY_URL} +``` + +For further help (duh) +```shell +dockerregistrypusher --help +``` + + +# Development +To be able to run linting / formatting and other `make` goodness, install with dev requirements +```shell +make install +``` + +# License +Free to use (MIT) diff --git a/clients/logging/__init__.py b/clients/logging/__init__.py new file mode 100644 index 0000000..8bf0ef2 --- /dev/null +++ b/clients/logging/__init__.py @@ -0,0 +1,517 @@ +import sys +import errno +import logging +import logging.handlers +import simplejson +import datetime +import textwrap +import os + +import colorama +import pygments +import pygments.formatters +import pygments.lexers + + +def make_dir_recursively(path): + """ + Create a directory in a location if it doesn't exist + + :param path: The path to create + """ + if not os.path.exists(path): + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +class Record(logging.LogRecord): + pass + + +class Severity(object): + + Verbose = 5 + Debug = logging.DEBUG + Info = logging.INFO + Warning = logging.WARNING + Error = logging.ERROR + + string_enum_dict = { + 'verbose': Verbose, + 'debug': Debug, + 'info': Info, + 'warn': Warning, + 'warning': Warning, + 'error': Error, + # Allow abbreviations + # Also provides backwards compatibility with log-console/file-severity syntax + 'V': Verbose, + 'D': Debug, + 'I': Info, + 'W': Warning, + 'E': Error, + } + + user_presentable_severities = ['verbose', 'debug', 'info', 'warn', 'error'] + + @staticmethod + def get_level_by_string(severity_string): + return Severity.string_enum_dict.get(severity_string, 0) + + +class _VariableLogging(logging.Logger): + + get_child = logging.Logger.getChild + + def __init__(self, name, level=logging.NOTSET): + logging.Logger.__init__(self, name, level) + self._bound_variables = {} + + # each time Logger.get_child is called, the Logger manager creates + # a new Logger instance and adds it to his list + # so we need to add the first error to the manager attributes + # so we can keep the first error in the whole application + if not hasattr(self.manager, 'first_error'): + setattr(self.manager, 'first_error', None) + + @property + def first_error(self): + return self.manager.first_error + + def clear_first_error(self): + if hasattr(self.manager, 'first_error'): + self.manager.first_error = None + + def _check_and_log(self, level, msg, args, kw_args): + if self.isEnabledFor(level): + kw_args.update(self._bound_variables) + self._log(level, msg, args, extra={'vars': kw_args}) + + def error(self, msg, *args, **kw_args): + if self.manager.first_error is None: + self.manager.first_error = {'msg': msg, 'args': args, 'kw_args': kw_args} + + self._check_and_log(Severity.Error, msg, args, kw_args) + + def warn(self, msg, *args, **kw_args): + self._check_and_log(Severity.Warning, msg, args, kw_args) + + def info(self, msg, *args, **kw_args): + self._check_and_log(Severity.Info, msg, args, kw_args) + + def debug(self, msg, *args, **kw_args): + self._check_and_log(Severity.Debug, msg, args, kw_args) + + def verbose(self, msg, *args, **kw_args): + self._check_and_log(Severity.Verbose, msg, args, kw_args) + + def log_and_raise(self, severity, error_msg, *args, **kwargs): + getattr(self, severity)(error_msg, *args, **kwargs) + + # format the exception into the raised error message if we got one + if 'exc' in kwargs: + error_msg = '{0}: {1}'.format(error_msg, kwargs['exc'].lower()) + + exception_type = kwargs.get('exc_type', RuntimeError) + + raise exception_type(error_msg) + + def bind(self, **kw_args): + self._bound_variables.update(kw_args) + + +class ObjectEncoder(simplejson.JSONEncoder): + def default(self, obj): + try: + return obj.__log__() + except: + try: + return obj.__repr__() + except: + return str(obj) + + +class _JsonFormatter(logging.Formatter): + @staticmethod + def format_to_json_str(params): + try: + + # default encoding is utf8 + return simplejson.dumps(params, cls=ObjectEncoder) + except: + + # this is the widest complementary encoding found + return simplejson.dumps( + params, cls=ObjectEncoder, encoding='raw_unicode_escape' + ) + + def format(self, record): + params = { + 'datetime': self.formatTime(record, self.datefmt), + 'name': record.name, + 'level': record.levelname.lower(), + 'message': record.getMessage(), + } + + params.update(record.vars) + + return _JsonFormatter.format_to_json_str(params) + + +class HumanReadableFormatter(logging.Formatter): + def __init__(self, enable_colors, *args, **kwargs): + super(logging.Formatter, self).__init__(*args, **kwargs) + self._enable_colors = enable_colors + + # Maps severity to its letter representation + _level_to_short_name = { + Severity.Verbose: 'V', + Severity.Debug: 'D', + Severity.Info: 'I', + Severity.Warning: 'W', + Severity.Error: 'E', + } + + # Maps severity to its color representation + _level_to_color = { + Severity.Info: colorama.Fore.LIGHTGREEN_EX, + Severity.Warning: colorama.Fore.LIGHTYELLOW_EX, + Severity.Error: colorama.Fore.LIGHTRED_EX, + } + + def format(self, record): + def _get_what_color(): + return { + Severity.Verbose: colorama.Fore.LIGHTCYAN_EX, + Severity.Debug: colorama.Fore.LIGHTCYAN_EX, + Severity.Info: colorama.Fore.CYAN, + Severity.Warning: colorama.Fore.LIGHTCYAN_EX, + Severity.Error: colorama.Fore.LIGHTCYAN_EX, + }.get(record.levelno, colorama.Fore.LIGHTCYAN_EX) + + # coloured using pygments + if self._enable_colors: + more = self._prettify_output(record.vars) if len(record.vars) else '' + else: + try: + more = simplejson.dumps(record.vars) if len(record.vars) else '' + + # defensive + except Exception as exc: + more = simplejson.dumps({'Log formatting error': str(exc)}) + + output = { + 'reset_color': colorama.Fore.RESET, + 'when': datetime.datetime.fromtimestamp(record.created).strftime( + '%d.%m.%y %H:%M:%S.%f' + ), + 'when_color': colorama.Fore.LIGHTYELLOW_EX, + 'who': record.name[-15:], + 'who_color': colorama.Fore.WHITE, + 'severity': HumanReadableFormatter._level_to_short_name[record.levelno], + 'severity_color': HumanReadableFormatter._level_to_color.get( + record.levelno, colorama.Fore.RESET + ), + 'what': record.getMessage(), + 'what_color': _get_what_color(), + 'more': more, + } + + # Slice ms to be at maximum of 3 digits + try: + time_parts = output['when'].split('.') + time_parts[-1] = time_parts[-1][:-3] + output['when'] = '.'.join(time_parts) + except: + pass + + # Disable coloring if requested + if not self._enable_colors: + for ansi_color in [f for f in output.keys() if 'color' in f]: + output[ansi_color] = '' + + return ( + '{when_color}{when}{reset_color} {who_color}{who:>15}{reset_color} ' + '{severity_color}({severity}){reset_color} {what_color}{what}{reset_color} ' + '{more}'.format(**output) + ) + + def _prettify_output(self, vars_dict): + """ + Creates a string formatted version according to the length of the values in the + dictionary, if the string value is larger than 40 chars, wrap the string using textwrap and + output it last. + + :param vars_dict: dictionary containing the message vars + :type vars_dict: dict(str: str) + :rtype: str + """ + short_values = [] + + # some params for the long texts + long_values = [] + content_indent = ' ' + wrap_width = 80 + + for var_name, var_value in vars_dict.items(): + + # if the value is a string over 40 chars long, + if isinstance(var_value, dict): + long_values.append( + (var_name, simplejson.dumps(var_value, indent=4, cls=ObjectEncoder)) + ) + elif isinstance(var_value, str) and len(var_value) > 40: + wrapped_text = textwrap.fill( + '"{0}"'.format(var_value), + width=wrap_width, + break_long_words=False, + initial_indent=content_indent, + subsequent_indent=content_indent, + replace_whitespace=False, + ) + long_values.append((var_name, wrapped_text)) + else: + short_values.append((var_name, str(var_value))) + + # this will return the following + # {a: b, c: d} (short stuff in the form of json dictionary) + # {"some value": + # "very long text for debugging purposes"} + + # The long text is not a full json string, but a raw string (not escaped), as to keep it human readable, + # but it is surrounded by double-quotes so the coloring lexer will eat it up + values_str = '' + if short_values: + values_str = _JsonFormatter.format_to_json_str( + {k: v for k, v in short_values} + ) + if long_values: + values_str += '\n' + + for lv_name, lv_value in long_values: + values_str += '{{{0}:\n{1}}}\n'.format( + _JsonFormatter.format_to_json_str(lv_name), lv_value.rstrip('\n') + ) + + colorized_output = pygments.highlight( + values_str, + pygments.lexers.JsonLexer(), + pygments.formatters.TerminalTrueColorFormatter(style='paraiso-dark'), + ) + + return colorized_output + + +class FilebeatJsonFormatter(logging.Formatter): + def format(self, record): + + # handle non-json-parsable vars: + try: + + # we can't delete from record.vars because of other handlers + more = dict(record.vars) if len(record.vars) else {} + try: + del more['ctx'] + except: + pass + except Exception as exc: + more = 'Record vars are not parsable: {0}'.format(str(exc)) + + try: + what = record.getMessage() + except Exception as exc: + what = 'Log message is not parsable: {0}'.format(str(exc)) + + output = { + 'when': datetime.datetime.fromtimestamp(record.created).isoformat(), + 'who': record.name, + 'severity': logging.getLevelName(record.levelno), + 'what': what, + 'more': more, + 'ctx': record.vars.get('ctx', ''), + 'lang': 'py', + } + + return _JsonFormatter.format_to_json_str(output) + + +class Client(object): + def __init__( + self, + name, + initial_severity, + initial_console_severity=None, + initial_file_severity=None, + output_dir=None, + output_stdout=True, + max_log_size_mb=5, + max_num_log_files=3, + log_file_name=None, + log_colors='on', + ): + + initial_console_severity = ( + initial_console_severity + if initial_console_severity is not None + else initial_severity + ) + initial_file_severity = ( + initial_file_severity + if initial_file_severity is not None + else initial_severity + ) + + colorama.init() + + # initialize root logger + logging.setLoggerClass(_VariableLogging) + self.logger = logging.getLogger(name) + + # the logger's log level must be set with lowest severity of its handlers + # since it is the logging gateway to the handlers, otherwise they won't get the message. + # ignore unset None / 0 which will disable logging altogether + initial_severities = [ + initial_severity, + initial_console_severity, + initial_file_severity, + ] + lowest_severity = min( + [ + Severity.get_level_by_string(severity) + for severity in initial_severities + if severity + ] + ) + + self.logger.setLevel(lowest_severity) + + if output_stdout: + + # tty friendliness: + # on - disable colors if stdout is not a tty + # always - never disable colors + # off - always disable colors + if log_colors == 'off': + enable_colors = False + elif log_colors == 'always': + enable_colors = True + else: # on - colors when stdout is a tty + enable_colors = sys.stdout.isatty() + + human_stdout_handler = logging.StreamHandler(sys.__stdout__) + human_stdout_handler.setFormatter(HumanReadableFormatter(enable_colors)) + human_stdout_handler.setLevel( + Severity.get_level_by_string(initial_console_severity) + ) + self.logger.addHandler(human_stdout_handler) + + if output_dir is not None: + log_file_name = ( + name.replace('-', '.') + if log_file_name is None + else log_file_name.replace('.log', '') + ) + self.enable_log_file_writing( + output_dir, + max_log_size_mb, + max_num_log_files, + log_file_name, + initial_file_severity, + ) + + def enable_log_file_writing( + self, + output_dir, + max_log_size_mb, + max_num_log_files, + log_file_name, + initial_file_severity, + ): + """ + Adding a rotating file handler to the logger if it doesn't already have one + and creating a log directory if it doesn't exist. + :param output_dir: The path to the logs directory. + :param max_log_size_mb: The max size of the log (for rotation purposes). + :param max_num_log_files: The max number of log files to keep as archive. + :param log_file_name: The name of the log file. + :param initial_file_severity: full string or abbreviation of severity for formatter. + """ + + # Checks if the logger already have a RotatingFileHandler + if not any( + isinstance(h, logging.handlers.RotatingFileHandler) + for h in self.logger.handlers + ): + make_dir_recursively(output_dir) + log_path = os.path.join(output_dir, '{0}.log'.format(log_file_name)) + + # Creates the log file if it doesn't already exist. + rotating_file_handler = logging.handlers.RotatingFileHandler( + log_path, + mode='a+', + maxBytes=max_log_size_mb * 1024 * 1024, + backupCount=max_num_log_files, + ) + + rotating_file_handler.setFormatter(FilebeatJsonFormatter()) + rotating_file_handler.setLevel( + Severity.get_level_by_string(initial_file_severity) + ) + self.logger.addHandler(rotating_file_handler) + + @staticmethod + def register_arguments(parser): + """ + Adds the logger args to the args list. + :param parser: The argparser + """ + parser.add_argument( + '--log-severity', + help='Set log severity', + choices=Severity.user_presentable_severities, + default='info', + ) + + # old-style abbreviation log-level for backwards compatibility + parser.add_argument( + '--log-console-severity', + help='Defines severity of logs printed to console', + choices=Severity.user_presentable_severities, + ) + + # old-style abbreviation log-level for backwards compatibility + parser.add_argument( + '--log-file-severity', + help='Defines severity of logs printed to file', + choices=Severity.user_presentable_severities, + ) + + parser.add_argument( + '--log-disable-stdout', + help='Disable logging to stdout', + action='store_true', + ) + parser.add_argument('--log-output-dir', help='Log files directory path') + parser.add_argument( + '--log-file-rotate-max-file-size', help='Max log file size', default=5 + ) + parser.add_argument( + '--log-file-rotate-num-files', help='Num of log files to keep', default=5 + ) + parser.add_argument( + '--log-file-name', + help='Override to filename (instead of deriving it from the logger name. ' + 'e.g. [node_name].[service_name].[service_instance].log', + ) + parser.add_argument( + '--log-colors', + help='CLI friendly color control. default is on (color when stdout+tty). ' + 'You can also force always/off.', + choices=['on', 'off', 'always'], + default='on', + ) diff --git a/core/__init__.py b/core/__init__.py new file mode 100755 index 0000000..508d685 --- /dev/null +++ b/core/__init__.py @@ -0,0 +1,2 @@ +from .manifest_creator import ImageManifestCreator +from .registry import Registry diff --git a/core/manifest_creator.py b/core/manifest_creator.py new file mode 100644 index 0000000..2117f18 --- /dev/null +++ b/core/manifest_creator.py @@ -0,0 +1,43 @@ +import os +import hashlib +import json + + +class ImageManifestCreator(object): + def __init__(self, config_path, layers_paths): + self._config_path = config_path + self._layers_paths = layers_paths + + def create(self): + manifest = dict() + manifest["schemaVersion"] = 2 + manifest["mediaType"] = "application/vnd.docker.distribution.manifest.v2+json" + manifest["config"] = dict() + manifest["config"][ + "mediaType" + ] = "application/vnd.docker.container.image.v1+json" + manifest["config"]["size"] = os.path.getsize(self._config_path) + manifest["config"]["digest"] = self._get_digest(self._config_path) + manifest["layers"] = [] + for layer in self._layers_paths: + layer_data = dict() + layer_data["mediaType"] = "application/vnd.docker.image.rootfs.diff.tar" + layer_data["size"] = os.path.getsize(layer) + layer_data["digest"] = self._get_digest(layer) + manifest["layers"].append(layer_data) + + return json.dumps(manifest) + + def _get_digest(self, filepath): + return "sha256:" + self.get_file_sha256(filepath) + + @staticmethod + def get_file_sha256(filepath): + sha256hash = hashlib.sha256() + with open(filepath, "rb") as f: + while True: + data = f.read(65536) + sha256hash.update(data) + if not data: + break + return sha256hash.hexdigest() diff --git a/core/registry.py b/core/registry.py new file mode 100644 index 0000000..d5f5fdc --- /dev/null +++ b/core/registry.py @@ -0,0 +1,287 @@ +import os +import os.path +import re +import json +import requests +import requests.auth +import tarfile +import tempfile +import hashlib +import urllib.parse + +from . import manifest_creator + + +class Registry(object): + def __init__( + self, + logger, + registry_url, + archive_path, + stream=False, + login=None, + password=None, + ssl_verify=True, + replace_tags_match=None, + replace_tags_target=None, + ): + self._logger = logger.get_child('registry') + + # enrich http suffix if missing + if urllib.parse.urlparse(registry_url).scheme not in ['http', 'https']: + registry_url = 'http://' + registry_url + self._registry_url = registry_url + self._archive_path = os.path.abspath(archive_path) + self._login = login + self._password = password + self._basicauth = None + self._stream = stream + self._ssl_verify = ssl_verify + self._replace_tags_match = replace_tags_match + self._replace_tags_target = replace_tags_target + if self._login: + self._basicauth = requests.auth.HTTPBasicAuth(self._login, self._password) + self._logger.debug( + 'Initialized', + registry_url=self._registry_url, + archive_path=self._archive_path, + login=self._login, + password=self._password, + ssl_verify=self._ssl_verify, + stream=self._stream, + replace_tags_match=self._replace_tags_match, + replace_tags_target=self._replace_tags_target, + ) + + def process_archive(self): + """ + Processing given archive and pushes the images it contains to the registry + """ + self._logger.info('Processing archive', archive_path=self._archive_path) + archive_manifest = self._get_manifest_from_tar() + self._logger.debug('Extracted archive manifest', manifest_file=archive_manifest) + + for image_config in archive_manifest: + repo_tags = image_config["RepoTags"] + config_loc = image_config["Config"] + config_parsed = self._get_config_from_tar(config_loc) + self._logger.verbose('Parsed config', config_parsed=config_parsed) + + with tempfile.TemporaryDirectory() as tmp_dir_name: + for repo in repo_tags: + image, tag = self._parse_image_tag(repo) + self._logger.info( + 'Extracting tar for image', + image=image, + tag=tag, + tmp_dir_name=tmp_dir_name, + ) + self._extract_tar_file(tmp_dir_name) + + # push individual image layers + layers = image_config["Layers"] + for layer in layers: + self._logger.info('Pushing layer', layer=layer) + push_url = self._initialize_push(image) + layer_path = os.path.join(tmp_dir_name, layer) + self._push_layer(layer_path, push_url) + + # then, push image config + self._logger.info( + 'Pushing image config', image=image, config_loc=config_loc + ) + push_url = self._initialize_push(image) + config_path = os.path.join(tmp_dir_name, config_loc) + self._push_config(config_path, push_url) + + # keep the pushed layers + properly_formatted_layers = [os.path.join(tmp_dir_name, layer) for layer in layers] + + # Now we need to create and push a manifest for the image + creator = manifest_creator.ImageManifestCreator( + config_path, properly_formatted_layers + ) + image_manifest = creator.create() + + # Override tags if needed: from --replace-tags-match and --replace-tags-target + tag = self._replace_tag(image, tag) + + self._logger.info('Pushing image manifest', image=image, tag=tag) + self._push_manifest(image_manifest, image, tag) + self._logger.info('Image Pushed', image=image, tag=tag) + + def _get_manifest_from_tar(self): + return self._extract_json_from_tar(self._archive_path, "manifest.json") + + def _get_config_from_tar(self, name): + return self._extract_json_from_tar(self._archive_path, name) + + def _extract_json_from_tar(self, tar_filepath, file_to_parse): + loaded = self._extract_file_from_tar(tar_filepath, file_to_parse) + stringified = self._parse_as_utf8(loaded) + return json.loads(stringified) + + @staticmethod + def _extract_file_from_tar(tar_filepath, file_to_extract): + manifest = tarfile.open(tar_filepath) + file_contents = manifest.extractfile(file_to_extract) + return file_contents + + @staticmethod + def _parse_as_utf8(to_parse): + as_str = (to_parse.read()).decode("utf-8") + to_parse.close() + return as_str + + def _conditional_print(self, what, end=None): + if self._stream: + if end: + print(what, end=end) + else: + print(what) + + def _extract_tar_file(self, tmp_dir_name): + with tarfile.open(self._archive_path) as fh: + fh.extractall(tmp_dir_name) + + def _push_manifest(self, manifest, image, tag): + headers = { + "Content-Type": "application/vnd.docker.distribution.manifest.v2+json" + } + url = self._registry_url + "/v2/" + image + "/manifests/" + tag + r = requests.put( + url, + headers=headers, + data=manifest, + auth=self._basicauth, + verify=self._ssl_verify, + ) + if r.status_code != 201: + self._logger.log_and_raise( + 'error', 'Failed to push manifest', image=image, tag=tag + ) + + def _initialize_push(self, repository): + """ + Request a push URL for the image repository for a layer or manifest + """ + self._logger.debug('Initializing push', repository=repository) + + response = requests.post( + self._registry_url + "/v2/" + repository + "/blobs/uploads/", + auth=self._basicauth, + verify=self._ssl_verify, + ) + upload_url = None + if response.headers.get("Location", None): + upload_url = response.headers.get("Location") + success = response.status_code == 202 + if not success: + self._logger.log_and_raise( + 'error', + 'Failed to initialize push', + status_code=response.status_code, + contents=response.content, + ) + return upload_url + + def _push_layer(self, layer_path, upload_url): + self._chunked_upload(layer_path, upload_url) + + def _push_config(self, layer_path, upload_url): + self._chunked_upload(layer_path, upload_url) + + def _chunked_upload(self, filepath, url): + content_path = os.path.abspath(filepath) + content_size = os.stat(content_path).st_size + with open(content_path, "rb") as f: + index = 0 + headers = {} + upload_url = url + sha256hash = hashlib.sha256() + + for chunk in self._read_in_chunks(f, sha256hash): + if "http" not in upload_url: + upload_url = self._registry_url + upload_url + offset = index + len(chunk) + headers['Content-Type'] = 'application/octet-stream' + headers['Content-Length'] = str(len(chunk)) + headers['Content-Range'] = '%s-%s' % (index, offset) + index = offset + last = False + if offset == content_size: + last = True + try: + self._conditional_print( + "Pushing... " + + str(round((offset / content_size) * 100, 2)) + + "% ", + end="\r", + ) + if last: + digest_str = str(sha256hash.hexdigest()) + requests.put( + f"{upload_url}&digest=sha256:{digest_str}", + data=chunk, + headers=headers, + auth=self._basicauth, + verify=self._ssl_verify, + ) + else: + response = requests.patch( + upload_url, + data=chunk, + headers=headers, + auth=self._basicauth, + verify=self._ssl_verify, + ) + if "Location" in response.headers: + upload_url = response.headers["Location"] + + except Exception as exc: + self._logger.error( + 'Failed to upload file image upload', filepath=filepath, exc=exc + ) + raise + f.close() + + self._conditional_print("") + + # chunk size default 2T (??) + @staticmethod + def _read_in_chunks(file_object, hashed, chunk_size=2097152): + while True: + data = file_object.read(chunk_size) + hashed.update(data) + if not data: + break + yield data + + @staticmethod + def _parse_image_tag(image_ref): + + # should be 2 parts exactly + image, tag = image_ref.split(":") + return image, tag + + def _replace_tag(self, image, orig_tag): + if self._replace_tags_match and self._replace_tags_match: + match_regex = re.compile(self._replace_tags_match) + if match_regex.match(orig_tag): + self._logger.info( + 'Replacing tag for image', + image=image, + orig_tag=orig_tag, + new_tag=self._replace_tags_target, + ) + return self._replace_tags_target + else: + self._logger.debug( + 'Replace tag match given but did not match', + image=image, + orig_tag=orig_tag, + replace_tags_match=self._replace_tags_match, + new_tag=self._replace_tags_target, + ) + + return orig_tag diff --git a/dockerregistrypusher b/dockerregistrypusher new file mode 100755 index 0000000..3dd686e --- /dev/null +++ b/dockerregistrypusher @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +""" +NOTE: It is important to keep this part BC with py2 as well +That is the reason we use py2 string formatting. +The reason is, python 2 code, that would call manof, would use this entrypoint and thus it should be BC with py2 +""" + +import os +import sys +import subprocess + + +args = ' '.join(sys.argv[1:]) +args = args.replace('\"', '\\\"') +manof_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) + +try: + subprocess.check_call( + 'source {0}/venv/bin/activate && python3 {0}/dockerregistrypusher.py {1}'.format(manof_path, args), + shell=True, + executable='/bin/bash') + +except Exception as exc: + + # by default: save us the traceback + # DEV NOTE: if you need to debug dockerregistrypusher, you might want to: + # - uncomment this + # - print the full CalledProcessError + # traceback.print_exc() + + sys.exit(1) diff --git a/dockerregistrypusher.py b/dockerregistrypusher.py new file mode 100644 index 0000000..0af8239 --- /dev/null +++ b/dockerregistrypusher.py @@ -0,0 +1,128 @@ +import argparse +import sys + +import core +import clients.logging + + +def run(args): + retval = 1 + + # plug in verbosity shorthands + if args.v: + args.log_severity = 'debug' + + logger = clients.logging.Client( + 'pusher', + initial_severity=args.log_severity, + initial_console_severity=args.log_console_severity, + initial_file_severity=args.log_file_severity, + output_stdout=not args.log_disable_stdout, + output_dir=args.log_output_dir, + max_log_size_mb=args.log_file_rotate_max_file_size, + max_num_log_files=args.log_file_rotate_num_files, + log_file_name=args.log_file_name, + log_colors=args.log_colors, + ).logger + + # start root logger with kwargs and create manof + reg_client = core.Registry( + logger=logger, + archive_path=args.archive_path, + registry_url=args.registry_url, + stream=args.stream, + login=args.login, + password=args.password, + ssl_verify=args.ssl_verify, + replace_tags_match=args.replace_tags_match, + replace_tags_target=args.replace_tags_target, + ) + reg_client.process_archive() + if logger.first_error is None: + retval = 0 + + return retval + + +def register_arguments(parser): + # global options for manof + clients.logging.Client.register_arguments(parser) + + # verbosity shorthands + parser.add_argument( + '-v', + help='Set log level to debug (same as --log-severity=debug)', + action='store_true', + default=False, + ) + + parser.add_argument( + 'archive_path', + metavar='ARCHIVE_PATH', + type=str, + help='The url of the target registry to push to', + ) + + parser.add_argument( + 'registry_url', + metavar='REGISTRY_URL', + type=str, + help='The url of the target registry to push to', + ) + parser.add_argument( + '-l', + '--login', + help='Basic-auth login name for registry', + required=False, + ) + + parser.add_argument( + '-p', + '--password', + help='Basic-auth login password for registry', + required=False, + ) + + parser.add_argument( + '--ssl-verify', + help='Skip SSL verification of the registry', + type=bool, + default=True, + ) + + parser.add_argument( + '--stream', + help='Add some streaming logging during push', + type=bool, + default=True, + ) + + parser.add_argument( + '--replace-tags-match', + help='A regex string to match on tags. If matches will be replaces with --replace-tags-target', + type=str, + required=False, + ) + + parser.add_argument( + '--replace-tags-target', + help='If an image tag matches value of --replace-tags-match value, replace it with this tag', + type=str, + required=False, + ) + + +if __name__ == '__main__': + # create an argument parser + arg_parser = argparse.ArgumentParser() + + # register all arguments and sub-commands + register_arguments(arg_parser) + + parsed_args = arg_parser.parse_args() + + # parse the known args, seeing how the targets may add arguments of their own and re-parse + ret_val = run(parsed_args) + + # return value + sys.exit(ret_val) diff --git a/install b/install new file mode 100755 index 0000000..c5b2cd2 --- /dev/null +++ b/install @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +import subprocess + + +def run(command): + subprocess.check_call(['/bin/bash', '-c', command], + stdout=sys.stdout, stderr=sys.stderr) + + +def register_arguments(): + parser = argparse.ArgumentParser() + + parser.add_argument('-o', '--offline-install-path', help='The dir path to install pip packages from.') + parser.add_argument('--no-link', action='store_true', + help='Do not remove or create symlink') + parser.add_argument('--dev', action='store_true', + help='Install dev requirements in addition to common requirements') + + return parser.parse_args() + + +def _create_sym_link(): + + # symlink to run + ln_cmd = 'ln -sfF {0}/dockerregistrypusher /usr/local/bin/dockerregistrypusher'.format(os.getcwd()) + + # first try without sudo, then with + try: + run(ln_cmd) + except subprocess.CalledProcessError: + run('sudo {0}'.format(ln_cmd)) + + +def main(): + args = register_arguments() + if args.offline_install_path is None: + local_pip_packages = '' + else: + local_pip_packages = '--no-index --find-links=file:{}'.format( + args.offline_install_path) + + retval = 0 + try: + requirements_file = 'requirements.txt' + if args.dev: + requirements_file = 'requirements_all.txt' + + # create venv, activate it and install requirements to it + # (from a local dir if it exists) + # "python -m pip install" instead of "pip install" handles a pip + # issue where it fails in a long-named dir + run('virtualenv --python=python3 venv && ' + 'source venv/bin/activate && ' + 'python -m pip install {0} incremental && '.format(local_pip_packages) + + 'python -m pip install {0} -r {1}'.format(local_pip_packages, requirements_file)) + + if not args.no_link: + _create_sym_link() + except subprocess.CalledProcessError as e: + retval = e.returncode + else: + print('Installation complete. Enjoy your dockerregistrypusher!') + + sys.exit(retval) + + +if __name__ == '__main__': + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1bcee9a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +# please keep this to a minimum - defaults are good +[tool.black] +target-version = ['py36', 'py37', 'py38'] +exclude = ''' +/( + .git + | .venv + | venv +)/ +''' diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c58f7cb --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +-r requirements/common.txt diff --git a/requirements/common.txt b/requirements/common.txt new file mode 100644 index 0000000..7bba6db --- /dev/null +++ b/requirements/common.txt @@ -0,0 +1,6 @@ +requests==2.25.1 +urllib3==1.26.2 +simplejson==3.8.2 +colorama==0.4.4 +pygments==2.2.0 +incremental==17.5.0 diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 0000000..adbb807 --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,5 @@ +mock~=4.0 +flake8~=3.8 +inflection~=0.5 +nose~=1.3 +git+git://github.com/psf/black diff --git a/requirements_all.txt b/requirements_all.txt new file mode 100644 index 0000000..a4938df --- /dev/null +++ b/requirements_all.txt @@ -0,0 +1,2 @@ +-r requirements/dev.txt +-r requirements/common.txt diff --git a/tests/pusher_tests.py b/tests/pusher_tests.py new file mode 100644 index 0000000..bbc84e6 --- /dev/null +++ b/tests/pusher_tests.py @@ -0,0 +1,95 @@ +import unittest +import requests +import docker +from dockertarpusher import Registry + + +class TestPusher(unittest.TestCase): + @classmethod + def setUpClass(self): + print("Setuping") + print("Pulling registry...") + client = docker.from_env() + client.images.pull("registry:2.7.1") + + @classmethod + def tearDownClass(self): + print("Cleaning up") + client = docker.from_env() + try: + client.containers.get("registrytest").remove(force=True) + except: + print("") + try: + client.volumes.get("registrytestvol").remove(force=True) + except: + print("") + + def startRegistry(self): + client = docker.from_env() + client.containers.run( + "registry:2.7.1", + detach=True, + ports={"5000/tcp": 5000}, + name="registrytest", + volumes={"registrytestvol": {"bind": "/var/lib/registry", "mode": "rw"}}, + ) + + def stopRegistry(self): + client = docker.from_env() + client.containers.get("registrytest").remove(force=True) + client.volumes.get("registrytestvol").remove(force=True) + + def setUp(self): + print("Starting clean registry") + try: + self.stopRegistry() + except: + print("Cannot stop registry, probably not exists") + self.startRegistry() + + def testOneLayer(self): + registryUrl = "http://localhost:5000" + reg = Registry(registryUrl, "tests/busybox.tar") + reg.processImage() + r = requests.get(registryUrl + "/v2/_catalog") + self.assertTrue("razikus/busybox" in r.json()["repositories"]) + r = requests.get(registryUrl + "/v2/razikus/busybox/tags/list") + self.assertTrue("razikus/busybox" == r.json()["name"]) + self.assertTrue("1.31" in r.json()["tags"]) + + def testOneLayerAndRun(self): + registryUrl = "http://localhost:5000" + reg = Registry(registryUrl, "tests/busybox.tar") + reg.processImage() + r = requests.get(registryUrl + "/v2/_catalog") + self.assertTrue("razikus/busybox" in r.json()["repositories"]) + r = requests.get(registryUrl + "/v2/razikus/busybox/tags/list") + self.assertTrue("razikus/busybox" == r.json()["name"]) + self.assertTrue("1.31" in r.json()["tags"]) + client = docker.from_env() + client.images.pull("localhost:5000/razikus/busybox:1.31") + client.containers.run("localhost:5000/razikus/busybox:1.31", remove=True) + client.images.remove("localhost:5000/razikus/busybox:1.31") + + def testMultipleLayersWithDockerSave(self): + client = docker.from_env() + client.images.pull("razikus/whoami:slim80") + image = client.images.get("razikus/whoami:slim80") + f = open("tests/whoami.tar", "wb") + for chunk in image.save(named=True): + f.write(chunk) + f.close() + + registryUrl = "http://localhost:5000" + reg = Registry(registryUrl, "tests/whoami.tar") + reg.processImage() + r = requests.get(registryUrl + "/v2/_catalog") + self.assertTrue("razikus/whoami" in r.json()["repositories"]) + r = requests.get(registryUrl + "/v2/razikus/whoami/tags/list") + self.assertTrue("razikus/whoami" == r.json()["name"]) + self.assertTrue("slim80" in r.json()["tags"]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO b/tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO new file mode 100644 index 0000000..37d454f --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO @@ -0,0 +1,11 @@ +Metadata-Version: 1.1 +Name: flake8-igz +Version: 0.1.0 +Summary: iguazio's twisted extension to flake8 +Home-page: UNKNOWN +Author: Adam Melnick +Author-email: adamm@iguazio.com +License: MIT +Description: UNKNOWN +Platform: UNKNOWN +Provides: flake8_igz diff --git a/tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt b/tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt new file mode 100644 index 0000000..65ce5d0 --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt @@ -0,0 +1,8 @@ +flake8_igz.py +setup.py +flake8_igz.egg-info/PKG-INFO +flake8_igz.egg-info/SOURCES.txt +flake8_igz.egg-info/dependency_links.txt +flake8_igz.egg-info/entry_points.txt +flake8_igz.egg-info/requires.txt +flake8_igz.egg-info/top_level.txt \ No newline at end of file diff --git a/tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt b/tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt b/tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt new file mode 100644 index 0000000..a40a758 --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt @@ -0,0 +1,8 @@ +[flake8.extension] +flake8-igz.class_name_camel_case = flake8_igz:class_name_camel_case +flake8-igz.ctx_log_non_string_first_param = flake8_igz:ctx_log_non_string_first_param +flake8-igz.logger_forbid_passing_self = flake8_igz:logger_forbid_passing_self +flake8-igz.multiline_string_double_quotes = flake8_igz:multiline_string_double_quotes +flake8-igz.multiline_string_on_newline = flake8_igz:multiline_string_on_newline +flake8-igz.single_quote_strings = flake8_igz:single_quote_strings + diff --git a/tools/flake8_plugin/flake8_igz.egg-info/requires.txt b/tools/flake8_plugin/flake8_igz.egg-info/requires.txt new file mode 100644 index 0000000..7607850 --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.egg-info/requires.txt @@ -0,0 +1 @@ +flake8>3.0.0 diff --git a/tools/flake8_plugin/flake8_igz.egg-info/top_level.txt b/tools/flake8_plugin/flake8_igz.egg-info/top_level.txt new file mode 100644 index 0000000..0ff4173 --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.egg-info/top_level.txt @@ -0,0 +1 @@ +flake8_igz diff --git a/tools/flake8_plugin/flake8_igz.py b/tools/flake8_plugin/flake8_igz.py new file mode 100644 index 0000000..d739392 --- /dev/null +++ b/tools/flake8_plugin/flake8_igz.py @@ -0,0 +1,99 @@ +import inflection +import token +import re + + +class Constants(object): + + string_prefixes = 'ubrUBR' + log_levels = ['verbose', 'debug', 'info', 'warn', 'error'] + + +class Utils(object): + @staticmethod + def get_string_tokens(tokens): + for tid, lexeme, start, end, _ in tokens: + if tid == token.STRING: + lexeme = lexeme.lstrip(Constants.string_prefixes) + yield lexeme, start, end + + +def single_quote_strings(logical_line, tokens): + for ( + lexeme, + start, + _, + ) in Utils.get_string_tokens(tokens): + if lexeme.startswith('"') and not lexeme.startswith('"""'): + yield start, 'I100 double-quote string used (expected single-quote)' + + +def multiline_string_on_newline(logical_line, tokens): + for ( + lexeme, + start, + end, + ) in Utils.get_string_tokens(tokens): + if lexeme.startswith('"""'): + if not re.match(r'^\"\"\"\n', lexeme): + yield start, 'I101 multiline string must start on next line after triple double-quotes' + if not re.search(r'\n\s*\"\"\"$', lexeme): + yield end, 'I102 multiline string must end with triple double-quotes in new line' + + +def multiline_string_double_quotes(logical_line, tokens): + for ( + lexeme, + start, + _, + ) in Utils.get_string_tokens(tokens): + if lexeme.startswith('\'\'\''): + yield start, 'I103 triple single-quotes used in multiline string (expected triple double-quotes)' + + +def ctx_log_non_string_first_param(logical_line, tokens): + if logical_line.startswith('ctx.log.'): + for idx, (tid, lexeme, start, _, _) in enumerate(tokens): + if tid == token.NAME and lexeme in Constants.log_levels: + + # plus one for the ( parentheses, plus one for the first param + first_param_token = tokens[idx + 2] + + if first_param_token[0] == token.STRING: + yield first_param_token[ + 2 + ], 'I104 ctx.log.{0} call with string as first param'.format(lexeme) + + +def class_name_camel_case(logical_line, tokens): + if logical_line.startswith('class'): + for idx, (tid, lexeme, start, _, _) in enumerate(tokens): + if tid == token.NAME and lexeme == 'class': + class_name_token = tokens[idx + 1] + camelized = inflection.camelize(class_name_token[1], True) + + if class_name_token[1] != camelized: + yield class_name_token[ + 2 + ], 'I105 class name not camel case. (suggestion: {0})'.format( + camelized + ) + + +def logger_forbid_passing_self(logical_line, tokens): + if logical_line.startswith('self._logger.'): + for idx, (tid, lexeme, start, _, _) in enumerate(tokens): + if tid == token.NAME and lexeme in Constants.log_levels: + + # plus one for the ( parentheses, plus one for the first param + first_param_token = tokens[idx + 2] + + if ( + first_param_token[1] == 'self' + and first_param_token[0] != token.STRING + ): + yield first_param_token[ + 2 + ], 'I106 self._logger.{0} call with self as first param'.format( + lexeme + ) diff --git a/tools/flake8_plugin/setup.py b/tools/flake8_plugin/setup.py new file mode 100644 index 0000000..8992eff --- /dev/null +++ b/tools/flake8_plugin/setup.py @@ -0,0 +1,35 @@ +from __future__ import with_statement +import setuptools + +requires = [ + 'flake8 > 3.0.0', +] + +flake8_entry_point = 'flake8.extension' + +setuptools.setup( + name='flake8-igz', + license='MIT', + version='0.1.0', + description='iguazio\'s twisted extension to flake8', + author='Adam Melnick', + author_email='adamm@iguazio.com', + provides=['flake8_igz'], + py_modules=['flake8_igz'], + install_requires=requires, + entry_points={ + flake8_entry_point: [ + 'flake8-igz.single_quote_strings = flake8_igz:single_quote_strings', + 'flake8-igz.multiline_string_on_newline =' + ' flake8_igz:multiline_string_on_newline', + 'flake8-igz.multiline_string_double_quotes =' + ' flake8_igz:multiline_string_double_quotes', + 'flake8-igz.ctx_log_non_string_first_param =' + ' flake8_igz:ctx_log_non_string_first_param', + 'flake8-igz.class_name_camel_case = flake8_igz:class_name_camel_case', + 'flake8-igz.logger_forbid_passing_self =' + ' flake8_igz:logger_forbid_passing_self', + ], + }, + classifiers=[], +) diff --git a/upload.sh b/upload.sh new file mode 100644 index 0000000..25b17dd --- /dev/null +++ b/upload.sh @@ -0,0 +1 @@ +python3 -m twine upload dist/* --verbose