diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000..40e8d31
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,22 @@
+[flake8]
+
+# no lines over 120 allowed
+max-line-length = 120
+
+# exclude these dirs
+exclude = .git,venv
+
+# ignore the following rules
+ignore =
+
+ # whitespace before ':'
+ E203
+
+ # line break before binary operator
+ W503
+
+ # F401 unused imports (TODO: unignore and use flake8-putty to allow this for 'models' when it supports flake8 v3)
+ F401
+
+ # bare except
+ E722
diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml
new file mode 100644
index 0000000..ca90839
--- /dev/null
+++ b/.github/workflows/CI.yaml
@@ -0,0 +1,28 @@
+name: CI
+
+on:
+ push:
+ branches:
+ - development
+ - master
+ pull_request:
+ branches:
+ - development
+
+jobs:
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.7
+
+ - name: Install dependencies
+ run: make install-ci
+
+ - name: Run lint
+ run: make lint
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7d014a0
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,11 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# pycharm proj
+.idea
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..73f69e0
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/
diff --git a/.idea/dockerregistrypusher.iml b/.idea/dockerregistrypusher.iml
new file mode 100644
index 0000000..757717e
--- /dev/null
+++ b/.idea/dockerregistrypusher.iml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000..11fe9bf
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,42 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..5509b4d
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..c29154e
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..7ee6e46
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Adam Raźniewski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..710deb5
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,52 @@
+VENV_PYTHON = ./venv/bin/python
+
+.PHONY: all
+all: venv lint
+ @echo Done.
+
+.PHONY: lint
+lint: flake8 fmt-check
+
+.PHONY: flake8
+flake8:
+ @echo "Running flake8 lint..."
+ $(VENV_PYTHON) -m flake8 .
+
+.PHONY: fmt
+fmt:
+ @echo "Running black fmt..."
+ $(VENV_PYTHON) -m black --skip-string-normalization .
+
+.PHONY: fmt-check
+fmt-check:
+ @echo "Running black fmt check..."
+ $(VENV_PYTHON) -m black --skip-string-normalization --check --diff .
+
+#.PHONY: test
+#test: test-unit test-integ
+# @echo Finished running Tests
+#
+#.PHONY: test-unit
+#test-unit: venv
+# $(VENV_PYTHON) -m nose tests/unit
+#
+#.PHONY: test-integ
+#test-integ: venv
+# $(VENV_PYTHON) -m nose tests/integration/cases/
+
+.PHONY: install
+install: venv
+ @echo Installed
+
+.PHONY: install-ci
+install-ci: install-venv install
+
+.PHONY: venv
+venv:
+ python ./install --dev
+ $(VENV_PYTHON) -m pip install -e ./tools/flake8_plugin
+
+.PHONY: install-venv
+install-venv:
+ python -m pip install --upgrade pip
+ python -m pip install virtualenv
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9d0134a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,43 @@
+# What is this?
+This package contains dockerregistrypusher CLI that allows to push image packed as tar (usually from docker save command) to a docker registry
+This project was forked from [Adam Raźniewski's dockerregistrypusher](https://github.com/Razikus/dockerregistrypusher) but with changes and adjustments to iguazio's needs as a CLI
+All rights reserved to the original author [Adam](https://github.com/Razikus)
+
+# Why?
+To push tar-packed image archives (created by `docker save`) to registries without going through (and taxing) docker-daemon
+
+Usage of CLI:
+
+# installation
+
+Install and create a symlink at `/usr/local/bin/dockerregistrypusher` (requires sudo)
+```shell
+./install
+```
+
+Or, install without symlink creation (no elevated permissions needed)
+```shell
+./install --no-link
+```
+
+# Running the CLI
+
+CLI structure
+```shell
+dockerregistrypusher [options] {TAR_PATH} {REGISTRY_URL}
+```
+
+For further help (duh)
+```shell
+dockerregistrypusher --help
+```
+
+
+# Development
+To be able to run linting / formatting and other `make` goodness, install with dev requirements
+```shell
+make install
+```
+
+# License
+Free to use (MIT)
diff --git a/clients/logging/__init__.py b/clients/logging/__init__.py
new file mode 100644
index 0000000..8bf0ef2
--- /dev/null
+++ b/clients/logging/__init__.py
@@ -0,0 +1,517 @@
+import sys
+import errno
+import logging
+import logging.handlers
+import simplejson
+import datetime
+import textwrap
+import os
+
+import colorama
+import pygments
+import pygments.formatters
+import pygments.lexers
+
+
+def make_dir_recursively(path):
+ """
+ Create a directory in a location if it doesn't exist
+
+ :param path: The path to create
+ """
+ if not os.path.exists(path):
+ try:
+ os.makedirs(path)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST and os.path.isdir(path):
+ pass
+ else:
+ raise
+
+
+class Record(logging.LogRecord):
+ pass
+
+
+class Severity(object):
+
+ Verbose = 5
+ Debug = logging.DEBUG
+ Info = logging.INFO
+ Warning = logging.WARNING
+ Error = logging.ERROR
+
+ string_enum_dict = {
+ 'verbose': Verbose,
+ 'debug': Debug,
+ 'info': Info,
+ 'warn': Warning,
+ 'warning': Warning,
+ 'error': Error,
+ # Allow abbreviations
+ # Also provides backwards compatibility with log-console/file-severity syntax
+ 'V': Verbose,
+ 'D': Debug,
+ 'I': Info,
+ 'W': Warning,
+ 'E': Error,
+ }
+
+ user_presentable_severities = ['verbose', 'debug', 'info', 'warn', 'error']
+
+ @staticmethod
+ def get_level_by_string(severity_string):
+ return Severity.string_enum_dict.get(severity_string, 0)
+
+
+class _VariableLogging(logging.Logger):
+
+ get_child = logging.Logger.getChild
+
+ def __init__(self, name, level=logging.NOTSET):
+ logging.Logger.__init__(self, name, level)
+ self._bound_variables = {}
+
+ # each time Logger.get_child is called, the Logger manager creates
+ # a new Logger instance and adds it to his list
+ # so we need to add the first error to the manager attributes
+ # so we can keep the first error in the whole application
+ if not hasattr(self.manager, 'first_error'):
+ setattr(self.manager, 'first_error', None)
+
+ @property
+ def first_error(self):
+ return self.manager.first_error
+
+ def clear_first_error(self):
+ if hasattr(self.manager, 'first_error'):
+ self.manager.first_error = None
+
+ def _check_and_log(self, level, msg, args, kw_args):
+ if self.isEnabledFor(level):
+ kw_args.update(self._bound_variables)
+ self._log(level, msg, args, extra={'vars': kw_args})
+
+ def error(self, msg, *args, **kw_args):
+ if self.manager.first_error is None:
+ self.manager.first_error = {'msg': msg, 'args': args, 'kw_args': kw_args}
+
+ self._check_and_log(Severity.Error, msg, args, kw_args)
+
+ def warn(self, msg, *args, **kw_args):
+ self._check_and_log(Severity.Warning, msg, args, kw_args)
+
+ def info(self, msg, *args, **kw_args):
+ self._check_and_log(Severity.Info, msg, args, kw_args)
+
+ def debug(self, msg, *args, **kw_args):
+ self._check_and_log(Severity.Debug, msg, args, kw_args)
+
+ def verbose(self, msg, *args, **kw_args):
+ self._check_and_log(Severity.Verbose, msg, args, kw_args)
+
+ def log_and_raise(self, severity, error_msg, *args, **kwargs):
+ getattr(self, severity)(error_msg, *args, **kwargs)
+
+ # format the exception into the raised error message if we got one
+ if 'exc' in kwargs:
+ error_msg = '{0}: {1}'.format(error_msg, kwargs['exc'].lower())
+
+ exception_type = kwargs.get('exc_type', RuntimeError)
+
+ raise exception_type(error_msg)
+
+ def bind(self, **kw_args):
+ self._bound_variables.update(kw_args)
+
+
+class ObjectEncoder(simplejson.JSONEncoder):
+ def default(self, obj):
+ try:
+ return obj.__log__()
+ except:
+ try:
+ return obj.__repr__()
+ except:
+ return str(obj)
+
+
+class _JsonFormatter(logging.Formatter):
+ @staticmethod
+ def format_to_json_str(params):
+ try:
+
+ # default encoding is utf8
+ return simplejson.dumps(params, cls=ObjectEncoder)
+ except:
+
+ # this is the widest complementary encoding found
+ return simplejson.dumps(
+ params, cls=ObjectEncoder, encoding='raw_unicode_escape'
+ )
+
+ def format(self, record):
+ params = {
+ 'datetime': self.formatTime(record, self.datefmt),
+ 'name': record.name,
+ 'level': record.levelname.lower(),
+ 'message': record.getMessage(),
+ }
+
+ params.update(record.vars)
+
+ return _JsonFormatter.format_to_json_str(params)
+
+
+class HumanReadableFormatter(logging.Formatter):
+ def __init__(self, enable_colors, *args, **kwargs):
+ super(logging.Formatter, self).__init__(*args, **kwargs)
+ self._enable_colors = enable_colors
+
+ # Maps severity to its letter representation
+ _level_to_short_name = {
+ Severity.Verbose: 'V',
+ Severity.Debug: 'D',
+ Severity.Info: 'I',
+ Severity.Warning: 'W',
+ Severity.Error: 'E',
+ }
+
+ # Maps severity to its color representation
+ _level_to_color = {
+ Severity.Info: colorama.Fore.LIGHTGREEN_EX,
+ Severity.Warning: colorama.Fore.LIGHTYELLOW_EX,
+ Severity.Error: colorama.Fore.LIGHTRED_EX,
+ }
+
+ def format(self, record):
+ def _get_what_color():
+ return {
+ Severity.Verbose: colorama.Fore.LIGHTCYAN_EX,
+ Severity.Debug: colorama.Fore.LIGHTCYAN_EX,
+ Severity.Info: colorama.Fore.CYAN,
+ Severity.Warning: colorama.Fore.LIGHTCYAN_EX,
+ Severity.Error: colorama.Fore.LIGHTCYAN_EX,
+ }.get(record.levelno, colorama.Fore.LIGHTCYAN_EX)
+
+ # coloured using pygments
+ if self._enable_colors:
+ more = self._prettify_output(record.vars) if len(record.vars) else ''
+ else:
+ try:
+ more = simplejson.dumps(record.vars) if len(record.vars) else ''
+
+ # defensive
+ except Exception as exc:
+ more = simplejson.dumps({'Log formatting error': str(exc)})
+
+ output = {
+ 'reset_color': colorama.Fore.RESET,
+ 'when': datetime.datetime.fromtimestamp(record.created).strftime(
+ '%d.%m.%y %H:%M:%S.%f'
+ ),
+ 'when_color': colorama.Fore.LIGHTYELLOW_EX,
+ 'who': record.name[-15:],
+ 'who_color': colorama.Fore.WHITE,
+ 'severity': HumanReadableFormatter._level_to_short_name[record.levelno],
+ 'severity_color': HumanReadableFormatter._level_to_color.get(
+ record.levelno, colorama.Fore.RESET
+ ),
+ 'what': record.getMessage(),
+ 'what_color': _get_what_color(),
+ 'more': more,
+ }
+
+ # Slice ms to be at maximum of 3 digits
+ try:
+ time_parts = output['when'].split('.')
+ time_parts[-1] = time_parts[-1][:-3]
+ output['when'] = '.'.join(time_parts)
+ except:
+ pass
+
+ # Disable coloring if requested
+ if not self._enable_colors:
+ for ansi_color in [f for f in output.keys() if 'color' in f]:
+ output[ansi_color] = ''
+
+ return (
+ '{when_color}{when}{reset_color} {who_color}{who:>15}{reset_color} '
+ '{severity_color}({severity}){reset_color} {what_color}{what}{reset_color} '
+ '{more}'.format(**output)
+ )
+
+ def _prettify_output(self, vars_dict):
+ """
+ Creates a string formatted version according to the length of the values in the
+ dictionary, if the string value is larger than 40 chars, wrap the string using textwrap and
+ output it last.
+
+ :param vars_dict: dictionary containing the message vars
+ :type vars_dict: dict(str: str)
+ :rtype: str
+ """
+ short_values = []
+
+ # some params for the long texts
+ long_values = []
+ content_indent = ' '
+ wrap_width = 80
+
+ for var_name, var_value in vars_dict.items():
+
+ # if the value is a string over 40 chars long,
+ if isinstance(var_value, dict):
+ long_values.append(
+ (var_name, simplejson.dumps(var_value, indent=4, cls=ObjectEncoder))
+ )
+ elif isinstance(var_value, str) and len(var_value) > 40:
+ wrapped_text = textwrap.fill(
+ '"{0}"'.format(var_value),
+ width=wrap_width,
+ break_long_words=False,
+ initial_indent=content_indent,
+ subsequent_indent=content_indent,
+ replace_whitespace=False,
+ )
+ long_values.append((var_name, wrapped_text))
+ else:
+ short_values.append((var_name, str(var_value)))
+
+ # this will return the following
+ # {a: b, c: d} (short stuff in the form of json dictionary)
+ # {"some value":
+ # "very long text for debugging purposes"}
+
+ # The long text is not a full json string, but a raw string (not escaped), as to keep it human readable,
+ # but it is surrounded by double-quotes so the coloring lexer will eat it up
+ values_str = ''
+ if short_values:
+ values_str = _JsonFormatter.format_to_json_str(
+ {k: v for k, v in short_values}
+ )
+ if long_values:
+ values_str += '\n'
+
+ for lv_name, lv_value in long_values:
+ values_str += '{{{0}:\n{1}}}\n'.format(
+ _JsonFormatter.format_to_json_str(lv_name), lv_value.rstrip('\n')
+ )
+
+ colorized_output = pygments.highlight(
+ values_str,
+ pygments.lexers.JsonLexer(),
+ pygments.formatters.TerminalTrueColorFormatter(style='paraiso-dark'),
+ )
+
+ return colorized_output
+
+
+class FilebeatJsonFormatter(logging.Formatter):
+ def format(self, record):
+
+ # handle non-json-parsable vars:
+ try:
+
+ # we can't delete from record.vars because of other handlers
+ more = dict(record.vars) if len(record.vars) else {}
+ try:
+ del more['ctx']
+ except:
+ pass
+ except Exception as exc:
+ more = 'Record vars are not parsable: {0}'.format(str(exc))
+
+ try:
+ what = record.getMessage()
+ except Exception as exc:
+ what = 'Log message is not parsable: {0}'.format(str(exc))
+
+ output = {
+ 'when': datetime.datetime.fromtimestamp(record.created).isoformat(),
+ 'who': record.name,
+ 'severity': logging.getLevelName(record.levelno),
+ 'what': what,
+ 'more': more,
+ 'ctx': record.vars.get('ctx', ''),
+ 'lang': 'py',
+ }
+
+ return _JsonFormatter.format_to_json_str(output)
+
+
+class Client(object):
+ def __init__(
+ self,
+ name,
+ initial_severity,
+ initial_console_severity=None,
+ initial_file_severity=None,
+ output_dir=None,
+ output_stdout=True,
+ max_log_size_mb=5,
+ max_num_log_files=3,
+ log_file_name=None,
+ log_colors='on',
+ ):
+
+ initial_console_severity = (
+ initial_console_severity
+ if initial_console_severity is not None
+ else initial_severity
+ )
+ initial_file_severity = (
+ initial_file_severity
+ if initial_file_severity is not None
+ else initial_severity
+ )
+
+ colorama.init()
+
+ # initialize root logger
+ logging.setLoggerClass(_VariableLogging)
+ self.logger = logging.getLogger(name)
+
+ # the logger's log level must be set with lowest severity of its handlers
+ # since it is the logging gateway to the handlers, otherwise they won't get the message.
+ # ignore unset None / 0 which will disable logging altogether
+ initial_severities = [
+ initial_severity,
+ initial_console_severity,
+ initial_file_severity,
+ ]
+ lowest_severity = min(
+ [
+ Severity.get_level_by_string(severity)
+ for severity in initial_severities
+ if severity
+ ]
+ )
+
+ self.logger.setLevel(lowest_severity)
+
+ if output_stdout:
+
+ # tty friendliness:
+ # on - disable colors if stdout is not a tty
+ # always - never disable colors
+ # off - always disable colors
+ if log_colors == 'off':
+ enable_colors = False
+ elif log_colors == 'always':
+ enable_colors = True
+ else: # on - colors when stdout is a tty
+ enable_colors = sys.stdout.isatty()
+
+ human_stdout_handler = logging.StreamHandler(sys.__stdout__)
+ human_stdout_handler.setFormatter(HumanReadableFormatter(enable_colors))
+ human_stdout_handler.setLevel(
+ Severity.get_level_by_string(initial_console_severity)
+ )
+ self.logger.addHandler(human_stdout_handler)
+
+ if output_dir is not None:
+ log_file_name = (
+ name.replace('-', '.')
+ if log_file_name is None
+ else log_file_name.replace('.log', '')
+ )
+ self.enable_log_file_writing(
+ output_dir,
+ max_log_size_mb,
+ max_num_log_files,
+ log_file_name,
+ initial_file_severity,
+ )
+
+ def enable_log_file_writing(
+ self,
+ output_dir,
+ max_log_size_mb,
+ max_num_log_files,
+ log_file_name,
+ initial_file_severity,
+ ):
+ """
+ Adding a rotating file handler to the logger if it doesn't already have one
+ and creating a log directory if it doesn't exist.
+ :param output_dir: The path to the logs directory.
+ :param max_log_size_mb: The max size of the log (for rotation purposes).
+ :param max_num_log_files: The max number of log files to keep as archive.
+ :param log_file_name: The name of the log file.
+ :param initial_file_severity: full string or abbreviation of severity for formatter.
+ """
+
+ # Checks if the logger already have a RotatingFileHandler
+ if not any(
+ isinstance(h, logging.handlers.RotatingFileHandler)
+ for h in self.logger.handlers
+ ):
+ make_dir_recursively(output_dir)
+ log_path = os.path.join(output_dir, '{0}.log'.format(log_file_name))
+
+ # Creates the log file if it doesn't already exist.
+ rotating_file_handler = logging.handlers.RotatingFileHandler(
+ log_path,
+ mode='a+',
+ maxBytes=max_log_size_mb * 1024 * 1024,
+ backupCount=max_num_log_files,
+ )
+
+ rotating_file_handler.setFormatter(FilebeatJsonFormatter())
+ rotating_file_handler.setLevel(
+ Severity.get_level_by_string(initial_file_severity)
+ )
+ self.logger.addHandler(rotating_file_handler)
+
+ @staticmethod
+ def register_arguments(parser):
+ """
+ Adds the logger args to the args list.
+ :param parser: The argparser
+ """
+ parser.add_argument(
+ '--log-severity',
+ help='Set log severity',
+ choices=Severity.user_presentable_severities,
+ default='info',
+ )
+
+ # old-style abbreviation log-level for backwards compatibility
+ parser.add_argument(
+ '--log-console-severity',
+ help='Defines severity of logs printed to console',
+ choices=Severity.user_presentable_severities,
+ )
+
+ # old-style abbreviation log-level for backwards compatibility
+ parser.add_argument(
+ '--log-file-severity',
+ help='Defines severity of logs printed to file',
+ choices=Severity.user_presentable_severities,
+ )
+
+ parser.add_argument(
+ '--log-disable-stdout',
+ help='Disable logging to stdout',
+ action='store_true',
+ )
+ parser.add_argument('--log-output-dir', help='Log files directory path')
+ parser.add_argument(
+ '--log-file-rotate-max-file-size', help='Max log file size', default=5
+ )
+ parser.add_argument(
+ '--log-file-rotate-num-files', help='Num of log files to keep', default=5
+ )
+ parser.add_argument(
+ '--log-file-name',
+ help='Override to filename (instead of deriving it from the logger name. '
+ 'e.g. [node_name].[service_name].[service_instance].log',
+ )
+ parser.add_argument(
+ '--log-colors',
+ help='CLI friendly color control. default is on (color when stdout+tty). '
+ 'You can also force always/off.',
+ choices=['on', 'off', 'always'],
+ default='on',
+ )
diff --git a/core/__init__.py b/core/__init__.py
new file mode 100755
index 0000000..508d685
--- /dev/null
+++ b/core/__init__.py
@@ -0,0 +1,2 @@
+from .manifest_creator import ImageManifestCreator
+from .registry import Registry
diff --git a/core/manifest_creator.py b/core/manifest_creator.py
new file mode 100644
index 0000000..2117f18
--- /dev/null
+++ b/core/manifest_creator.py
@@ -0,0 +1,43 @@
+import os
+import hashlib
+import json
+
+
+class ImageManifestCreator(object):
+ def __init__(self, config_path, layers_paths):
+ self._config_path = config_path
+ self._layers_paths = layers_paths
+
+ def create(self):
+ manifest = dict()
+ manifest["schemaVersion"] = 2
+ manifest["mediaType"] = "application/vnd.docker.distribution.manifest.v2+json"
+ manifest["config"] = dict()
+ manifest["config"][
+ "mediaType"
+ ] = "application/vnd.docker.container.image.v1+json"
+ manifest["config"]["size"] = os.path.getsize(self._config_path)
+ manifest["config"]["digest"] = self._get_digest(self._config_path)
+ manifest["layers"] = []
+ for layer in self._layers_paths:
+ layer_data = dict()
+ layer_data["mediaType"] = "application/vnd.docker.image.rootfs.diff.tar"
+ layer_data["size"] = os.path.getsize(layer)
+ layer_data["digest"] = self._get_digest(layer)
+ manifest["layers"].append(layer_data)
+
+ return json.dumps(manifest)
+
+ def _get_digest(self, filepath):
+ return "sha256:" + self.get_file_sha256(filepath)
+
+ @staticmethod
+ def get_file_sha256(filepath):
+ sha256hash = hashlib.sha256()
+ with open(filepath, "rb") as f:
+ while True:
+ data = f.read(65536)
+ sha256hash.update(data)
+ if not data:
+ break
+ return sha256hash.hexdigest()
diff --git a/core/registry.py b/core/registry.py
new file mode 100644
index 0000000..d5f5fdc
--- /dev/null
+++ b/core/registry.py
@@ -0,0 +1,287 @@
+import os
+import os.path
+import re
+import json
+import requests
+import requests.auth
+import tarfile
+import tempfile
+import hashlib
+import urllib.parse
+
+from . import manifest_creator
+
+
+class Registry(object):
+ def __init__(
+ self,
+ logger,
+ registry_url,
+ archive_path,
+ stream=False,
+ login=None,
+ password=None,
+ ssl_verify=True,
+ replace_tags_match=None,
+ replace_tags_target=None,
+ ):
+ self._logger = logger.get_child('registry')
+
+ # enrich http suffix if missing
+ if urllib.parse.urlparse(registry_url).scheme not in ['http', 'https']:
+ registry_url = 'http://' + registry_url
+ self._registry_url = registry_url
+ self._archive_path = os.path.abspath(archive_path)
+ self._login = login
+ self._password = password
+ self._basicauth = None
+ self._stream = stream
+ self._ssl_verify = ssl_verify
+ self._replace_tags_match = replace_tags_match
+ self._replace_tags_target = replace_tags_target
+ if self._login:
+ self._basicauth = requests.auth.HTTPBasicAuth(self._login, self._password)
+ self._logger.debug(
+ 'Initialized',
+ registry_url=self._registry_url,
+ archive_path=self._archive_path,
+ login=self._login,
+ password=self._password,
+ ssl_verify=self._ssl_verify,
+ stream=self._stream,
+ replace_tags_match=self._replace_tags_match,
+ replace_tags_target=self._replace_tags_target,
+ )
+
+ def process_archive(self):
+ """
+ Processing given archive and pushes the images it contains to the registry
+ """
+ self._logger.info('Processing archive', archive_path=self._archive_path)
+ archive_manifest = self._get_manifest_from_tar()
+ self._logger.debug('Extracted archive manifest', manifest_file=archive_manifest)
+
+ for image_config in archive_manifest:
+ repo_tags = image_config["RepoTags"]
+ config_loc = image_config["Config"]
+ config_parsed = self._get_config_from_tar(config_loc)
+ self._logger.verbose('Parsed config', config_parsed=config_parsed)
+
+ with tempfile.TemporaryDirectory() as tmp_dir_name:
+ for repo in repo_tags:
+ image, tag = self._parse_image_tag(repo)
+ self._logger.info(
+ 'Extracting tar for image',
+ image=image,
+ tag=tag,
+ tmp_dir_name=tmp_dir_name,
+ )
+ self._extract_tar_file(tmp_dir_name)
+
+ # push individual image layers
+ layers = image_config["Layers"]
+ for layer in layers:
+ self._logger.info('Pushing layer', layer=layer)
+ push_url = self._initialize_push(image)
+ layer_path = os.path.join(tmp_dir_name, layer)
+ self._push_layer(layer_path, push_url)
+
+ # then, push image config
+ self._logger.info(
+ 'Pushing image config', image=image, config_loc=config_loc
+ )
+ push_url = self._initialize_push(image)
+ config_path = os.path.join(tmp_dir_name, config_loc)
+ self._push_config(config_path, push_url)
+
+ # keep the pushed layers
+ properly_formatted_layers = [os.path.join(tmp_dir_name, layer) for layer in layers]
+
+ # Now we need to create and push a manifest for the image
+ creator = manifest_creator.ImageManifestCreator(
+ config_path, properly_formatted_layers
+ )
+ image_manifest = creator.create()
+
+ # Override tags if needed: from --replace-tags-match and --replace-tags-target
+ tag = self._replace_tag(image, tag)
+
+ self._logger.info('Pushing image manifest', image=image, tag=tag)
+ self._push_manifest(image_manifest, image, tag)
+ self._logger.info('Image Pushed', image=image, tag=tag)
+
+ def _get_manifest_from_tar(self):
+ return self._extract_json_from_tar(self._archive_path, "manifest.json")
+
+ def _get_config_from_tar(self, name):
+ return self._extract_json_from_tar(self._archive_path, name)
+
+ def _extract_json_from_tar(self, tar_filepath, file_to_parse):
+ loaded = self._extract_file_from_tar(tar_filepath, file_to_parse)
+ stringified = self._parse_as_utf8(loaded)
+ return json.loads(stringified)
+
+ @staticmethod
+ def _extract_file_from_tar(tar_filepath, file_to_extract):
+ manifest = tarfile.open(tar_filepath)
+ file_contents = manifest.extractfile(file_to_extract)
+ return file_contents
+
+ @staticmethod
+ def _parse_as_utf8(to_parse):
+ as_str = (to_parse.read()).decode("utf-8")
+ to_parse.close()
+ return as_str
+
+ def _conditional_print(self, what, end=None):
+ if self._stream:
+ if end:
+ print(what, end=end)
+ else:
+ print(what)
+
+ def _extract_tar_file(self, tmp_dir_name):
+ with tarfile.open(self._archive_path) as fh:
+ fh.extractall(tmp_dir_name)
+
+ def _push_manifest(self, manifest, image, tag):
+ headers = {
+ "Content-Type": "application/vnd.docker.distribution.manifest.v2+json"
+ }
+ url = self._registry_url + "/v2/" + image + "/manifests/" + tag
+ r = requests.put(
+ url,
+ headers=headers,
+ data=manifest,
+ auth=self._basicauth,
+ verify=self._ssl_verify,
+ )
+ if r.status_code != 201:
+ self._logger.log_and_raise(
+ 'error', 'Failed to push manifest', image=image, tag=tag
+ )
+
+ def _initialize_push(self, repository):
+ """
+ Request a push URL for the image repository for a layer or manifest
+ """
+ self._logger.debug('Initializing push', repository=repository)
+
+ response = requests.post(
+ self._registry_url + "/v2/" + repository + "/blobs/uploads/",
+ auth=self._basicauth,
+ verify=self._ssl_verify,
+ )
+ upload_url = None
+ if response.headers.get("Location", None):
+ upload_url = response.headers.get("Location")
+ success = response.status_code == 202
+ if not success:
+ self._logger.log_and_raise(
+ 'error',
+ 'Failed to initialize push',
+ status_code=response.status_code,
+ contents=response.content,
+ )
+ return upload_url
+
+ def _push_layer(self, layer_path, upload_url):
+ self._chunked_upload(layer_path, upload_url)
+
+ def _push_config(self, layer_path, upload_url):
+ self._chunked_upload(layer_path, upload_url)
+
+ def _chunked_upload(self, filepath, url):
+ content_path = os.path.abspath(filepath)
+ content_size = os.stat(content_path).st_size
+ with open(content_path, "rb") as f:
+ index = 0
+ headers = {}
+ upload_url = url
+ sha256hash = hashlib.sha256()
+
+ for chunk in self._read_in_chunks(f, sha256hash):
+ if "http" not in upload_url:
+ upload_url = self._registry_url + upload_url
+ offset = index + len(chunk)
+ headers['Content-Type'] = 'application/octet-stream'
+ headers['Content-Length'] = str(len(chunk))
+ headers['Content-Range'] = '%s-%s' % (index, offset)
+ index = offset
+ last = False
+ if offset == content_size:
+ last = True
+ try:
+ self._conditional_print(
+ "Pushing... "
+ + str(round((offset / content_size) * 100, 2))
+ + "% ",
+ end="\r",
+ )
+ if last:
+ digest_str = str(sha256hash.hexdigest())
+ requests.put(
+ f"{upload_url}&digest=sha256:{digest_str}",
+ data=chunk,
+ headers=headers,
+ auth=self._basicauth,
+ verify=self._ssl_verify,
+ )
+ else:
+ response = requests.patch(
+ upload_url,
+ data=chunk,
+ headers=headers,
+ auth=self._basicauth,
+ verify=self._ssl_verify,
+ )
+ if "Location" in response.headers:
+ upload_url = response.headers["Location"]
+
+ except Exception as exc:
+ self._logger.error(
+ 'Failed to upload file image upload', filepath=filepath, exc=exc
+ )
+ raise
+ f.close()
+
+ self._conditional_print("")
+
+ # chunk size default 2T (??)
+ @staticmethod
+ def _read_in_chunks(file_object, hashed, chunk_size=2097152):
+ while True:
+ data = file_object.read(chunk_size)
+ hashed.update(data)
+ if not data:
+ break
+ yield data
+
+ @staticmethod
+ def _parse_image_tag(image_ref):
+
+ # should be 2 parts exactly
+ image, tag = image_ref.split(":")
+ return image, tag
+
+ def _replace_tag(self, image, orig_tag):
+ if self._replace_tags_match and self._replace_tags_match:
+ match_regex = re.compile(self._replace_tags_match)
+ if match_regex.match(orig_tag):
+ self._logger.info(
+ 'Replacing tag for image',
+ image=image,
+ orig_tag=orig_tag,
+ new_tag=self._replace_tags_target,
+ )
+ return self._replace_tags_target
+ else:
+ self._logger.debug(
+ 'Replace tag match given but did not match',
+ image=image,
+ orig_tag=orig_tag,
+ replace_tags_match=self._replace_tags_match,
+ new_tag=self._replace_tags_target,
+ )
+
+ return orig_tag
diff --git a/dockerregistrypusher b/dockerregistrypusher
new file mode 100755
index 0000000..3dd686e
--- /dev/null
+++ b/dockerregistrypusher
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+"""
+NOTE: It is important to keep this part BC with py2 as well
+That is the reason we use py2 string formatting.
+The reason is, python 2 code, that would call manof, would use this entrypoint and thus it should be BC with py2
+"""
+
+import os
+import sys
+import subprocess
+
+
+args = ' '.join(sys.argv[1:])
+args = args.replace('\"', '\\\"')
+manof_path = os.path.join(os.path.dirname(os.path.realpath(__file__)))
+
+try:
+ subprocess.check_call(
+ 'source {0}/venv/bin/activate && python3 {0}/dockerregistrypusher.py {1}'.format(manof_path, args),
+ shell=True,
+ executable='/bin/bash')
+
+except Exception as exc:
+
+ # by default: save us the traceback
+ # DEV NOTE: if you need to debug dockerregistrypusher, you might want to:
+ # - uncomment this
+ # - print the full CalledProcessError
+ # traceback.print_exc()
+
+ sys.exit(1)
diff --git a/dockerregistrypusher.py b/dockerregistrypusher.py
new file mode 100644
index 0000000..0af8239
--- /dev/null
+++ b/dockerregistrypusher.py
@@ -0,0 +1,128 @@
+import argparse
+import sys
+
+import core
+import clients.logging
+
+
+def run(args):
+ retval = 1
+
+ # plug in verbosity shorthands
+ if args.v:
+ args.log_severity = 'debug'
+
+ logger = clients.logging.Client(
+ 'pusher',
+ initial_severity=args.log_severity,
+ initial_console_severity=args.log_console_severity,
+ initial_file_severity=args.log_file_severity,
+ output_stdout=not args.log_disable_stdout,
+ output_dir=args.log_output_dir,
+ max_log_size_mb=args.log_file_rotate_max_file_size,
+ max_num_log_files=args.log_file_rotate_num_files,
+ log_file_name=args.log_file_name,
+ log_colors=args.log_colors,
+ ).logger
+
+ # start root logger with kwargs and create manof
+ reg_client = core.Registry(
+ logger=logger,
+ archive_path=args.archive_path,
+ registry_url=args.registry_url,
+ stream=args.stream,
+ login=args.login,
+ password=args.password,
+ ssl_verify=args.ssl_verify,
+ replace_tags_match=args.replace_tags_match,
+ replace_tags_target=args.replace_tags_target,
+ )
+ reg_client.process_archive()
+ if logger.first_error is None:
+ retval = 0
+
+ return retval
+
+
+def register_arguments(parser):
+ # global options for manof
+ clients.logging.Client.register_arguments(parser)
+
+ # verbosity shorthands
+ parser.add_argument(
+ '-v',
+ help='Set log level to debug (same as --log-severity=debug)',
+ action='store_true',
+ default=False,
+ )
+
+ parser.add_argument(
+ 'archive_path',
+ metavar='ARCHIVE_PATH',
+ type=str,
+ help='The url of the target registry to push to',
+ )
+
+ parser.add_argument(
+ 'registry_url',
+ metavar='REGISTRY_URL',
+ type=str,
+ help='The url of the target registry to push to',
+ )
+ parser.add_argument(
+ '-l',
+ '--login',
+ help='Basic-auth login name for registry',
+ required=False,
+ )
+
+ parser.add_argument(
+ '-p',
+ '--password',
+ help='Basic-auth login password for registry',
+ required=False,
+ )
+
+ parser.add_argument(
+ '--ssl-verify',
+ help='Skip SSL verification of the registry',
+ type=bool,
+ default=True,
+ )
+
+ parser.add_argument(
+ '--stream',
+ help='Add some streaming logging during push',
+ type=bool,
+ default=True,
+ )
+
+ parser.add_argument(
+ '--replace-tags-match',
+ help='A regex string to match on tags. If matches will be replaces with --replace-tags-target',
+ type=str,
+ required=False,
+ )
+
+ parser.add_argument(
+ '--replace-tags-target',
+ help='If an image tag matches value of --replace-tags-match value, replace it with this tag',
+ type=str,
+ required=False,
+ )
+
+
+if __name__ == '__main__':
+ # create an argument parser
+ arg_parser = argparse.ArgumentParser()
+
+ # register all arguments and sub-commands
+ register_arguments(arg_parser)
+
+ parsed_args = arg_parser.parse_args()
+
+ # parse the known args, seeing how the targets may add arguments of their own and re-parse
+ ret_val = run(parsed_args)
+
+ # return value
+ sys.exit(ret_val)
diff --git a/install b/install
new file mode 100755
index 0000000..c5b2cd2
--- /dev/null
+++ b/install
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+import subprocess
+
+
+def run(command):
+ subprocess.check_call(['/bin/bash', '-c', command],
+ stdout=sys.stdout, stderr=sys.stderr)
+
+
+def register_arguments():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument('-o', '--offline-install-path', help='The dir path to install pip packages from.')
+ parser.add_argument('--no-link', action='store_true',
+ help='Do not remove or create symlink')
+ parser.add_argument('--dev', action='store_true',
+ help='Install dev requirements in addition to common requirements')
+
+ return parser.parse_args()
+
+
+def _create_sym_link():
+
+ # symlink to run
+ ln_cmd = 'ln -sfF {0}/dockerregistrypusher /usr/local/bin/dockerregistrypusher'.format(os.getcwd())
+
+ # first try without sudo, then with
+ try:
+ run(ln_cmd)
+ except subprocess.CalledProcessError:
+ run('sudo {0}'.format(ln_cmd))
+
+
+def main():
+ args = register_arguments()
+ if args.offline_install_path is None:
+ local_pip_packages = ''
+ else:
+ local_pip_packages = '--no-index --find-links=file:{}'.format(
+ args.offline_install_path)
+
+ retval = 0
+ try:
+ requirements_file = 'requirements.txt'
+ if args.dev:
+ requirements_file = 'requirements_all.txt'
+
+ # create venv, activate it and install requirements to it
+ # (from a local dir if it exists)
+ # "python -m pip install" instead of "pip install" handles a pip
+ # issue where it fails in a long-named dir
+ run('virtualenv --python=python3 venv && '
+ 'source venv/bin/activate && '
+ 'python -m pip install {0} incremental && '.format(local_pip_packages) +
+ 'python -m pip install {0} -r {1}'.format(local_pip_packages, requirements_file))
+
+ if not args.no_link:
+ _create_sym_link()
+ except subprocess.CalledProcessError as e:
+ retval = e.returncode
+ else:
+ print('Installation complete. Enjoy your dockerregistrypusher!')
+
+ sys.exit(retval)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..1bcee9a
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,10 @@
+# please keep this to a minimum - defaults are good
+[tool.black]
+target-version = ['py36', 'py37', 'py38']
+exclude = '''
+/(
+ .git
+ | .venv
+ | venv
+)/
+'''
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..c58f7cb
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+-r requirements/common.txt
diff --git a/requirements/common.txt b/requirements/common.txt
new file mode 100644
index 0000000..7bba6db
--- /dev/null
+++ b/requirements/common.txt
@@ -0,0 +1,6 @@
+requests==2.25.1
+urllib3==1.26.2
+simplejson==3.8.2
+colorama==0.4.4
+pygments==2.2.0
+incremental==17.5.0
diff --git a/requirements/dev.txt b/requirements/dev.txt
new file mode 100644
index 0000000..adbb807
--- /dev/null
+++ b/requirements/dev.txt
@@ -0,0 +1,5 @@
+mock~=4.0
+flake8~=3.8
+inflection~=0.5
+nose~=1.3
+git+git://github.com/psf/black
diff --git a/requirements_all.txt b/requirements_all.txt
new file mode 100644
index 0000000..a4938df
--- /dev/null
+++ b/requirements_all.txt
@@ -0,0 +1,2 @@
+-r requirements/dev.txt
+-r requirements/common.txt
diff --git a/tests/pusher_tests.py b/tests/pusher_tests.py
new file mode 100644
index 0000000..bbc84e6
--- /dev/null
+++ b/tests/pusher_tests.py
@@ -0,0 +1,95 @@
+import unittest
+import requests
+import docker
+from dockertarpusher import Registry
+
+
+class TestPusher(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ print("Setuping")
+ print("Pulling registry...")
+ client = docker.from_env()
+ client.images.pull("registry:2.7.1")
+
+ @classmethod
+ def tearDownClass(self):
+ print("Cleaning up")
+ client = docker.from_env()
+ try:
+ client.containers.get("registrytest").remove(force=True)
+ except:
+ print("")
+ try:
+ client.volumes.get("registrytestvol").remove(force=True)
+ except:
+ print("")
+
+ def startRegistry(self):
+ client = docker.from_env()
+ client.containers.run(
+ "registry:2.7.1",
+ detach=True,
+ ports={"5000/tcp": 5000},
+ name="registrytest",
+ volumes={"registrytestvol": {"bind": "/var/lib/registry", "mode": "rw"}},
+ )
+
+ def stopRegistry(self):
+ client = docker.from_env()
+ client.containers.get("registrytest").remove(force=True)
+ client.volumes.get("registrytestvol").remove(force=True)
+
+ def setUp(self):
+ print("Starting clean registry")
+ try:
+ self.stopRegistry()
+ except:
+ print("Cannot stop registry, probably not exists")
+ self.startRegistry()
+
+ def testOneLayer(self):
+ registryUrl = "http://localhost:5000"
+ reg = Registry(registryUrl, "tests/busybox.tar")
+ reg.processImage()
+ r = requests.get(registryUrl + "/v2/_catalog")
+ self.assertTrue("razikus/busybox" in r.json()["repositories"])
+ r = requests.get(registryUrl + "/v2/razikus/busybox/tags/list")
+ self.assertTrue("razikus/busybox" == r.json()["name"])
+ self.assertTrue("1.31" in r.json()["tags"])
+
+ def testOneLayerAndRun(self):
+ registryUrl = "http://localhost:5000"
+ reg = Registry(registryUrl, "tests/busybox.tar")
+ reg.processImage()
+ r = requests.get(registryUrl + "/v2/_catalog")
+ self.assertTrue("razikus/busybox" in r.json()["repositories"])
+ r = requests.get(registryUrl + "/v2/razikus/busybox/tags/list")
+ self.assertTrue("razikus/busybox" == r.json()["name"])
+ self.assertTrue("1.31" in r.json()["tags"])
+ client = docker.from_env()
+ client.images.pull("localhost:5000/razikus/busybox:1.31")
+ client.containers.run("localhost:5000/razikus/busybox:1.31", remove=True)
+ client.images.remove("localhost:5000/razikus/busybox:1.31")
+
+ def testMultipleLayersWithDockerSave(self):
+ client = docker.from_env()
+ client.images.pull("razikus/whoami:slim80")
+ image = client.images.get("razikus/whoami:slim80")
+ f = open("tests/whoami.tar", "wb")
+ for chunk in image.save(named=True):
+ f.write(chunk)
+ f.close()
+
+ registryUrl = "http://localhost:5000"
+ reg = Registry(registryUrl, "tests/whoami.tar")
+ reg.processImage()
+ r = requests.get(registryUrl + "/v2/_catalog")
+ self.assertTrue("razikus/whoami" in r.json()["repositories"])
+ r = requests.get(registryUrl + "/v2/razikus/whoami/tags/list")
+ self.assertTrue("razikus/whoami" == r.json()["name"])
+ self.assertTrue("slim80" in r.json()["tags"])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO b/tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO
new file mode 100644
index 0000000..37d454f
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.egg-info/PKG-INFO
@@ -0,0 +1,11 @@
+Metadata-Version: 1.1
+Name: flake8-igz
+Version: 0.1.0
+Summary: iguazio's twisted extension to flake8
+Home-page: UNKNOWN
+Author: Adam Melnick
+Author-email: adamm@iguazio.com
+License: MIT
+Description: UNKNOWN
+Platform: UNKNOWN
+Provides: flake8_igz
diff --git a/tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt b/tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt
new file mode 100644
index 0000000..65ce5d0
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.egg-info/SOURCES.txt
@@ -0,0 +1,8 @@
+flake8_igz.py
+setup.py
+flake8_igz.egg-info/PKG-INFO
+flake8_igz.egg-info/SOURCES.txt
+flake8_igz.egg-info/dependency_links.txt
+flake8_igz.egg-info/entry_points.txt
+flake8_igz.egg-info/requires.txt
+flake8_igz.egg-info/top_level.txt
\ No newline at end of file
diff --git a/tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt b/tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt b/tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt
new file mode 100644
index 0000000..a40a758
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.egg-info/entry_points.txt
@@ -0,0 +1,8 @@
+[flake8.extension]
+flake8-igz.class_name_camel_case = flake8_igz:class_name_camel_case
+flake8-igz.ctx_log_non_string_first_param = flake8_igz:ctx_log_non_string_first_param
+flake8-igz.logger_forbid_passing_self = flake8_igz:logger_forbid_passing_self
+flake8-igz.multiline_string_double_quotes = flake8_igz:multiline_string_double_quotes
+flake8-igz.multiline_string_on_newline = flake8_igz:multiline_string_on_newline
+flake8-igz.single_quote_strings = flake8_igz:single_quote_strings
+
diff --git a/tools/flake8_plugin/flake8_igz.egg-info/requires.txt b/tools/flake8_plugin/flake8_igz.egg-info/requires.txt
new file mode 100644
index 0000000..7607850
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.egg-info/requires.txt
@@ -0,0 +1 @@
+flake8>3.0.0
diff --git a/tools/flake8_plugin/flake8_igz.egg-info/top_level.txt b/tools/flake8_plugin/flake8_igz.egg-info/top_level.txt
new file mode 100644
index 0000000..0ff4173
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.egg-info/top_level.txt
@@ -0,0 +1 @@
+flake8_igz
diff --git a/tools/flake8_plugin/flake8_igz.py b/tools/flake8_plugin/flake8_igz.py
new file mode 100644
index 0000000..d739392
--- /dev/null
+++ b/tools/flake8_plugin/flake8_igz.py
@@ -0,0 +1,99 @@
+import inflection
+import token
+import re
+
+
+class Constants(object):
+
+ string_prefixes = 'ubrUBR'
+ log_levels = ['verbose', 'debug', 'info', 'warn', 'error']
+
+
+class Utils(object):
+ @staticmethod
+ def get_string_tokens(tokens):
+ for tid, lexeme, start, end, _ in tokens:
+ if tid == token.STRING:
+ lexeme = lexeme.lstrip(Constants.string_prefixes)
+ yield lexeme, start, end
+
+
+def single_quote_strings(logical_line, tokens):
+ for (
+ lexeme,
+ start,
+ _,
+ ) in Utils.get_string_tokens(tokens):
+ if lexeme.startswith('"') and not lexeme.startswith('"""'):
+ yield start, 'I100 double-quote string used (expected single-quote)'
+
+
+def multiline_string_on_newline(logical_line, tokens):
+ for (
+ lexeme,
+ start,
+ end,
+ ) in Utils.get_string_tokens(tokens):
+ if lexeme.startswith('"""'):
+ if not re.match(r'^\"\"\"\n', lexeme):
+ yield start, 'I101 multiline string must start on next line after triple double-quotes'
+ if not re.search(r'\n\s*\"\"\"$', lexeme):
+ yield end, 'I102 multiline string must end with triple double-quotes in new line'
+
+
+def multiline_string_double_quotes(logical_line, tokens):
+ for (
+ lexeme,
+ start,
+ _,
+ ) in Utils.get_string_tokens(tokens):
+ if lexeme.startswith('\'\'\''):
+ yield start, 'I103 triple single-quotes used in multiline string (expected triple double-quotes)'
+
+
+def ctx_log_non_string_first_param(logical_line, tokens):
+ if logical_line.startswith('ctx.log.'):
+ for idx, (tid, lexeme, start, _, _) in enumerate(tokens):
+ if tid == token.NAME and lexeme in Constants.log_levels:
+
+ # plus one for the ( parentheses, plus one for the first param
+ first_param_token = tokens[idx + 2]
+
+ if first_param_token[0] == token.STRING:
+ yield first_param_token[
+ 2
+ ], 'I104 ctx.log.{0} call with string as first param'.format(lexeme)
+
+
+def class_name_camel_case(logical_line, tokens):
+ if logical_line.startswith('class'):
+ for idx, (tid, lexeme, start, _, _) in enumerate(tokens):
+ if tid == token.NAME and lexeme == 'class':
+ class_name_token = tokens[idx + 1]
+ camelized = inflection.camelize(class_name_token[1], True)
+
+ if class_name_token[1] != camelized:
+ yield class_name_token[
+ 2
+ ], 'I105 class name not camel case. (suggestion: {0})'.format(
+ camelized
+ )
+
+
+def logger_forbid_passing_self(logical_line, tokens):
+ if logical_line.startswith('self._logger.'):
+ for idx, (tid, lexeme, start, _, _) in enumerate(tokens):
+ if tid == token.NAME and lexeme in Constants.log_levels:
+
+ # plus one for the ( parentheses, plus one for the first param
+ first_param_token = tokens[idx + 2]
+
+ if (
+ first_param_token[1] == 'self'
+ and first_param_token[0] != token.STRING
+ ):
+ yield first_param_token[
+ 2
+ ], 'I106 self._logger.{0} call with self as first param'.format(
+ lexeme
+ )
diff --git a/tools/flake8_plugin/setup.py b/tools/flake8_plugin/setup.py
new file mode 100644
index 0000000..8992eff
--- /dev/null
+++ b/tools/flake8_plugin/setup.py
@@ -0,0 +1,35 @@
+from __future__ import with_statement
+import setuptools
+
+requires = [
+ 'flake8 > 3.0.0',
+]
+
+flake8_entry_point = 'flake8.extension'
+
+setuptools.setup(
+ name='flake8-igz',
+ license='MIT',
+ version='0.1.0',
+ description='iguazio\'s twisted extension to flake8',
+ author='Adam Melnick',
+ author_email='adamm@iguazio.com',
+ provides=['flake8_igz'],
+ py_modules=['flake8_igz'],
+ install_requires=requires,
+ entry_points={
+ flake8_entry_point: [
+ 'flake8-igz.single_quote_strings = flake8_igz:single_quote_strings',
+ 'flake8-igz.multiline_string_on_newline ='
+ ' flake8_igz:multiline_string_on_newline',
+ 'flake8-igz.multiline_string_double_quotes ='
+ ' flake8_igz:multiline_string_double_quotes',
+ 'flake8-igz.ctx_log_non_string_first_param ='
+ ' flake8_igz:ctx_log_non_string_first_param',
+ 'flake8-igz.class_name_camel_case = flake8_igz:class_name_camel_case',
+ 'flake8-igz.logger_forbid_passing_self ='
+ ' flake8_igz:logger_forbid_passing_self',
+ ],
+ },
+ classifiers=[],
+)
diff --git a/upload.sh b/upload.sh
new file mode 100644
index 0000000..25b17dd
--- /dev/null
+++ b/upload.sh
@@ -0,0 +1 @@
+python3 -m twine upload dist/* --verbose