diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d62f174..0920f6dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,35 @@ # Change Log -## [1.2.1](https://github.com/pyouroboros/ouroboros/tree/1.2.1) (2019-02-13) +## [1.3.0](https://github.com/pyouroboros/ouroboros/tree/1.3.0) (2019-02-25) +[Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.2.1...1.3.0) + +**Implemented enhancements:** + +- Start new container in detached mode [\#222](https://github.com/pyouroboros/ouroboros/pull/222) ([nightvisi0n](https://github.com/nightvisi0n)) +- Optimise dockerfile layers [\#218](https://github.com/pyouroboros/ouroboros/pull/218) ([nightvisi0n](https://github.com/nightvisi0n)) + +**Fixed bugs:** + +- Catch Failed self-updates [\#230](https://github.com/pyouroboros/ouroboros/issues/230) +- Cron scheduled missed following successful runs [\#229](https://github.com/pyouroboros/ouroboros/issues/229) +- Catch attribute.id error [\#226](https://github.com/pyouroboros/ouroboros/issues/226) +- AttachStdout and AttachStderr are not carried over properly [\#221](https://github.com/pyouroboros/ouroboros/issues/221) +- Exception when updating container started with --rm \(autoremove\) [\#219](https://github.com/pyouroboros/ouroboros/issues/219) +- Issue with Swarm Mode V2 [\#216](https://github.com/pyouroboros/ouroboros/issues/216) +- Fix docker swarm mode [\#227](https://github.com/pyouroboros/ouroboros/pull/227) ([mathcantin](https://github.com/mathcantin)) + +**Other Pull Requests** + +- v1.3.0 Merge [\#241](https://github.com/pyouroboros/ouroboros/pull/241) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- v1.3.0 to develop [\#240](https://github.com/pyouroboros/ouroboros/pull/240) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) +- Catch self update apierror [\#238](https://github.com/pyouroboros/ouroboros/pull/238) ([circa10a](https://github.com/circa10a)) +- Catch attribute error [\#237](https://github.com/pyouroboros/ouroboros/pull/237) ([circa10a](https://github.com/circa10a)) +- Check for autoremove [\#236](https://github.com/pyouroboros/ouroboros/pull/236) ([circa10a](https://github.com/circa10a)) +- Add misfire\_grace\_time for cron scheduler [\#234](https://github.com/pyouroboros/ouroboros/pull/234) ([circa10a](https://github.com/circa10a)) +- Check all services by default on swarm mode [\#228](https://github.com/pyouroboros/ouroboros/pull/228) [[cleanup](https://github.com/pyouroboros/ouroboros/labels/cleanup)] ([mathcantin](https://github.com/mathcantin)) +- remove git in pypi + branch develop + version bump + maintainer\_email [\#214](https://github.com/pyouroboros/ouroboros/pull/214) ([DirtyCajunRice](https://github.com/DirtyCajunRice)) + +## [1.2.1](https://github.com/pyouroboros/ouroboros/tree/1.2.1) (2019-02-14) [Full Changelog](https://github.com/pyouroboros/ouroboros/compare/1.2.0...1.2.1) **Fixed bugs:** diff --git a/Dockerfile b/Dockerfile index 1cda4204..a5fce1ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM amd64/python:3.7.2-alpine -LABEL maintainers="dirtycajunrice,circa10a,tkdeviant" +LABEL maintainers="dirtycajunrice,circa10a" ENV TZ UTC @@ -8,9 +8,11 @@ WORKDIR /app COPY /requirements.txt /setup.py /ouroboros /README.md /app/ +RUN apk add --no-cache tzdata && \ + pip install --no-cache-dir -r requirements.txt + COPY /pyouroboros /app/pyouroboros -RUN apk add --no-cache tzdata && \ - pip install --no-cache-dir . +RUN pip install --no-cache-dir . ENTRYPOINT ["ouroboros"] \ No newline at end of file diff --git a/Dockerfile.arm b/Dockerfile.arm index 2d3e876c..f027b096 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -1,6 +1,6 @@ FROM arm32v6/python:3.7.2-alpine -LABEL maintainers="dirtycajunrice,circa10a,tkdeviant" +LABEL maintainers="dirtycajunrice,circa10a" ENV TZ UTC @@ -8,9 +8,11 @@ WORKDIR /app COPY /requirements.txt /setup.py /ouroboros /README.md /app/ +RUN apk add --no-cache tzdata && \ + pip install --no-cache-dir -r requirements.txt + COPY /pyouroboros /app/pyouroboros -RUN apk add --no-cache tzdata && \ - pip install --no-cache-dir . +RUN pip install --no-cache-dir . ENTRYPOINT ["ouroboros"] \ No newline at end of file diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 6b99bf36..dd3bd512 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,6 +1,6 @@ FROM arm64v8/python:3.7.2-alpine -LABEL maintainers="dirtycajunrice,circa10a,tkdeviant" +LABEL maintainers="dirtycajunrice,circa10a" ENV TZ UTC @@ -8,9 +8,11 @@ WORKDIR /app COPY /requirements.txt /setup.py /ouroboros /README.md /app/ +RUN apk add --no-cache tzdata && \ + pip install --no-cache-dir -r requirements.txt + COPY /pyouroboros /app/pyouroboros -RUN apk add --no-cache tzdata && \ - pip install --no-cache-dir . +RUN pip install --no-cache-dir . ENTRYPOINT ["ouroboros"] \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index d08646d3..ccfbfdf4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -127,8 +127,6 @@ pipeline { sh """ python3 -m venv venv && venv/bin/pip install twine venv/bin/python setup.py sdist && venv/bin/python -m twine upload --skip-existing -u ${PYPI_CREDS_USR} -p ${PYPI_CREDS_PSW} dist/* - git tag ${TAG} - git push --tags """ } } diff --git a/README.md b/README.md index f30f4ef2..928ebfee 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![BuyUsCoffee](https://img.shields.io/badge/BuyMeACoffee-Donate-ff813f.svg?logo=CoffeeScript&style=flat-square)](https://buymeacoff.ee/ouroboros) [![Build Status](https://jenkins.cajun.pro/buildStatus/icon?job=Ouroboros/master)](https://jenkins.cajun.pro/job/Ouroboros/job/master/) [![Release](https://img.shields.io/github/release/pyouroboros/ouroboros.svg?style=flat-square)](https://hub.docker.com/r/pyouroboros/ouroboros/) -[![Pypi Downloads](https://img.shields.io/pypi/dm/ouroboros-cli.svg?style=flat-square)](https://pypi.org/project/ouroboros-cli/) [![Python Version](https://img.shields.io/pypi/pyversions/ouroboros-cli.svg?style=flat-square)](https://pypi.org/project/ouroboros-cli/) [![Docker Pulls](https://img.shields.io/docker/pulls/pyouroboros/ouroboros.svg?style=flat-square)](https://hub.docker.com/r/pyouroboros/ouroboros/) [![Layers](https://images.microbadger.com/badges/image/pyouroboros/ouroboros.svg)](https://microbadger.com/images/pyouroboros/ouroboros) diff --git a/pyouroboros/__init__.py b/pyouroboros/__init__.py index 0ffeb46f..6a92b727 100644 --- a/pyouroboros/__init__.py +++ b/pyouroboros/__init__.py @@ -1,2 +1,2 @@ -VERSION = "1.2.1" +VERSION = "1.3.0" BRANCH = "master" diff --git a/pyouroboros/dockerclient.py b/pyouroboros/dockerclient.py index d43c3d88..9385f84e 100644 --- a/pyouroboros/dockerclient.py +++ b/pyouroboros/dockerclient.py @@ -4,7 +4,7 @@ from os.path import isdir, isfile, join from docker.errors import DockerException, APIError, NotFound -from pyouroboros.helpers import set_properties +from pyouroboros.helpers import set_properties, remove_sha_prefix, get_digest class Docker(object): @@ -55,7 +55,7 @@ def connect(self): return client -class Container(object): +class BaseImageObject(object): def __init__(self, docker_client): self.docker = docker_client self.logger = self.docker.logger @@ -66,6 +66,44 @@ def __init__(self, docker_client): self.data_manager.total_updated[self.socket] = 0 self.notification_manager = self.docker.notification_manager + def _pull(self, tag): + """Docker pull image tag""" + self.logger.debug('Checking tag: %s', tag) + try: + if self.config.auth_json: + self.client.login(self.config.auth_json.get( + "username"), self.config.auth_json.get("password")) + + if self.config.dry_run: + # The authentication doesn't work with this call + # See bugs https://github.com/docker/docker-py/issues/2225 + return self.client.images.get_registry_data(tag) + else: + return self.client.images.pull(tag) + except APIError as e: + if '' in str(e): + self.logger.debug("Docker api issue. Ignoring") + raise ConnectionError + elif 'unauthorized' in str(e): + if self.config.dry_run: + self.logger.error('dry run : Upstream authentication issue while checking %s. See: ' + 'https://github.com/docker/docker-py/issues/2225', tag) + raise ConnectionError + else: + self.logger.critical("Invalid Credentials. Exiting") + exit(1) + elif 'Client.Timeout' in str(e): + self.logger.critical( + "Couldn't find an image on docker.com for %s. Local Build?", tag) + raise ConnectionError + elif ('pull access' or 'TLS handshake') in str(e): + self.logger.critical("Couldn't pull. Skipping. Error: %s", e) + raise ConnectionError + + +class Container(BaseImageObject): + def __init__(self, docker_client): + super().__init__(docker_client) self.monitored = self.monitor_filter() # Container sub functions @@ -135,35 +173,7 @@ def pull(self, current_tag): raise ConnectionError elif ':' not in tag: tag = f'{tag}:latest' - self.logger.debug('Checking tag: %s', tag) - try: - if self.config.dry_run: - registry_data = self.client.images.get_registry_data(tag) - return registry_data - else: - if self.config.auth_json: - return_image = self.client.images.pull(tag, auth_config=self.config.auth_json) - else: - return_image = self.client.images.pull(tag) - return return_image - except APIError as e: - if '' in str(e): - self.logger.debug("Docker api issue. Ignoring") - raise ConnectionError - elif 'unauthorized' in str(e): - if self.config.dry_run: - self.logger.error('dry run : Upstream authentication issue while checking %s. See: ' - 'https://github.com/docker/docker-py/issues/2225', tag) - raise ConnectionError - else: - self.logger.critical("Invalid Credentials. Exiting") - exit(1) - elif 'Client.Timeout' in str(e): - self.logger.critical("Couldn't find an image on docker.com for %s. Local Build?", tag) - raise ConnectionError - elif ('pull access' or 'TLS handshake') in str(e): - self.logger.critical("Couldn't pull. Skipping. Error: %s", e) - raise ConnectionError + return self._pull(tag) # Filters def running_filter(self): @@ -176,7 +186,10 @@ def running_filter(self): else: try: if 'ouroboros' not in container.image.tags[0]: - running_containers.append(container) + if container.attrs['HostConfig']['AutoRemove']: + self.logger.debug("Skipping %s due to --rm property.", container.name) + else: + running_containers.append(container) except IndexError: self.logger.error("%s has no tags.. you should clean it up! Ignoring.", container.id) continue @@ -238,10 +251,13 @@ def socket_check(self): latest_image = self.pull(current_tag) except ConnectionError: continue - if current_image.id != latest_image.id: - updateable.append((container, current_image, latest_image)) - else: - continue + try: + if current_image.id != latest_image.id: + updateable.append((container, current_image, latest_image)) + else: + continue + except AttributeError: + self.logger.error("Issue detecting %s's image tag. Skipping...", container.name) # Get container list to restart after update complete depends_on = container.labels.get('com.ouroboros.depends_on', False) @@ -342,25 +358,22 @@ def update_self(self, count=None, old_container=None, me_list=None, new_image=No self.logger.debug('I need to update! Starting the ouroboros ;)') self_name = 'ouroboros-updated' if old_container.name == 'ouroboros' else 'ouroboros' new_config = set_properties(old=old_container, new=new_image, self_name=self_name) - me_created = self.client.api.create_container(**new_config) - new_me = self.client.containers.get(me_created.get("Id")) - new_me.start() - self.logger.debug('If you strike me down, I shall become more powerful than you could possibly imagine') - self.logger.debug('https://bit.ly/2VVY7GH') - sleep(30) + try: + me_created = self.client.api.create_container(**new_config) + new_me = self.client.containers.get(me_created.get("Id")) + new_me.start() + self.logger.debug('If you strike me down, I shall become \ + more powerful than you could possibly imagine.') + self.logger.debug('https://bit.ly/2VVY7GH') + sleep(30) + except APIError as e: + self.logger.error("Self update failed.") + self.logger.error(e) -class Service(object): +class Service(BaseImageObject): def __init__(self, docker_client): - self.docker = docker_client - self.logger = self.docker.logger - self.config = self.docker.config - self.client = self.docker.client - self.socket = self.docker.socket - self.data_manager = self.docker.data_manager - self.data_manager.total_updated[self.socket] = 0 - self.notification_manager = self.docker.notification_manager - + super().__init__(docker_client) self.monitored = self.monitor_filter() def monitor_filter(self): @@ -371,7 +384,7 @@ def monitor_filter(self): for service in services: ouro_label = service.attrs['Spec']['Labels'].get('com.ouroboros.enable') - if ouro_label.lower() in ["true", "yes"]: + if not self.config.label_enable or ouro_label.lower() in ["true", "yes"]: monitored_services.append(service) self.data_manager.monitored_containers[self.socket] = len(monitored_services) @@ -381,38 +394,9 @@ def monitor_filter(self): def pull(self, tag): """Docker pull image tag""" - self.logger.debug('Checking tag: %s', tag) - try: - if self.config.dry_run: - registry_data = self.client.images.get_registry_data(tag) - return registry_data - else: - if self.config.auth_json: - return_image = self.client.images.pull(tag, auth_config=self.config.auth_json) - else: - return_image = self.client.images.pull(tag) - return return_image - except APIError as e: - if '' in str(e): - self.logger.debug("Docker api issue. Ignoring") - raise ConnectionError - elif 'unauthorized' in str(e): - if self.config.dry_run: - self.logger.error('dry run : Upstream authentication issue while checking %s. See: ' - 'https://github.com/docker/docker-py/issues/2225', tag) - raise ConnectionError - else: - self.logger.critical("Invalid Credentials. Exiting") - exit(1) - elif 'Client.Timeout' in str(e): - self.logger.critical("Couldn't find an image on docker.com for %s. Local Build?", tag) - raise ConnectionError - elif ('pull access' or 'TLS handshake') in str(e): - self.logger.critical("Couldn't pull. Skipping. Error: %s", e) - raise ConnectionError + return self._pull(tag) def update(self): - updated_count = 0 updated_service_tuples = [] self.monitored = self.monitor_filter() @@ -423,7 +407,7 @@ def update(self): image_string = service.attrs['Spec']['TaskTemplate']['ContainerSpec']['Image'] if '@' in image_string: tag = image_string.split('@')[0] - sha256 = image_string.split('@')[1][7:] + sha256 = remove_sha_prefix(image_string.split('@')[1]) else: self.logger.error('No image SHA for %s. Skipping', image_string) continue @@ -433,13 +417,15 @@ def update(self): except ConnectionError: continue - if self.config.dry_run: - # Ugly hack for repo digest - if sha256 != latest_image.id: + latest_image_sha256 = get_digest(latest_image) + self.logger.debug('Latest sha256 for %s is %s', tag, latest_image_sha256) + + if sha256 != latest_image_sha256: + if self.config.dry_run: + # Ugly hack for repo digest self.logger.info('dry run : %s would be updated', service.name) - continue + continue - if sha256 != latest_image.id: updated_service_tuples.append( (service, sha256[-10:], latest_image) ) @@ -452,17 +438,13 @@ def update(self): socket=self.socket, kind='update', mode='service') self.logger.info('%s will be updated', service.name) - service.update(image=tag) - - updated_count += 1 - - self.logger.debug("Incrementing total service updated count") + service.update(image=f"{tag}@sha256:{latest_image_sha256}") self.data_manager.total_updated[self.socket] += 1 self.data_manager.add(label=service.name, socket=self.socket) self.data_manager.add(label='all', socket=self.socket) - if updated_count > 0: + if updated_service_tuples: self.notification_manager.send( container_tuples=updated_service_tuples, socket=self.socket, diff --git a/pyouroboros/helpers.py b/pyouroboros/helpers.py index 252e6083..54125051 100644 --- a/pyouroboros/helpers.py +++ b/pyouroboros/helpers.py @@ -4,6 +4,7 @@ def set_properties(old, new, self_name=None): 'name': self_name if self_name else old.name, 'hostname': old.attrs['Config']['Hostname'], 'user': old.attrs['Config']['User'], + 'detach': True, 'domainname': old.attrs['Config']['Domainname'], 'tty': old.attrs['Config']['Tty'], 'ports': None if not old.attrs['Config'].get('ExposedPorts') else [ @@ -22,3 +23,18 @@ def set_properties(old, new, self_name=None): } return properties + + +def remove_sha_prefix(digest): + if digest.startswith("sha256:"): + return digest[7:] + return digest + + +def get_digest(image): + digest = image.attrs.get( + "Descriptor", {} + ).get("digest") or image.attrs.get( + "RepoDigests" + )[0].split('@')[1] or image.id + return remove_sha_prefix(digest) diff --git a/pyouroboros/ouroboros.py b/pyouroboros/ouroboros.py index 5223edc5..214e239c 100644 --- a/pyouroboros/ouroboros.py +++ b/pyouroboros/ouroboros.py @@ -164,7 +164,8 @@ def main(): hour=config.cron[1], day=config.cron[2], month=config.cron[3], - day_of_week=config.cron[4] + day_of_week=config.cron[4], + misfire_grace_time=15 ) else: if config.run_once: diff --git a/setup.py b/setup.py index caeaa52b..1f0b455a 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,25 @@ from setuptools import setup, find_packages from pyouroboros import VERSION -requirements = ['docker>=3.7.0', - 'apscheduler>=3.5.3', - 'prometheus_client>=0.5.0', - 'requests>=2.21.0', - 'influxdb>=5.2.1', - 'apprise>=0.5.2'] - -def readme(): - with open('./README.md') as f: +def read(filename): + with open(filename) as f: return f.read() +def get_requirements(filename="requirements.txt"): + """returns a list of all requirements""" + requirements = read(filename) + return list(filter(None, [req.strip() for req in requirements.split() if not req.startswith('#')])) + + setup( name='ouroboros-cli', version=VERSION, maintainer='circa10a', + maintainer_email='caleblemoine@gmail.com', description='Automatically update running docker containers', - long_description=readme(), + long_description=read('README.md'), long_description_content_type='text/markdown', url='https://github.com/pyouroboros/ouroboros', license='MIT', @@ -28,6 +28,6 @@ def readme(): 'Programming Language :: Python :: 3.7'], packages=find_packages(), scripts=['ouroboros'], - install_requires=requirements, + install_requires=get_requirements(), python_requires='>=3.6.2' )