From ad097f2d80216001ea903c8dc6cc40073103a164 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Mon, 9 Oct 2017 08:51:14 -0500 Subject: [PATCH] add worker stuffs --- .coveragerc | 9 +++++++ .travis.yml | 35 ++++++++++++++++++++++++++ Dockerfile | 29 ++++++++++++++++++++++ README.md | 2 -- README.rst | 10 ++++++++ deli_worker/__init__.py | 0 deli_worker/main.py | 18 ++++++++++++++ deli_worker/settings.py | 54 +++++++++++++++++++++++++++++++++++++++++ docker-entrypoint.sh | 4 +++ requirements.txt | 1 + setup.cfg | 26 ++++++++++++++++++++ setup.py | 6 +++++ test-requirements.txt | 0 tox.ini | 25 +++++++++++++++++++ trigger_docker_build.sh | 3 +++ 15 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 .coveragerc create mode 100644 .travis.yml create mode 100644 Dockerfile delete mode 100644 README.md create mode 100644 README.rst create mode 100644 deli_worker/__init__.py create mode 100644 deli_worker/main.py create mode 100644 deli_worker/settings.py create mode 100644 docker-entrypoint.sh create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tox.ini create mode 100755 trigger_docker_build.sh diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..a388931 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ +[run] +omit = + */test/* + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise NotImplementedError \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..dfdf961 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +language: python +python: +- '3.6' +env: +#- TOXENV=py36 +- TOXENV=flake8 +- TOXENV=build +install: +- pip install tox +script: tox +services: +- postgresql + +stages: + - test + - name: pypi + if: tag IS present + - name: docker + if: tag IS present + +jobs: + include: + - stage: pypi + script: 'echo "Deploying to PyPi"' + deploy: + provider: pypi + user: rmb938 + password: + secure: fhdvEq81B9Fl9+gzqVRSNVOBn9Pw5eWGiOJhHp+3N4M7AOUGta35mu5K7BSgIZuwQ+pgWWdQOcrszNyEs9GFIibLiE3aTKhPOIxnR89bINAZLrvm5RfXik1kjOKhx0jCsrEtFfQAfw9hg8cr2I099BBrAoRYQJZrHIVLaO77uvj/rd583OTsLE6smRkjj89yqktuTmhtdh5XK8aFr+wXNFIKcAxq4uDh6L/X521uRSkwvRjlA05OI9/Pyv4B52yXPJK5+QhwrwY0EwdbmfIpI81IQhzfErYwh7cUPyFgExTZitbpEd2+JS8k9VSV709sfHzeglbEe2Z5wkEGCGEPV85IQMpicBV9Am8omOc87qp9pXTerrYS8HSYcXWZqma75sdHy22VTVyPz5hTXu8ssA3tA/QqsES78BJIWOLPdkd1/CyJCTX7fm2rGdwVITn7floGGmhh4DjdciuAHr3/hd+w3uB5D1c57voXGSGfy5tCc+LanItrfberz61oyr0FlOjSo6OCbE41UEYVrRYlRhV5dMXHC2qEk3M9uyJGkf3ZMQlir+/OsTDJD/GDdGz8r8Y2aQ7cLz6fgRvzw9yAuiGhDmWNiADNjJa+JYpjRDYt7xQLZtaeHH8GXQLqIV7BaN9nF0Rhe9beUYiMnDnx9PDRK2Qf0Y5stJy50cwFdGE= + on: + tags: true + - stage: docker + env: + - secure: "1maT8WTEjmCMWLQQV/ZN9KI98TK5ISfCDI265jUuDdhXEHViDRB22AEjCqRRvds1VIWca3o0sivf8d2fAAI15+O2j6UtVg4fnKScycUWx8wbPB0nzCoFLhcRypoDv7Xb19njadAerKDLxMDFRZH2jbpjvANlwaDrb8bgt90lvBFg1lPRzqW5EU1QKe5MtMLBWx1cSiBKguvAsVDNGWyKXwL4cX52EFRQPryyaINDcgbvGUq5SFLRMUd4olE0HhmSsS21I+aZUmIOgJLhi1mwGc9cUYwlz9oSNAYxCK4p/gPem/SDJjNIy+6JmYr7eR3l8ZzOCN7Xso47QydEC+XFBFVwY3IRqQtzKf8DRlI14aV/CHLYc8H15Xlx8hp6Q8ycTGXq4g0FXUDyfTIS2XSYJWmN5tfjloIU0h8C/e3qX76i2lOuHH197T5pHNMhoTdOncDjj+5MqaAeTv5cPFrq5lrWjrZAvW9+PbMPPE+NzP3jyflHAiRYLFTullYW3099VoSkURUeDsMRDF8RxnccKlrmylP4KBqszURCf3Fkl0c4WsdOsQervrPoLkk6Zna3s3uDph36GOYJqs3m1i+eG0EwqrBtrlNO7wp31IS4/EbAuoRBn+TaXN0zaMlgUylorP4Ap5qdGRDyFg7/uMieOn5m3hbi303KAo9ScIlTgVs=" + script: ./trigger_docker_build.sh \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..06e2887 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3-alpine +WORKDIR /usr/src/app + +COPY . . +RUN apk -U add git && python setup.py sdist bdist_wheel + +FROM python:3-alpine +WORKDIR /usr/src/app + +# Install build time dependencies for uwsgi +# Install dumb-init +# Remove build time dependencies +# Install runtime dependencies +RUN apk --no-cache add --virtual build-deps \ + build-base linux-headers pcre-dev postgresql-dev && \ + pip install dumb-init psycopg2 && \ + apk del build-deps && \ + apk --no-cache add bash openssl pcre libpq ca-certificates + +# COPY tar.gz from build container +# Install it +# TODO: this should be replaced by a build arg to install the correct version from pypi +COPY --from=0 /usr/src/app/dist/. . +RUN bash -c "pip install *" + +# add entrypoint +COPY docker-entrypoint.sh /bin/docker-entrypoint.sh + +ENTRYPOINT ["/bin/docker-entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 6c8d574..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# deli-worker -Task Processor for Sandwich Cloud diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..bcd3941 --- /dev/null +++ b/README.rst @@ -0,0 +1,10 @@ +deli-worker +----------- + +.. image:: https://travis-ci.org/sandwichcloud/deli-worker.svg?branch=master + :target: https://travis-ci.org/sandwichcloud/deli-worker + +.. image:: https://badge.fury.io/py/deli-worker.svg + :target: https://badge.fury.io/py/deli-worker + +Worker for Sandwich Cloud diff --git a/deli_worker/__init__.py b/deli_worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/deli_worker/main.py b/deli_worker/main.py new file mode 100644 index 0000000..fdff9af --- /dev/null +++ b/deli_worker/main.py @@ -0,0 +1,18 @@ +import sys + +from ingredients_tasks.celary import Messaging, database + + +def main(): + messaging = Messaging('127.0.0.1', 5672, 'sandwich', 'hunter2', '/') + messaging.connect() + database.connect() + # with database.session(): + # pass + messaging.app.main = 'deli_worker' + sys.argv.extend(['worker', '-l', 'info', '-Ofair']) + messaging.start(argv=None) + + +if __name__ == '__main__': + main() diff --git a/deli_worker/settings.py b/deli_worker/settings.py new file mode 100644 index 0000000..9cb835d --- /dev/null +++ b/deli_worker/settings.py @@ -0,0 +1,54 @@ +import os + +from dotenv import load_dotenv + +load_dotenv(os.path.join(os.getcwd(), '.env')) + +#################### +# DATABASE # +#################### + +DATABASE_DB = 'sandwich' +DATABASE_PORT = '5432' +DATABASE_POOL_SIZE = 20 + +if os.environ.get('CLI'): + DATABASE_POOL_SIZE = -1 + +DATABASE_HOST = os.environ['DATABASE_HOST'] +DATABASE_USERNAME = os.environ['DATABASE_USERNAME'] +DATABASE_PASSWORD = os.environ['DATABASE_PASSWORD'] + +#################### +# RABBITMQ # +#################### + +RABBITMQ_VHOST = '/' +RABBITMQ_PORT = 5672 +RABBITMQ_HOST = os.environ['RABBITMQ_HOST'] +RABBITMQ_USERNAME = os.environ['RABBITMQ_USERNAME'] +RABBITMQ_PASSWORD = os.environ['RABBITMQ_PASSWORD'] + +#################### +# VMWare # +#################### + +VCENTER_HOST = os.environ['VCENTER_HOST'] +VCENTER_PORT = os.environ['VCENTER_PORT'] +VCENTER_USERNAME = os.environ['VCENTER_USERNAME'] +VCENTER_PASSWORD = os.environ['VCENTER_PASSWORD'] + +VCENTER_IMAGES_FOLDER = 'images' +VCENTER_INSTANCES_FOLDER = 'instances' + +VCENTER_CLUSTER = 'Management' +VCENTER_DATASTORE = 'datastore2' + +#################### +# DHCP # +#################### + +DHCP_SERVER_IP = os.environ['DHCP_SERVER_IP'] +DHCP_OMAPI_PORT = os.environ.get('DHCP_OMAPI_PORT', 7911) +DHCP_KEY_NAME = os.environ.get('DHCP_KEY_NAME', None) +DHCP_B64_KEY = os.environ.get('DHCP_B64_KEY', None) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 0000000..1290152 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/dumb-init /bin/bash + +export SETTINGS_MODULE=deli_worker.settings +deli_worker $@ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9697d13 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +ingredients.tasks==0.0.1 # MIT \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..890cfac --- /dev/null +++ b/setup.cfg @@ -0,0 +1,26 @@ +[metadata] +name = deli-worker +author = Ryan Belgrave +author-email = rmb1993@gmail.com +summary = Worker for Sandwich Cloud +description-file = README.rst +home-page = https://github.com/sandwichcloud/deli-worker +license = MIT +classifier = + Development Status :: 3 - Alpha + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Programming Language :: Python + +[files] +packages = + deli_worker + +[entry_points] +console_scripts = + deli_worker = deli_worker.main:main + +[pbr] +skip_authors = 1 +skip_changelog = 1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8935042 --- /dev/null +++ b/setup.py @@ -0,0 +1,6 @@ +from setuptools import setup + +setup( + setup_requires=['pbr>=3.0.1', 'setuptools>=17.1'], + pbr=True, +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..864e11a --- /dev/null +++ b/tox.ini @@ -0,0 +1,25 @@ +[tox] +envlist = flake8, build + +[testenv:flake8] +commands = + flake8 +deps = + flake8 + +;[testenv:py36] +;commands = +; py.test +;deps = +; -r{toxinidir}/test-requirements.txt + +[testenv:build] +commands = + python setup.py sdist bdist_wheel + +[flake8] +exclude = .tox, .git, __pycache__, build, dist, *.egg-info, ./.cache, ./.eggs +max-line-length = 120 + +[pytest] +addopts = --tb=short -rwxs -v --cov=deli_worker --cov-report term-missing:skip-covered \ No newline at end of file diff --git a/trigger_docker_build.sh b/trigger_docker_build.sh new file mode 100755 index 0000000..73d1585 --- /dev/null +++ b/trigger_docker_build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +curl -H "Content-Type: application/json" -d "{\"source_type\": \"Tag\", \"source_name\": \"$TRAVIS_TAG\"}" -X POST "$DOCKER_HUB_TRIGGER" \ No newline at end of file