diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..51df6bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +.bash_history +.bash_logout +.bash_profile +.bashrc +.cache +.emacs +.sass-cache +.ssh/ +__pycache__ +settings/local.py +assets +celerybeat-schedule.bak +celerybeat-schedule.dat +celerybeat-schedule.dir +celerybeat.pid +tests/data/assets +*.pyc +/bin +/lib +/lib64 +/logs +/man +/share +pip-selfcheck.json +pyvenv.cfg +nohup.out +npm-debug.log +celerybeat-schedule +*~ +s3.cfg +tests/data/s3.cfg +backend_configs \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a85f31b --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Hangar51 + +Hangar51 is an asset (image and file) storage and retrieval application. \ No newline at end of file diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..487edf5 --- /dev/null +++ b/api/__init__.py @@ -0,0 +1,69 @@ +from flask import Blueprint, g, jsonify, request +from functools import wraps +from mongoframes import * + +from models.accounts import Account + +api = Blueprint('api', __name__) + +__all__ = [ + # Blueprint + 'api', + + # Decorators + 'authenticated', + + # Responses + 'fail', + 'success' + ] + + +# Decorators + +def authenticated(func): + """ + Wrap this decorator around any view that requires a valid account key. + """ + + # Wrap the function with the decorator + @wraps(func) + def wrapper(*args, **kwargs): + + api_key = request.values.get('api_key') + if not api_key: + return fail('`api_key` not specified.') + + # Find the account + account = Account.one(Q.api_key == api_key.strip()) + if not account: + return fail('Not a valid `api_key`.') + + # Set the account against the global context + g.account = account + + return func(*args, **kwargs) + + return wrapper + + +# Responses + +def fail(reason, issues=None): + """Return a fail response""" + response = {'status': 'fail', 'payload': {'reason': reason}} + if issues: + response['payload']['issues'] = issues + return jsonify(response) + +def success(payload=None): + """Return a success response""" + response = {'status': 'success'} + if payload: + response['payload'] = payload + return jsonify(response) + + +# Place imports here to prevent cross import clashes + +from api import assets \ No newline at end of file diff --git a/api/assets.py b/api/assets.py new file mode 100644 index 0000000..30b486f --- /dev/null +++ b/api/assets.py @@ -0,0 +1,380 @@ +from datetime import datetime, timezone +import io +import imghdr +import json +import mimetypes +import re +import time + +from flask import current_app, g, make_response, request +from mongoframes import * +import os +from PIL import Image +from PIL.ExifTags import TAGS +from slugify import Slugify + +from api import * +from forms.assets import * +from models.assets import Asset, Variation +from utils import get_file_length, generate_uid + +# Fix for missing mimetypes +mimetypes.add_type('text/csv', '.csv') +mimetypes.add_type('image/webp', '.webp') + + +# Routes + +@api.route('/download') +@authenticated +def download(): + """Download an asset""" + + # Validate the parameters + form = DownloadForm(request.values) + if not form.validate(): + return fail('Invalid request', issues=form.errors) + form_data = form.data + + # Get the asset + asset = Asset.one(And( + Q.account == g.account, + Q.uid == form_data['uid'] + )) + + # Retrieve the original file + backend = g.account.get_backend_instance() + f = backend.retrieve(asset.store_key) + + # Build the file response to return + response = make_response(f.read()) + response.headers['Content-Type'] = asset.content_type + response.headers['Content-Disposition'] = \ + 'attachment; filename={0}'.format(asset.store_key) + + return response + +@api.route('/get') +@authenticated +def get(): + """Get the details for an asset""" + + # Validate the parameters + form = GetForm(request.values) + if not form.validate(): + return fail('Invalid request', issues=form.errors) + form_data = form.data + + # Get the asset + asset = Asset.one(And( + Q.account == g.account, + Q.uid == form_data['uid'] + )) + + return success(asset.to_json_type()) + +@api.route('/', endpoint='list') +@authenticated +def _list(): + """List assets""" + + # Validate the parameters + form = ListForm(request.values) + if not form.validate(): + return fail('Invalid request', issues=form.errors) + form_data = form.data + + # Build the query + query = [ + Q.account == g.account, + Or( + Exists(Q.expires, False), + Q.expires > time.mktime(datetime.now(timezone.utc).timetuple()) + ) + ] + + # `q` + if form_data['q'] and form_data['q'].strip(): + # Replace `*` instances with non-greedy re dot matches + q = re.escape(form_data['q']).replace('\*', '.*?') + q_exp = re.compile(r'^{q}$'.format(q=q), re.I) + query.append(Q.store_key == q_exp) + + # `type` + if form_data['type']: + query.append(Q.type == form_data['type']) + + # `order` + sort = { + 'created': [('created', ASC)], + '-created': [('created', DESC)], + 'store_key': [('store_key', ASC)] + }[form_data['order'] or 'store_key'] + + # Paginate the results + paginator = Paginator( + Asset, + filter=And(*query), + projection={ + 'created': True, + 'store_key': True, + 'type': True, + 'uid': True + }, + sort=sort, + per_page=1000 + ) + + # Attempt to select the requested page + try: + page = paginator[form_data['page']] + except InvalidPage: + return fail('Invalid page') + + return success({ + 'assets': [a.to_json_type() for a in page.items], + 'total_assets': paginator.item_count, + 'total_pages': paginator.page_count + }) + +@api.route('/generate-variations', methods=['POST']) +@authenticated +def generate_variations(): + """Generate one or more variations for of an image asset""" + + # Validate the parameters + form = GenerateVariationsForm(request.values) + if not form.validate(): + return fail('Invalid request', issues=form.errors) + form_data = form.data + + # Find the asset + asset = Asset.one(And(Q.account == g.account, Q.uid == form_data['uid'])) + + # Check the asset is an image + if asset.type != 'image': + return fail('Variations can only be generated for images') + + # Parse the variation data + variations = json.loads(form_data['variations']) + + # Has the user specified how they want the results delivered? + on_delivery = form_data['on_delivery'] or 'wait' + if on_delivery == 'wait': + # Caller is waiting for a response so generate the variations now + + # Retrieve the original file + backend = g.account.get_backend_instance() + f = backend.retrieve(asset.store_key) + im = Image.open(f) + + # Generate the variations + new_variations = {} + for name, ops in variations.items(): + new_variations[name] = asset.add_variation(f, im, name, ops) + new_variations[name] = new_variations[name].to_json_type() + + # Update the assets modified timestamp + asset.update('modified') + + return success(new_variations) + + else: + # Caller doesn't want to wait for a response so generate the variations + # in the background. + current_app.celery.send_task( + 'generate_variations', + [g.account._id, asset.uid, variations, form_data['webhook'].strip()] + ) + + return success() + +@api.route('/set-expires', methods=['POST']) +@authenticated +def set_expires(): + """Set the expiry date for an asset""" + + # Validate the parameters + form = SetExpiresForm(request.values) + if not form.validate(): + return fail('Invalid request', issues=form.errors) + form_data = form.data + + # Get the asset + asset = Asset.one(And( + Q.account == g.account, + Q.uid == form_data['uid'] + )) + + # Update the assets `expires` value + if 'expires' in form_data: + # Set `expires` + asset.expires = form_data['expires'] + asset.update('expires', 'modified') + + else: + # Unset `expires` + Asset.get_collection().update( + {'_id': asset._id}, + {'$unset': {'expires': ''}} + ) + asset.update('modified') + + return success() + +@api.route('/upload', methods=['POST']) +@authenticated +def upload(): + """Upload an asset""" + + # Check a file has been provided + fs = request.files.get('asset') + if not fs: + return fail('No `asset` sent.') + + # Validate the parameters + form = UploadForm(request.values) + if not form.validate(): + return fail('Invalid request', issues=form.errors) + + # Prep the asset name for + form_data = form.data + + # Name + name = form_data['name'] + if not name: + name = os.path.splitext(fs.filename)[0] + name = slugify_name(name) + + # Extension + ext = os.path.splitext(fs.filename)[1].lower()[1:] + + # If there's no extension associated with then see if we can guess it using + # the imghdr module + if not ext: + fs.stream.seek(0) + ext = imghdr.what(fs.filename,fs.stream.read()) or '' + + # If the file is a recognized image format then attempt to read it as an + # image otherwise leave it as a file. + asset_file = fs.stream + asset_meta = {} + asset_type = Asset.get_type(ext) + if asset_type is 'image': + try: + asset_file, asset_meta = prep_image(asset_file) + except IOError as e: + return fail('File appears to be an image but it cannot be read.') + + # Add basic file information to the asset meta + asset_meta.update({ + 'filename': fs.filename, + 'length': get_file_length(asset_file) + }) + + # Create the asset + asset = Asset( + account=g.account._id, + name=name, + ext=ext, + meta=asset_meta, + type=asset_type, + variations=[] + ) + + if form_data['expires']: + asset.expires = form_data['expires'] + + # Generate a unique Id for the asset + asset.uid = generate_uid(6) + while Asset.count(And(Q.account == g.account, Q.uid == asset.uid)) > 0: + asset.uid = generate_uid(6) + + # Store the original file + asset.store_key = Asset.get_store_key(asset) + backend = g.account.get_backend_instance() + backend.store(asset_file, asset.store_key) + + # Save the asset + asset.insert() + + return success(asset.to_json_type()) + + +# Utils + +def prep_image(f): + """Prepare an image as a file""" + + # Attempt to load the image + im = Image.open(f) + fmt = im.format + + # Orient the image + if hasattr(im, '_getexif') and im._getexif(): + # Only JPEG images contain the _getexif tag, however if it's present we + # can use it make sure the image is correctly orientated. + + # Convert the exif data to a dictionary with alphanumeric keys + exif = {TAGS[k]: v for k, v in im._getexif().items() if k in TAGS} + + # Check for an orientation setting and orient the image if required + orientation = exif.get('Orientation') + if orientation == 2: + im = im.transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 3: + im = im.transpose(Image.ROTATE_180) + elif orientation == 4: + im = im.transpose(Image.FLIP_TOP_BOTTOM) + elif orientation == 5: + im = im.transpose(Image.FLIP_LEFT_RIGHT) + im = im.transpose(Image.ROTATE_90) + elif orientation == 6: + im = im.transpose(Image.ROTATE_270) + elif orientation == 7: + im = im.transpose(Image.FLIP_TOP_BOTTOM) + im = im.transpose(Image.ROTATE_90) + elif orientation == 8: + im = im.transpose(Image.ROTATE_90) + + # Convert the image back to a stream + f = io.BytesIO() + im.save(f, format=fmt) + f.seek(0) + + + # Strip meta data from file + im_no_exif = None + if im.format == 'GIF': + im_no_exif = im + else: + f = io.BytesIO() + im_no_exif = Image.new(im.mode, im.size) + im_no_exif.putdata(list(im.getdata())) + im_no_exif.save(f, format=fmt) + + f.seek(0) + + # Extract any available meta information + meta = { + 'image': { + 'mode': im.mode, + 'size': im.size + } + } + + return f, meta + +def slugify_name(name): + """Get a slugifier used to ensure asset names are safe""" + + # Create the slugifier + slugifier = Slugify() + + # Configure the slugifier + slugifier.to_lower = True + slugifier.safe_chars = '-/' + slugifier.max_length = 200 + + # Names cannot start or end with forward slashes '/' + return slugifier(name).strip('/') \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..bacb621 --- /dev/null +++ b/app.py @@ -0,0 +1,111 @@ +""" +Initialization of the application. +""" + +import argparse +from celery import Celery +from celery.bin import Option +from flask import Flask, jsonify +from mongoframes import Frame +import pymongo +from raven.contrib.flask import Sentry +from werkzeug.contrib.fixers import ProxyFix + + +__all__ = ['create_app'] + + +sentry = Sentry() + +def create_app(env): + """ + We use an application factory to allow the app to be configured from the + command line at start up. + """ + + # Create the app + app = Flask(__name__) + + # Configure the application to the specified config + app.config['ENV'] = env + app.config.from_object('settings.{0}.Config'.format(env)) + + # Add celery + app.celery = create_celery_app(app) + + # Add sentry logging if the DSN is provided + if app.config['SENTRY_DSN']: + app.sentry = sentry.init_app(app) + + # Add mongo support + app.mongo = pymongo.MongoClient(app.config['MONGO_URI']) + app.db = app.mongo.get_default_database() + Frame._client = app.mongo + + if app.config.get('MONGO_PASSWORD'): + Frame.get_db().authenticate( + app.config.get('MONGO_USERNAME'), + app.config.get('MONGO_PASSWORD') + ) + + # Fix for REMOTE_ADDR value + app.wsgi_app = ProxyFix(app.wsgi_app) + + # Import views as a blueprint + import api + app.register_blueprint(api.api) + return app + +def create_celery_app(app): + """ + This function integrates celery into Flask, see ref here: + http://flask.pocoo.org/docs/0.10/patterns/celery/ + """ + + # Create a new celery object and configure it using the apps config + celery = Celery(app.import_name, broker=app.config['CELERY_BROKER_URL']) + app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL'] + celery.conf.update(app.config) + + # Add the env option + option = Option( + '-e', + '--env', + choices=['dev', 'local', 'prod'], + default='local', + dest='env' + ) + + celery.user_options['beat'].add(option) + celery.user_options['worker'].add(option) + + # Create a sub class of the celery Task class that exectures within the + # application's context. + TaskBase = celery.Task + class ContextTask(TaskBase): + abstract = True + def __call__(self, *args, **kwargs): + with app.app_context(): + return TaskBase.__call__(self, *args, **kwargs) + celery.Task = ContextTask + + return celery + + +if __name__ == "__main__": + + # Parse the command-line arguments + parser = argparse.ArgumentParser(description='Application server') + parser.add_argument( + '-e', + '--env', + choices=['dev', 'local', 'prod'], + default='local', + dest='env', + required=False + ) + args = parser.parse_args() + + # Create and run the application + app = create_app(args.env) + app.run(port=app.config.get('PORT', 5152)) \ No newline at end of file diff --git a/backends/__init__.py b/backends/__init__.py new file mode 100644 index 0000000..1402770 --- /dev/null +++ b/backends/__init__.py @@ -0,0 +1,111 @@ +import glob +import importlib +import inspect +import os + +from utils.forms import FormData + +__all__ = ['Backend'] + + +class Backend: + """ + The `Backend` class allows different types of storage models to be used for + asset storage. + + Different backends should inherit from the base `Backend` class and override + its classmethods. + """ + + # Each backend must have a unique name which + name = '' + + # Backend configurations are validated when an account is added or the + # configuration for an account is changed. Validation is done using a + # `WTForm.Form` instance. + config_form = None + + def __init__(self, **config): + raise NotImplementedError() + + def delete(self, key): + """Delete a file from the store""" + raise NotImplementedError() + + def retrieve(self, key): + """Retrieve a file from the store""" + raise NotImplementedError() + + def store(self, f, key): + """Store a file""" + raise NotImplementedError() + + @classmethod + def validate_config(cls, **config): + """Validate a set of config values""" + if not cls.config_form: + raise NotImplementedError() + + # Validate the configuration against the form + form = cls.config_form(**config) + if not form.validate(): + return False, form.errors + + return True, {} + + @classmethod + def get_backend(self, name): + """Return the named backend""" + + # Check if the cache exists, if not build it + assert name in Backend.list_backends(), \ + 'No backend named `{name}`'.format(name=name) + + return Backend._cache[name] + + @classmethod + def list_backends(self): + """Return a list of available backends""" + + # Check for a cached list of backends + if hasattr(Backend, '_cache'): + return sorted(Backend._cache.keys()) + + # Find all python files within this (the backends) folder + module_names = glob.glob(os.path.dirname(__file__) + '/*.py') + module_names = [os.path.basename(n)[:-3] \ + for n in module_names if os.path.isfile(n)] + + # Build a list of the backends installed + backends = [] + + for module_name in module_names: + + # Don't import self + if module_name == '__init__': + continue + + # Import the module + module = importlib.import_module('backends.' + module_name) + + # Check each member of the module to see if it's a Backend + for member in inspect.getmembers(module): + + # Must be a class + if not inspect.isclass(member[1]): + continue + + # Must be a sub-class of `Backend` + if not issubclass(member[1], (Backend,)): + continue + + # Must not be the `Backend` class itself + if member[1] == Backend: + continue + + backends.append(member[1]) + + # Cache the result + Backend._cache = {b.name: b for b in backends} + + return Backend.list_backends() \ No newline at end of file diff --git a/backends/local.py b/backends/local.py new file mode 100644 index 0000000..f47da15 --- /dev/null +++ b/backends/local.py @@ -0,0 +1,71 @@ +from flask import current_app +from wtforms import Form, ValidationError +from wtforms.fields import * +import io +import os +import shutil +from wtforms.validators import * + +from backends import Backend + +__all__ = ['LocalBackend'] + + +class ConfigForm(Form): + + asset_root = StringField( + 'Please specify the `asset_root` directory path where assets will be \ +stored', + [Required()] + ) + + def validate_asset_root(form, field): + """Validate the asset root directory exists""" + + # Check the asset root directory exists + if not os.path.exists(field.data): + raise ValidationError('Asset root directory does not exist.') + + +class LocalBackend(Backend): + """ + Backend to support storing files on the local file system. + """ + + name = 'local' + config_form = ConfigForm + + def __init__(self, **config): + self.asset_root = config['asset_root'] + + def delete(self, key): + """Delete a file from the store""" + + # Remove the file if it exists + abs_path = os.path.join(self.asset_root, key) + if os.path.exists(abs_path): + os.remove(abs_path) + + def retrieve(self, key): + """Retrieve a file from the store""" + + # Return the file as a byte stream + abs_path = os.path.join(self.asset_root, key) + with open(abs_path, 'rb') as f: + stream = io.BytesIO(f.read()) + + return stream + + def store(self, f, key): + """Store a file""" + + # Determine the storage location + filepath, filename = os.path.split(key) + abs_path = os.path.join(self.asset_root, filepath) + + # Ensure the location exists + os.makedirs(abs_path, exist_ok=True) + + # Save the file + with open(os.path.join(abs_path, filename), 'wb') as store: + store.write(f.read()) \ No newline at end of file diff --git a/backends/s3.py b/backends/s3.py new file mode 100644 index 0000000..74af747 --- /dev/null +++ b/backends/s3.py @@ -0,0 +1,113 @@ +import boto3 +from botocore.client import ClientError +import io +import os +import tempfile +import uuid +from wtforms import Form, ValidationError +from wtforms.fields import * +from wtforms.validators import * + +from backends import Backend +from models.assets import Asset + +__all__ = ['S3Backend'] + + +class ConfigForm(Form): + + access_key = StringField( + 'Please specify your AWS `access_key`', + [Required()] + ) + secret_key = StringField( + 'Please specify your AWS `secret_key`', + [Required()] + ) + bucket = StringField( + 'Please specify the S3 `bucket` that will store the assets', + [Required()] + ) + + def validate_access_key(form, field): + access_key = field.data + secret_key = form.secret_key.data + bucket = form.bucket.data + + # To validate we can make a connection we need all values to have been + # specified. + if not (access_key and secret_key and bucket): + return + + s3 = boto3.resource( + 's3', + aws_access_key_id=access_key, + aws_secret_access_key=secret_key + ) + + try: + s3.meta.client.head_bucket(Bucket=bucket) + except ClientError as e: + raise ValidationError(str(e)) + + +class S3Backend(Backend): + """ + Backend to support storing files on the local file system. + """ + + name = 's3' + config_form = ConfigForm + + def __init__(self, **config): + self.s3 = boto3.resource( + 's3', + aws_access_key_id=config['access_key'], + aws_secret_access_key=config['secret_key'] + ) + self.bucket = self.s3.Bucket(config['bucket']) + + def delete(self, key): + """Delete a file from the store""" + self.bucket.delete_objects(Delete={'Objects': [{'Key': key}]}) + + def retrieve(self, key): + """Retrieve a file from the store""" + + # Create a temporary directory to download the file to + with tempfile.TemporaryDirectory() as dirname: + + # Create a temporary filepath to store and retrieve the file from + filepath = os.path.join(dirname, uuid.uuid4().hex) + + # Download the file + self.bucket.download_file(key, filepath) + + # Convert the file to a stream + with open(filepath, 'rb') as f: + stream = io.BytesIO(f.read()) + + # Remove the temporary file + os.remove(filepath) + + return stream + + def store(self, f, key): + """Store a file""" + + # Guess the content type + content_type = Asset.guess_content_type(key) + + # Set the file to be cached to a year from now + cache_control = 'max-age=%d, public' % (365 * 24 * 60 * 60) + + # Store the object + obj = self.s3.Object(self.bucket.name, key) + if content_type: + obj.put( + Body=f, + ContentType=content_type, + CacheControl=cache_control + ) + else: + obj.put(Body=f, CacheControl=cache_control) \ No newline at end of file diff --git a/commands/__init__.py b/commands/__init__.py new file mode 100644 index 0000000..282f82e --- /dev/null +++ b/commands/__init__.py @@ -0,0 +1,130 @@ +""" +Command line tools for managing the application. +""" + +from blessings import Terminal +from flask import current_app +from flask.ext.script import Command, Option +from pymongo import IndexModel, ASCENDING, DESCENDING +from random import choice +from string import ascii_lowercase + +from models.accounts import Account +from models.assets import Asset + +__all__ = [ + 'AppCommand' + ] + + +class AppCommand(Command): + """ + A base command class for the application. + """ + + def __init__(self, *args, **kwargs): + super(AppCommand, self).__init__(*args, **kwargs) + + # Create a terminal instance we can use for formatting command line + # output. + self.t = Terminal() + + def confirm(self, question, s): + """Ask the you user to confirm an action before performing it""" + + # In test mode we confirms are automatic + if current_app.config['ENV'] is 'test': + return True + + # Ask the user to confirm the action by request they repeat a randomly + # generated string. + answer = input( + '{t.blue}{question}: {t.bold}{s}{t.normal}\n'.format( + question=question, + s=s, + t=self.t + ) + ) + + # Check the answer + if answer != s: + print(self.t.bold_red("Strings didn't match")) + return False + + return True + + def err(self, *errors, **error_dict): + """ + Output a list of errors to the command line, for example: + + self.err( + 'Unable to find account: foobar', + ... + ) + + Optional if a dictionary of errors is sent as keywords (this is typical + used to output error information from a form) then that will be + automatically converted to a list of error messages. + """ + + # Convert error dict to additional messages + if error_dict: + errors += tuple((f, ''.join(e)) for f, e in error_dict.items()) + + # Build the error output + output = [('Failed', 'underline_bold_red')] + for error in errors: + if isinstance(error, tuple): + field, error = error + output.append(('- {0} - {1}'.format(field, error), 'red')) + + else: + output.append((error, 'red')) + + self.out(*output) + + def out(self, *strings): + """ + Output one or more strings (optionally with formats) to the commandline, + for example: + + self.out( + 'hi', + ('I am red', 'red'), + ... + ) + + """ + + # We add an additional blank line to the head and tail of the output + # accept when testing where we remove these to make it easier to compare + # output. + if current_app.config['ENV'] != 'test': + print() + + for s in strings: + if isinstance(s, tuple): + # Formatted string + s, fmt = s + + # When testing we ignore formatting to make it easier to compare + # output. + if current_app.config['ENV'] == 'test': + # Unformatted string + print(s) + + else: + # Formatted string + fmt = getattr(self.t, fmt) + print(fmt(s)) + + else: + # Unformatted string + print(s) + + if current_app.config['ENV'] != 'test': + print() + +# Prevent cross import clashes by importing other commands here +from commands.accounts import * +from commands.app import * \ No newline at end of file diff --git a/commands/accounts.py b/commands/accounts.py new file mode 100644 index 0000000..c08b6e3 --- /dev/null +++ b/commands/accounts.py @@ -0,0 +1,355 @@ +""" +Command line tools for managing accounts. +""" + +from blessings import Terminal +from flask import current_app +from flask.ext.script import Command, Option +import json +from mongoframes import * +import re + +from backends import Backend +from commands import AppCommand +from forms.accounts import * +from models.accounts import Account +from models.assets import Asset + +__all__ = [ + 'AddAccount', + 'ConfigAccount', + 'DeleteAccount', + 'GenerateNewAPIKey', + 'ListAccounts', + 'ListBackends', + 'RenameAccount', + 'ViewAccount' + ] + + +class AddAccount(AppCommand): + """ + Add an account. + + `python manage.py add-account {name} {backend}` + """ + + def get_options(self): + return [ + Option(dest='name'), + Option(dest='backend') + ] + + def run(self, name, backend): + + # Validate the parameters + form = AddAccountForm(name=name, backend=backend) + if not form.validate(): + self.err(**form.errors) + return + + # Ask the user for the backend configuration options + backend = Backend.get_backend(backend) + config = {'backend': form.data['backend']} + for field in backend.config_form(): + self.out((field.label.text, 'blue')) + value = input('> ').strip() + if value: + config[field.name] = value + + # Validate the users configuration + result = backend.validate_config(**config) + if not result[0]: + self.err('Invalid backend config:', **result[1]) + return + + # Create the new account + account = Account( + name=form.data['name'], + backend=config + ) + account.insert() + + self.out(('Account added: {0}'.format(account.api_key), 'bold_green')) + + +class ConfigAccount(AppCommand): + """ + Configure an account storage backend. + + `python manage.py config-account {name} {congig-file.json}` + """ + + def get_options(self): + return [Option(dest='name')] + + def run(self, name): + + # Validate the parameters + form = ConfigAccountBackendForm(name=name) + if not form.validate(): + self.err(**form.errors) + return + + # Find the account to be configured + account = Account.one(Q.name == form.data['name']) + + # Let the user know to use dash to clear existing values + self.out(( + '* Enter dash (-) to clear the existing value', + 'underline_bold_blue' + )) + + # Ask the user for the backend configuration options + backend = Backend.get_backend(account.backend['backend']) + config = {'backend': account.backend['backend']} + for field in backend.config_form(): + + # Request the value + self.out((field.label.text, 'blue')) + value = input('({0}) > '.format( + account.backend.get(field.name, ''))) + value = value.strip() + + # Check if the value should be set to the original, cleared or used + # as provided. + if value: + if value == '-': + continue + else: + config[field.name] = value + else: + if account.backend.get(field.name): + config[field.name] = account.backend.get(field.name) + + # Validate the users configuration + result = backend.validate_config(**config) + if not result[0]: + self.err('Invalid backend config:', **result[1]) + return + + # Update the accounts backend + account.backend = config + account.update('modified', 'backend') + + self.out(('Account configured', 'bold_green')) + + +class DeleteAccount(AppCommand): + """ + Delete an account. + + `python manage.py delete-account {name}` + """ + + def get_options(self): + return [Option(dest='name')] + + def run(self, name): + + # Validate the parameters + form = DeleteAccountForm(name=name) + if not form.validate(): + self.err(**form.errors) + return + + # Find the account to be deleted + account = Account.one(Q.name == form.data['name']) + + # Confirm the account deletion + if not self.confirm('Enter the following string to confirm you want to \ +delete this account deletion', account.name): + return + + # Delete the account + account.delete() + + self.out(('Account deleted', 'bold_green')) + + +class GenerateNewAPIKey(AppCommand): + """ + Generate a new API key for an account. + + `python manage.py generate-new-api-key {name}` + """ + + def get_options(self): + return [Option(dest='name')] + + def run(self, name): + + # Validate the parameters + form = GenerateNewAccountAPIKeyForm(name=name) + if not form.validate(): + self.err(**form.errors) + return + + # Find the account to generate a new API key for + account = Account.one(Q.name == form.data['name']) + + # Confirm the account deletion + if not self.confirm('Enter the following string to confirm you want to \ +generate a new API key for this account', account.name): + return + + # Generate a new API key for the account + account.api_key = account.generate_api_key() + account.update('modified', 'api_key') + + self.out(('New key generated: ' + account.api_key, 'bold_green')) + + +class ListAccounts(AppCommand): + """ + List all accounts. + + `python manage.py list-accounts -q` + """ + + def get_options(self): + return [Option( + '-q', + dest='q', + default='', + help='filter the list by the specified value' + )] + + def run(self, q=''): + + # If `q` is specified we filter the list of accounts to only accounts + # with names that contain the value of `q`. + filter = {} + if q: + filter = Q.name == re.compile(re.escape(q), re.I) + + # Get the list of accounts + accounts = Account.many(filter, sort=[('name', ASC)]) + + # Print a list of accounts + output = [] + + if q: + output.append(( + "Accounts matching '{0}' ({1}):".format(q, len(accounts)), + 'underline_bold_blue' + )) + else: + output.append(( + 'Accounts ({0}):'.format(len(accounts)), + 'underline_bold_blue' + )) + + for account in accounts: + output.append(( + '- {name} (using {backend})'.format( + name=account.name, + backend=account.backend.get('backend', 'unknown') + ), + 'blue' + )) + + self.out(*output) + + +class ListBackends(AppCommand): + """ + List the available storage backends. + + `python manage.py list-backends` + """ + + def run(self): + + # Print a list of available backends + output = [( + 'Backends ({0}):'.format(len(Backend.list_backends())), + 'underline_bold_blue' + )] + for backend in Backend.list_backends(): + output.append(('- ' + backend, 'blue')) + + self.out(*output) + + +class RenameAccount(AppCommand): + """ + Rename an existing account. + + `python manage.py rename-account {name} {new_name}` + """ + + def get_options(self): + return [ + Option(dest='name'), + Option(dest='new_name') + ] + + def run(self, name, new_name): + + # Validate the parameters + form = RenameAccountForm(name=name, new_name=new_name) + + if not form.validate(): + self.err(**form.errors) + return + + # Find the account to rename + account = Account.one(Q.name == form.data['name']) + + # Set the accounts new name + account.name = form.data['new_name'] + account.update('modified', 'name') + + self.out(('Account renamed: ' + account.name, 'bold_green')) + + +class ViewAccount(AppCommand): + """ + View the details for an account. + + `python manage.py view-account {name}` + """ + + def get_options(self): + return [Option(dest='name')] + + def run(self, name): + + # Validate the parameters + form = ViewAccountForm(name=name) + if not form.validate(): + self.err(**form.errors) + return + + # Find the account to view + account = Account.one(Q.name == form.data['name']) + + # Output details of the account + output = [("About '{0}':".format(account.name), 'underline_bold_blue')] + + pairs = [ + ('created', account.created), + ('modified', account.modified), + ('assets', Asset.count(Q.account == account)), + ('api_key', account.api_key), + ('backend', account.backend.get('backend', 'unknown')) + ] + + for key in sorted(account.backend.keys()): + if key == 'backend': + continue + pairs.append(('> ' + key, account.backend[key])) + + # Find the longest key so we pad/align values + width = sorted([len(p[0]) for p in pairs])[-1] + 2 + + for pair in pairs: + pair_str = '- {key:-<{width}} {value}'.format( + key=pair[0].ljust(width, '-'), + value=pair[1], + width=width + ) + output.append((pair_str, 'blue')) + + self.out(*output) \ No newline at end of file diff --git a/commands/app.py b/commands/app.py new file mode 100644 index 0000000..6509775 --- /dev/null +++ b/commands/app.py @@ -0,0 +1,53 @@ +""" +Command line tools for managing the application. +""" + +from flask import current_app +from mongoframes import * + +from commands import AppCommand +from models.accounts import Account +from models.assets import Asset, Variation + + +class Drop(AppCommand): + """ + Drop the application. + """ + + def run(self): + + # Confirm the drop + if not self.confirm('Enter the following string to confirm drop', \ + 'hangar51'): + return + + # Delete all accounts, assets and files + accounts = Account.many() + for account in accounts: + account.purge() + + # Drop the collections + Asset.get_collection().drop() + Account.get_collection().drop() + + +class Init(AppCommand): + """ + Initialize the application. + """ + + models = [ + Account, + Asset + ] + + def run(self): + + # Initialzie the application database + for model in self.models: + + # (Re)create indexes for collections that specify them + if hasattr(model, '_indexes'): + model.get_collection().drop_indexes() + model.get_collection().create_indexes(model._indexes) \ No newline at end of file diff --git a/fabfile.py b/fabfile.py new file mode 100644 index 0000000..26f9591 --- /dev/null +++ b/fabfile.py @@ -0,0 +1,121 @@ +""" +Fabric is used to deploy changes and run tasks across multiple environments. +""" + +import os, sys, shutil, time, datetime + +from fabric.api import * +from fabric.colors import red, blue, green, yellow +from fabric.contrib.console import confirm +from fabric.contrib.files import exists +from fabric.operations import local + + +# Setup up the environments +env.branch = 'master' +env.env = 'local' +env.home = '' +env.hosts = ['localhost'] +env.project_repo = '' +env.runner = local + +@task +def prod(): + env.home = '/sites/hangar51' + env.env = 'prod' + env.hosts = [''] + env.runner = run + env.user = 'hangar51' + + +# Tasks + +@task +def deploy(): + """Push changes to the server""" + with cd(env.home): + # Checkout to the relevant branch + env.runner('git checkout ' + env.branch) + + # Pull down the changes from the repo + env.runner('git pull') + +@task(alias='m') +def manage(manage_task): + """Run a manage task""" + if env.env == 'local': + env.runner('python manage.py {task}'.format(task=manage_task)) + else: + env.runner( + 'bin/python manage.py --env {env} {task}'.format( + env=env.env, + task=manage_task + ) + ) + + +@task(alias='pip') +def install_pip_requirements(): + """Install all pip requirements""" + + with cd(env.home): + if env.env == 'local': + # For local environments assume the virtual environment is activated + # and attempt the pip install. + local('pip install -r requirements.txt') + else: + env.runner('bin/pip install -r requirements.txt') + +@task(alias='up') +def start_app(): + """Start the application""" + if env.env == 'local': + local('python app.py') + else: + sudo('/usr/bin/supervisorctl start hangar51_server', shell=False) + +@task(alias='down') +def stop_app(): + """Stop the application""" + if env.env == 'local': + # When the app is run locally it doesn't deamonize + pass + + else: + sudo('/usr/bin/supervisorctl stop hangar51_server', shell=False) + +@task(alias='cycle') +def cycle_app(): + """Cycle the application""" + stop_app() + start_app() + +@task(alias='tasks_up') +def start_background_tasks(): + """Start the celery task queue""" + if env.env == 'local': + # When run locally we only fire up the worker (not beats) + local('celery -A run_tasks worker') + + else: + # Start the tasks worker + sudo('/usr/bin/supervisorctl start hangar51_worker', shell=False) + + # Start the beat + if env.host_string == env.hosts[0]: + sudo('/usr/bin/supervisorctl start hangar51_beat', shell=False) + +@task(alias='tasks_down') +def stop_background_tasks(): + """Stop the celery task queue""" + if env.env == 'local': + # When the tasks are run locally celery doesn't deamonize + pass + + else: + # Stop the tasks worker + sudo('/usr/bin/supervisorctl stop hangar51_beat', shell=False) + + # Stop the beat + if env.host_string == env.hosts[0]: + sudo('/usr/bin/supervisorctl stop hangar51_worker', shell=False) \ No newline at end of file diff --git a/face_detect_requirements.txt b/face_detect_requirements.txt new file mode 100644 index 0000000..8e1580d --- /dev/null +++ b/face_detect_requirements.txt @@ -0,0 +1,57 @@ +# To support face detection you will need to install the following set of +# requirements and in addition: +# +# - Make sure you have cmake 3+ installed +# - Make sure you have boost installed +# (sudo apt-get install libboost-python-dev cmake) + +amqp==1.4.9 +anyjson==0.3.3 +awesome-slugify==1.6.5 +billiard==3.3.0.23 +blessings==1.6 +blinker==1.4 +boto3==1.3.0 +botocore==1.4.5 +celery==3.1.23 +cffi==1.3.1 +cycler==0.10.0 +dask==0.8.2 +decorator==4.0.9 +dlib==18.17.100 +docopt==0.4.0 +docutils==0.12 +Flask==0.10.1 +Flask-Script==2.0.5 +futures==2.2.0 +gunicorn==19.3.0 +itsdangerous==0.24 +Jinja2==2.8 +jmespath==0.7.1 +kombu==3.0.35 +MarkupSafe==0.23 +matplotlib==1.5.1 +mongoframes==1.0.0 +networkx==1.11 +nose==1.3.7 +numpy==1.11.0 +Pillow==3.1.1 +py==1.4.31 +pycparser==2.14 +pymongo==3.2.2 +pyparsing==2.1.1 +pytest==2.8.7 +pytest-flask==0.10.0 +python-dateutil==2.4.2 +pytz==2016.2 +raven==5.5.0 +regex==2016.3.2 +requests==2.7.0 +scikit-image==0.12.3 +scipy==0.18.1 +shortuuid==0.4.3 +six==1.9.0 +toolz==0.7.4 +Unidecode==0.4.19 +Werkzeug==0.10.4 +WTForms==2.0.2 \ No newline at end of file diff --git a/forms/__init__.py b/forms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/forms/accounts.py b/forms/accounts.py new file mode 100644 index 0000000..8da45f2 --- /dev/null +++ b/forms/accounts.py @@ -0,0 +1,83 @@ +import json +from mongoframes import * +import os +from wtforms import Form, ValidationError +from wtforms.fields import * +from wtforms.validators import * + +from backends import Backend +from models.accounts import Account + +__all__ = [ + 'AddAccountForm', + 'ConfigAccountBackendForm', + 'DeleteAccountForm', + 'GenerateNewAccountAPIKeyForm', + 'RenameAccountForm', + 'ViewAccountForm' + ] + + +# A valid name matcher +valid_name_regex = Regexp('\A[A-Za-z0-9_]+\Z') + + +class _FindAccountForm(Form): + + name = StringField('name', [Required(), Length(max=20)]) + + def validate_name(form, field): + """Validate that the account name exists""" + if Account.count(Q.name == field.data) == 0: + raise ValidationError('Account not found.') + + +class AddAccountForm(Form): + + name = StringField('name', [ + Required(), + Length(min=2, max=20), + valid_name_regex + ]) + backend = StringField('backend', [Required()]) + + def validate_backend(form, field): + """Validate that the backend is supported""" + if not Backend.get_backend(field.data): + raise ValidationError('Not a supported backend.') + + def validate_name(form, field): + """Validate that the account name isn't taken""" + if Account.count(Q.name == field.data) > 0: + raise ValidationError('Account name already taken.') + return + + +class ConfigAccountBackendForm(_FindAccountForm): + pass + + +class DeleteAccountForm(_FindAccountForm): + pass + + +class GenerateNewAccountAPIKeyForm(_FindAccountForm): + pass + + +class RenameAccountForm(_FindAccountForm): + + new_name = StringField('name', [ + Required(), + Length(max=20), + valid_name_regex + ]) + + def validate_new_name(form, field): + """Validate that the new account name isn't taken""" + if Account.count(Q.name == field.data) > 0: + raise ValidationError('Account name already taken.') + + +class ViewAccountForm(_FindAccountForm): + pass \ No newline at end of file diff --git a/forms/assets.py b/forms/assets.py new file mode 100644 index 0000000..db9d3b9 --- /dev/null +++ b/forms/assets.py @@ -0,0 +1,230 @@ +from flask import g +import json +from mongoframes import * +from numbers import Number +import re +from wtforms import Form, ValidationError +from wtforms.fields import * +from wtforms.validators import * + +from models.assets import Asset + +__all__ = [ + 'DownloadForm', + 'GenerateVariationsForm', + 'GetForm', + 'ListForm', + 'SetExpiresForm', + 'UploadForm' + ] + + +class _FindAssetForm(Form): + + uid = StringField('uid') + + def validate_uid(form, field): + """Validate that the asset exists""" + asset = Asset.one(And(Q.account == g.account, Q.uid == field.data)) + if not asset or asset.expired: + raise ValidationError('Asset not found.') + + +class DownloadForm(_FindAssetForm): + + pass + + +class GenerateVariationsForm(_FindAssetForm): + + variations = StringField('variations', [Required()]) + on_delivery = StringField( + 'on_delivery', + [Optional(), AnyOf(['forget', 'wait'])] + ) + webhook = StringField('webhook', [Optional(), URL()]) + + def validate_variations(form, field): + # A valid name matcher + valid_name_regex = re.compile('\A[A-Za-z0-9\-]+\Z') + + # Check a valid JSON string has been provided + try: + variations = json.loads(field.data) + except ValueError: + raise ValidationError('Invalid JSON string') + + # Validate the structure of the JSON data (must be a dictionary with at + # least one item). + if not isinstance(variations, dict) or len(variations) == 0: + raise ValidationError( + 'Must be a dictionary containing at least one item') + + # Validate each item in the dictionary contains a valid set of image + # operations. + supported_formats = Asset.SUPPORTED_IMAGE_EXT['out'] + for name, ops in variations.items(): + + # Check the variation name is allowed + if not valid_name_regex.match(name): + raise ValidationError( + 'Invalid variations name (a-Z, 0-9, -)') + + # Check a list of ops has been specified for the variation + if len(ops) == 0: + raise ValidationError('Empty ops list'.format(name=name)) + + for op in ops: + # Validate op is a list with 2 values + if not isinstance(op, list) and len(op) != 2: + raise ValidationError('Invalid op [name, value]') + + # Crop + if op[0] == 'crop': + # Crop region must be a 4 item list + if not isinstance(op[1], list) and len(op[1]) != 4: + raise ValidationError( + 'Invalid crop region [t, r, b, l] (0.0-1.0)') + + # Check each value is a number + if False in [isinstance(v, Number) for v in op[1]]: + raise ValidationError( + 'Invalid crop region [t, r, b, r] (0.0-1.0)') + + # All values must be between 0 and 1 + if False in [v >= 0 and v <= 1 for v in op[1]]: + raise ValidationError( + 'Invalid crop region [t, r, b, l] (0.0-1.0)') + + # Width and height must both be great than 0 + if (op[1][2] - op[1][0]) <= 0 or (op[1][1] - op[1][3]) <= 0: + raise ValidationError( + 'Invalid crop region, width and height must be ' + + 'greater than 0' + ) + + # Face + elif op[0] == 'face': + + # Face options must be a dictionary + if not isinstance(op[1], dict): + raise ValidationError( + "Invalid ouput format {'bias': [0.0, -0.2], ...}") + + # Bias + if 'bias' in op[1]: + bias = op[1]['bias'] + + # Bias must have 2 values + if not isinstance(bias, list) and len(bias) != 2: + raise ValidationError( + 'Invalid face bias [horz, vert] as decimals') + + # Bias values must be numbers + if not (isinstance(bias[0], Number) \ + and isinstance(bias[1], Number)): + raise ValidationError( + 'Invalid face bias [horz, vert] as decimals') + + # Padding + if 'padding' in op[1]: + padding = op[1]['padding'] + + # Padding must be a number + if not isinstance(padding, Number): + raise ValidationError( + 'Invalid face padding must be a number') + + # Min padding must be a number + if 'min_padding' in op[1]: + padding = op[1]['min_padding'] + + # Min padding must be a number + if not isinstance(padding, Number): + raise ValidationError( + 'Invalid face min padding must be a number') + + # Fit + elif op[0] == 'fit': + # Dimensions must be a 2 item list + if not isinstance(op[1], list) and len(op[1]) != 2: + raise ValidationError( + 'Invalid fit dimensions [width, height] in pixels') + + # Dimensions must be integers + if not (isinstance(op[1][0], int) \ + and isinstance(op[1][1], int)): + raise ValidationError( + 'Invalid fit dimensions [width, height] in pixels') + + # Dimensions must both be greater than 0 + if not (op[1][0] > 0 and op[1][1] > 0): + raise ValidationError( + 'Fit dimensions must be greater than 0') + + # Ouput + elif op[0] == 'output': + # Format options must be a dictionary + if not isinstance(op[1], dict): + raise ValidationError( + "Invalid ouput format {'format': 'jpg', ...}") + + # Format must be supported + if op[1].get('format') not in supported_formats: + raise ValidationError( + 'Output format not supported ({formats})'.format( + formats='|'.join(supported_formats) + ) + ) + + # If quality is specified + fmt = op[1].get('format') + if 'quality' in op[1]: + # Must be a format that supports quality + if fmt not in ['jpg', 'webp']: + raise ValidationError( + 'Output quality only allowed for jpg and webp') + + # Quality must be an integer between 1 and 100 + quality = op[1].get('quality') + if not isinstance(quality, int) \ + or quality < 0 or quality > 100: + raise ValidationError( + 'Invalid output quality (0-100)') + + # Rotate + elif op[0] == 'rotate': + if op[1] not in [0, 90, 180, 270]: + raise ValidationError( + 'Rotate angle must be 0, 90, 180 or 270') + + # Unknown ops + else: + raise ValidationError('Unknown op {op}'.format(op=op[0])) + + +class GetForm(_FindAssetForm): + + pass + + +class ListForm(Form): + + q = StringField('q') + type = StringField('type', [Optional(), AnyOf(['file', 'image'])]) + page = IntegerField('Page', default=1) + order = StringField( + 'order', + [Optional(), AnyOf(['created', '-created', 'store_key'])] + ) + + +class SetExpiresForm(_FindAssetForm): + + expires = FloatField('expires', [Optional(), NumberRange(min=1)]) + + +class UploadForm(Form): + + name = StringField('name') + expires = FloatField('expires', [Optional(), NumberRange(min=1)]) \ No newline at end of file diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..70c626d --- /dev/null +++ b/manage.py @@ -0,0 +1,38 @@ +""" +Command line tools for managing the application. +""" + +from flask.ext.script import Manager + +from app import create_app +import commands + + +# Set up the manager +manager = Manager(create_app) +manager.add_option( + '-e', + '--env', + choices=['dev', 'local', 'prod'], + default='local', + dest='env', + required=False + ) + +# Add commands +manager.add_command('drop', commands.Drop) +manager.add_command('init', commands.Init) + +# Accounts +manager.add_command('add-account', commands.AddAccount) +manager.add_command('config-account', commands.ConfigAccount) +manager.add_command('delete-account', commands.DeleteAccount) +manager.add_command('generate-new-api-key', commands.GenerateNewAPIKey) +manager.add_command('list-accounts', commands.ListAccounts) +manager.add_command('list-backends', commands.ListBackends) +manager.add_command('rename-account', commands.RenameAccount) +manager.add_command('view-account', commands.ViewAccount) + + +if __name__ == "__main__": + manager.run() \ No newline at end of file diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/accounts.py b/models/accounts.py new file mode 100644 index 0000000..82fcd6e --- /dev/null +++ b/models/accounts.py @@ -0,0 +1,58 @@ +from mongoframes import * +from uuid import uuid4 + +from backends import Backend + +__all__ = ['Account'] + + +class Account(Frame): + """ + Accounts provide access control and backend configuration. + """ + + _fields = { + 'created', + 'modified', + 'name', + 'api_key', + 'backend' + } + _indexes = [ + IndexModel([('name', ASC)], unique=True), + IndexModel([('api_key', ASC)], unique=True) + ] + + def __str__(self): + return self.name + + def get_backend_instance(self): + """Return a configured instance of the backend for the account""" + backendCls = Backend.get_backend(self.backend['backend']) + return backendCls(**self.backend) + + def purge(self): + """Deletes the account along with all related assets and files.""" + from models.assets import Asset + + # Purge all assets + for asset in Asset.many(Q.account == self): + asset.account = self + asset.purge() + + # Delete self + self.delete() + + @staticmethod + def generate_api_key(): + return str(uuid4()) + + @staticmethod + def on_insert(sender, frames): + # Set an API key for newely created accounts + for frame in frames: + frame.api_key = sender.generate_api_key() + +Account.listen('insert', Account.timestamp_insert) +Account.listen('insert', Account.on_insert) +Account.listen('update', Account.timestamp_update) \ No newline at end of file diff --git a/models/assets.py b/models/assets.py new file mode 100644 index 0000000..04cc4e2 --- /dev/null +++ b/models/assets.py @@ -0,0 +1,497 @@ +from datetime import datetime, timezone +from flask import current_app +import io +import mimetypes +from mongoframes import * +import numpy +from PIL import Image +import time + +# Fix for missing mimetypes +mimetypes.add_type('text/csv', '.csv') +mimetypes.add_type('image/webp', '.webp') + + +from utils import get_file_length, generate_uid + +__all__ = [ + 'Asset', + 'Variation' + ] + + +class Variation(SubFrame): + """ + A variation of an asset transformed by one or more operations. + """ + + _fields = { + 'name', + 'version', + 'ext', + 'meta', + 'store_key' + } + + def __str__(self): + return self.store_key + + @staticmethod + def find_face(im, bias=None, padding=0, min_padding=0): + """ + Find a face in an image and return it's coordinates. If no face can be + found then None is returned. + """ + + # Import optional libraries required for face detection + import dlib + from skimage import io as skimage_io + skimage_io.use_plugin('pil') + + # Check we have already aquired a face detector and if not do so now + if not hasattr(Variation, '_face_detector'): + Variation._face_detector = dlib.get_frontal_face_detector() + + # Convert the image to an array that can be read by skimage + w, h = im.size + skimage_im = numpy.array(im.getdata(), numpy.uint8).reshape(h, w, 3) + + d = dlib.get_frontal_face_detector() + faces = d(skimage_im, 1) + + # Detect faces + faces = Variation._face_detector(skimage_im, 1) + + # If no faces were detected there's nothing more to do, we return `None` + if len(faces) == 0: + return + + # If a face was found apply any bias and padding to it + face = faces[0] + rect = [face.left(), face.top(), face.right(), face.bottom()] + + # Apply bias + if bias: + # Shift the center of the face + bias_x = int(face.width() * bias[0]) + bias_y = int(face.width() * bias[1]) + rect[0] += bias_x + rect[1] += bias_y + rect[2] += bias_x + rect[3] += bias_y + + # Apply padding + if padding > 0: + + # Determine the maximum amount of padding that can be applied in any + # direction. + max_padding = rect[0] + max_padding = min(rect[1], max_padding) + max_padding = min(rect[2], max_padding) + max_padding = min(rect[3], max_padding) + + # Calculate the padding to apply + pad = [ + int(face.width() * padding), + int(face.height() * padding) + ] + + # Ensure that the minimum padding is observed + if min_padding > 0: + pad = [ + min(pad[0], max(max_padding, int(pad[0] * min_padding))), + min(pad[1], max(max_padding, int(pad[1] * min_padding))) + ] + + # Apply the padding to the face rectangle + rect[0] = max(rect[0] - pad[0], 0) + rect[1] = max(rect[1] - pad[1], 0) + rect[2] = min(rect[2] + pad[0], im.size[0]) + rect[3] = min(rect[3] + pad[1], im.size[1]) + + return rect + + @staticmethod + def get_store_key(asset, variation): + """Return the store key for an asset variation""" + return '.'.join([ + asset.name, + asset.uid, + variation.name, + variation.version, + variation.ext + ]) + + @staticmethod + def optimize_ops(ops): + """ + Optimize/reduce a list of image operations to the fewest possible + operations to achieve the same image transform. + + Due to the limited set of image operations that are possible we can + always reduce the operations list to a limited ordered set, e.g: + + - crop + - rotate + - face (if supported) + - fit + - output + + """ + + def rotate_crop_cw(crop): + # Rotate a crop region 90 degrees clockwise + crop = [c - 0.5 for c in crop] + crop[1] *= -1 + crop[3] *= -1 + crop = [c + 0.5 for c in crop] + crop.append(crop.pop(0)) + return crop + + # Initial transform settings + angle = 0 + crop = [0, 1, 1, 0] + fit = None + fmt = None + face = None + + # Optimize the ops + for op in ops: + if op[0] == 'crop': + # Apply the crop as a crop of the last crop + sub_crop = list(op[1]) + + # Rotate the crop to be aligned with the current angle + for i in range(0, int(angle / 90)): + sub_crop = rotate_crop_cw(sub_crop) + + # Crop the existing crop + w = crop[1] - crop[3] + h = crop[2] - crop[0] + + crop = [ + crop[0] + (h * sub_crop[0]), # Top + crop[1] - (w * (1 - sub_crop[1])), # Right + crop[2] - (h * (1 - sub_crop[2])), # Bottom + crop[3] + (w * sub_crop[3]) # Left + ] + + elif op[0] == 'face': + face = op[1] + + elif op[0] == 'fit': + # Set the fit dimensions allowing for the current orientation of + # the image. + if angle in [0, 180]: + fit = [op[1][0], op[1][1]] + else: + fit = [op[1][1], op[1][0]] + + elif op[0] == 'output': + fmt = op[1] + + elif op[0] == 'rotate': + # Set the rotation of the image clamping it to (0, 90, 180, 270) + angle = (angle + op[1]) % 360 + + # Build the optimized list of ops + less_ops = [] + + # Crop + if crop != [0, 1, 1, 0]: + less_ops.append(['crop', crop]) + + # Rotate + if angle != 0: + less_ops.append(['rotate', angle]) + + # Face + if face is not None: + less_ops.append(['face', face]) + + # Fit + if fit is not None: + less_ops.append(['fit', fit]) + + # Output + if fmt is not None: + less_ops.append(['output', fmt]) + + return less_ops + + @staticmethod + def transform_image(im, ops): + """ + Perform a list of operations against an image and return the resulting + image. + """ + + # Optimize the list of operations + # + # IMPORTANT! The optimized operations method doesn't work correctly in + # a number of cases and therefore has been removed for the moment until + # those issues can be resolved (hint I think the stack of operations + # needs to be optimized in reverse). + # + # ~ Anthony Blackshaw , 31 August 2017 + # + # ops = Variation.optimize_ops(ops) + + # Perform the operations + fmt = {'format': 'jpeg', 'ext': 'jpg'} + for op in ops: + + # Crop + if op[0] == 'crop': + im = im.crop([ + int(op[1][3] * im.size[0]), # Left + int(op[1][0] * im.size[1]), # Top + int(op[1][1] * im.size[0]), # Right + int(op[1][2] * im.size[1]) # Bottom + ]) + + # Face + elif op[0] == 'face': + # If face detection isn't supported ignore the operation + if not current_app.config['SUPPORT_FACE_DETECTION']: + continue + + # Ensure the image we use to find a face with is RGB format + face_im = im.convert('RGB') + + # Due to performance constraints we don't attempt face + # recognition on images over 2000x2000 pixels, instead we scale + # the images within these bounds ahead of the action. + ratio = 1.0 + if im.size[0] > 2000 or im.size[1] > 2000: + face_im.thumbnail((2000, 2000), Image.ANTIALIAS) + ratio = float(im.size[0]) / float(face_im.size[0]) + + # Attempt to find the face + face_rect = Variation.find_face(face_im, **op[1]) + + # If no face is detected there's nothing more to do + if face_rect is None: + continue + + # Scale the rectangle by the reduced ratio + if ratio: + face_rect = [int(d * ratio) for d in face_rect] + + # If a face was found crop it from the image + im = im.crop(face_rect) + + # Fit + elif op[0] == 'fit': + im.thumbnail(op[1], Image.ANTIALIAS) + + # Rotate + elif op[0] == 'rotate': + if op[1] == 90: + im = im.transpose(Image.ROTATE_270) + + elif op[1] == 180: + im = im.transpose(Image.ROTATE_180) + + elif op[1] == 270: + im = im.transpose(Image.ROTATE_90) + + # Output + elif op[0] == 'output': + fmt = op[1] + + # Set the extension for the output and the format required by + # Pillow. + fmt['ext'] = fmt['format'] + if fmt['format'] == 'jpg': + fmt['format'] = 'jpeg' + + # Add the optimize flag for JPEGs and PNGs + if fmt['format'] in ['jpeg', 'png']: + fmt['optimize'] = True + + # Allow gifs to store multiple frames + if fmt['format'] in ['gif', 'webp']: + fmt['save_all'] = True + fmt['optimize'] = True + + # Variations are output in web safe colour modes, if the + # original image isn't using a web safe colour mode supported by + # the output format it will be converted to one. + if fmt['format'] == 'gif' and im.mode != 'P': + im = im.convert('P') + + elif fmt['format'] == 'jpeg' and im.mode != 'RGB': + im = im.convert('RGB') + + elif fmt['format'] == 'png' \ + and im.mode not in ['P', 'RGB', 'RGBA']: + im = im.convert('RGB') + + elif fmt['format'] == 'webp' and im.mode != 'RGBA': + im = im.convert('RGBA') + + return im, fmt + + +class Asset(Frame): + """ + An asset stored in Hangar51. + """ + + _fields = { + 'created', + 'modified', + 'account', + 'name', + 'uid', + 'ext', + 'type', + 'expires', + 'meta', + 'store_key', + 'variations' + } + _indexes = [ + IndexModel([('account', ASC), ('uid', ASC)], unique=True) + ] + + _private_fields = ['_id', 'account'] + + _default_projection = {'variations': {'$sub': Variation}} + + # A list of support image extensions + SUPPORTED_IMAGE_EXT = { + 'in': [ + 'bmp', + 'gif', + 'jpg', 'jpeg', + 'png', + 'tif', 'tiff', + 'webp' + ], + 'out': ['jpg', 'gif', 'png', 'webp'] + } + + def __str__(self): + return self.store_key + + @property + def content_type(self): + """Return a content type for the asset based on the extension""" + return self.guess_content_type(self.store_key) + + @property + def expired(self): + if self.expires is None: + return False + now = time.mktime(datetime.now(timezone.utc).timetuple()) + return self.expires < now + + def add_variation(self, f, im, name, ops): + """Add a variation to the asset""" + from models.accounts import Account + + # Make sure we have access to the associated account frame + if not isinstance(self.account, Account): + self.account = Account.one(Q._id == self.account) + + # Transform the original image to generate the variation + vim = None + if im.format.lower() == 'gif' and im.is_animated: + # By-pass transforms for animated gifs + fmt = {'ext': 'gif', 'fmt': 'gif'} + + else: + # Transform the image based on the variation + vim = im.copy() + vim, fmt = Variation.transform_image(vim, ops) + + # Prepare the variation file for storage + f = io.BytesIO() + vim.save(f, **fmt) + f.seek(0) + + # Add the variation to the asset + variation = Variation( + name=name, + ext=fmt['ext'], + meta={ + 'length': get_file_length(f), + 'image': { + 'mode': (vim or im).mode, + 'size': (vim or im).size + } + } + ) + + # Set a version + variation.version = generate_uid(3) + while self.get_variation(name, variation.version): + variation.version = generate_uid(3) + + # Store the variation + variation.store_key = Variation.get_store_key(self, variation) + backend = self.account.get_backend_instance() + backend.store(f, variation.store_key) + + # We use the $push operator to store the variation to prevent race + # conditions if multiple processes attempt to update the assets + # variations at the same time. + self.get_collection().update( + {'_id': self._id}, + {'$push': {'variations': variation._document}} + ) + + return variation + + def get_variation(self, name, version): + """Return a variation with the given name and version""" + if not self.variations: + return + + # Attempt to find the variation + for variation in self.variations: + if variation.name == name and variation.version == version: + return variation + + def purge(self): + """Deletes the asset along with all related files.""" + from models.accounts import Account + + # Make sure we have access to the associated account frame + if not isinstance(self.account, Account): + self.account = Account.one(Q._id == self.account) + + # Get the backend required to delete the asset + backend = self.account.get_backend_instance() + + # Delete the original file + backend.delete(self.store_key) + + # Delete all variation files + for variation in self.variations: + backend.delete(variation.store_key) + + self.delete() + + @staticmethod + def get_store_key(asset): + """Return the store key for an asset""" + return '.'.join([asset.name, asset.uid, asset.ext]) + + @staticmethod + def get_type(ext): + """Return the type of asset for the given filename extension""" + if ext.lower() in Asset.SUPPORTED_IMAGE_EXT['in']: + return 'image' + return 'file' + + @staticmethod + def guess_content_type(filename): + """Guess the content type for a given filename""" + return mimetypes.guess_type(filename)[0] + + +Asset.listen('insert', Asset.timestamp_insert) +Asset.listen('update', Asset.timestamp_update) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1fe6d14 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,51 @@ +amqp==1.4.9 +anyjson==0.3.3 +awesome-slugify==1.6.5 +billiard==3.3.0.23 +blessings==1.6 +blinker==1.4 +boto3==1.3.0 +botocore==1.4.5 +celery==3.1.23 +#cffi==1.3.1 +cycler==0.10.0 +dask==0.8.2 +decorator==4.0.9 +#dlib==18.17.100 +docopt==0.4.0 +docutils==0.12 +fake-factory==0.6.0 +Flask==0.10.1 +Flask-Script==2.0.5 +futures==2.2.0 +gunicorn==19.3.0 +itsdangerous==0.24 +Jinja2==2.8 +jmespath==0.7.1 +kombu==3.0.35 +MarkupSafe==0.23 +#matplotlib==1.5.1 +MongoFrames==1.2.4 +networkx==1.11 +nose==1.3.7 +numpy==1.11.0 +Pillow==5.0.0 +py==1.4.31 +pycparser==2.14 +pymongo==3.3.0 +pyparsing==2.1.1 +pytest==2.8.7 +pytest-flask==0.10.0 +python-dateutil==2.5.3 +pytz==2016.10 +raven==5.5.0 +regex==2016.3.2 +requests==2.7.0 +#scikit-image==0.12.3 +#scipy==0.18.1 +shortuuid==0.4.3 +six==1.10.0 +toolz==0.7.4 +Unidecode==0.4.19 +Werkzeug==0.10.4 +WTForms==2.0.2 diff --git a/root/nginx/local_template b/root/nginx/local_template new file mode 100644 index 0000000..a58cb25 --- /dev/null +++ b/root/nginx/local_template @@ -0,0 +1,25 @@ +server { + listen 80; + server_name hangar51.local; + client_max_body_size 200M; + + # Compression + gzip on; + gzip_http_version 1.1; + gzip_vary on; + gzip_comp_level 6; + gzip_proxied any; + gzip_types text/plain application/json text/javascript; + gzip_buffers 16 8k; + gzip_disable "MSIE [1-6]\.(?!.*SV1)"; + + # Proxying connections to application server + location / { + proxy_pass http://127.0.0.1:5152/; + proxy_redirect off; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} \ No newline at end of file diff --git a/root/nginx/prod_template b/root/nginx/prod_template new file mode 100644 index 0000000..111951a --- /dev/null +++ b/root/nginx/prod_template @@ -0,0 +1,42 @@ +server { + # Force HTTP to HTTPS + listen 51.254.63.38:80; + server_name hangar51.getme.co.uk; + rewrite ^(.*) https://hangar51.getme.co.uk$1 redirect; +} + +server { + listen 51.254.63.38:443; + server_name hangar51.getme.co.uk; + client_max_body_size 200M; + + # SSL + ssl on; + ssl_certificate /etc/nginx/ssl/getme.co.uk.crt; + ssl_certificate_key /etc/nginx/ssl/getme.co.uk.key; + add_header Strict-Transport-Security max-age=15768000; + + # Compression + gzip on; + gzip_http_version 1.1; + gzip_vary on; + gzip_comp_level 6; + gzip_proxied any; + gzip_types text/plain application/json text/javascript; + gzip_buffers 16 8k; + gzip_disable "MSIE [1-6]\.(?!.*SV1)"; + + # Logging + access_log /sites/hangar51/logs/nginx.access.log main; + error_log /sites/hangar51/logs/nginx.error.log; + + # Proxying connections to application server + location / { + proxy_pass http://127.0.0.1:5152/; + proxy_redirect off; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} diff --git a/root/supervisor/prod_template b/root/supervisor/prod_template new file mode 100644 index 0000000..677607c --- /dev/null +++ b/root/supervisor/prod_template @@ -0,0 +1,26 @@ +[program:hangar51_server] +command=/sites/hangar51/bin/gunicorn --workers 2 --bind 0.0.0.0:5152 'app:create_app("prod")' +directory=/sites/hangar51 +user=hangar51 +autostart=true +autorestart=true +startsecs=3 +stopsignal=KILL + +[program:hangar51_beat] +command=/sites/hangar51/bin/celery -A run_tasks beat --env prod +directory=/sites/hangar51 +user=hangar51 +autostart=false +autorestart=true +startsecs=3 +stopsignal=KILL + +[program:hangar51_worker] +command=/sites/hangar51/bin/celery -A run_tasks worker --env prod +directory=/sites/hangar51 +user=hangar51 +autostart=true +autorestart=true +startsecs=3 +stopsignal=KILL \ No newline at end of file diff --git a/run_tasks.py b/run_tasks.py new file mode 100644 index 0000000..fff8ac5 --- /dev/null +++ b/run_tasks.py @@ -0,0 +1,23 @@ +import argparse +from celery import Celery + +from app import create_app +from tasks import setup_tasks + + +# Parse the command-line arguments +parser = argparse.ArgumentParser(description='Application server') +parser.add_argument( + '-e', + '--env', + choices=['dev', 'local', 'prod'], + default='local', + dest='env', + required=False + ) +args, unknown = parser.parse_known_args() + +# Create and run the application +celery = create_app(args.env).celery + +setup_tasks(celery) \ No newline at end of file diff --git a/settings/__init__.py b/settings/__init__.py new file mode 100644 index 0000000..f8fc562 --- /dev/null +++ b/settings/__init__.py @@ -0,0 +1,28 @@ +from datetime import timedelta + +class DefaultConfig: + + # Database + MONGO_URI = 'mongodb://localhost:27017/hangar51' + MONGO_USERNAME = 'hangar51' + MONGO_PASSWORD = '' + + # Debugging + DEBUG = False + SENTRY_DSN = '' + + # Networking + PREFERRED_URL_SCHEME = 'http' + SERVER_NAME = '' + + # Tasks (background) + CELERY_BROKER_URL = '' + CELERYBEAT_SCHEDULE = { + 'purge_expired_assets': { + 'task': 'purge_expired_assets', + 'schedule': timedelta(seconds=3600) + } + } + + # Additional variation support + SUPPORT_FACE_DETECTION = False \ No newline at end of file diff --git a/settings/local_template b/settings/local_template new file mode 100644 index 0000000..46648e0 --- /dev/null +++ b/settings/local_template @@ -0,0 +1,17 @@ +from . import DefaultConfig + + +class Config(DefaultConfig): + + # Debugging + DEBUG = True + + # Networking + SERVER_NAME = 'hangar51.local' + + # Tasks (background) + CELERY_BROKER_URL = 'amqp://hangar51:password@localhost/hangar51' + + # Additional variation support + SUPPORT_FACE_DETECTION = True + diff --git a/settings/test.py b/settings/test.py new file mode 100644 index 0000000..eb9efb9 --- /dev/null +++ b/settings/test.py @@ -0,0 +1,13 @@ +from . import DefaultConfig + +class Config(DefaultConfig): + + # Database + MONGO_URI = 'mongodb://localhost:27017/hangar51_test' + MONGO_PASSWORD = 'password' + + # Debugging + DEBUG = True + + # Networking + SERVER_NAME = '127.0.0.1' \ No newline at end of file diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..ae323d8 --- /dev/null +++ b/tasks.py @@ -0,0 +1,77 @@ +import argparse +from celery import Celery +from datetime import datetime, timezone +import json +from mongoframes import * +from PIL import Image +import requests +import time + +from app import create_app +from models.accounts import Account +from models.assets import Asset, Variation + +__all__ = ['setup_tasks'] + + +# Define the tasks for the application +def setup_tasks(celery): + + @celery.task(name='generate_variations') + def generate_variations(account_id, asset_uid, variations, webhook=''): + """Generate a set of variations for an image asset""" + + # Find the account + account = Account.by_id(account_id) + if not account: + return + + # Find the asset + asset = Asset.one(And(Q.account == account, Q.uid == asset_uid)) + if not asset: + return + + # Check the asset hasn't expired + if asset.expired: + return + + # Retrieve the original file + backend = account.get_backend_instance() + f = backend.retrieve(asset.store_key) + im = Image.open(f) + + # Generate the variations + new_variations = {} + for name, ops in variations.items(): + variation = asset.add_variation(im, name, ops) + new_variations[name] = variation.to_json_type() + + # Update the assets modified timestamp + asset.update('modified') + + # If a webhook has been provide call it with details of the new + # variations. + if webhook: + requests.get( + webhook, + data={ + 'account': account.name, + 'asset': asset.uid, + 'variations': json.dumps(variations) + } + ) + + @celery.task(name='purge_expired_assets') + def purge_expired_assets(): + """Purge assets which have expired""" + + # Get any asset that has expired + now = time.mktime(datetime.now(timezone.utc).timetuple()) + assets = Asset.many(And( + Exists(Q.expires, True), + Q.expires <= now + )) + + # Purge each asset + for asset in assets: + asset.purge() \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/templates/index.html @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e194060 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,267 @@ +""" +A set of pytest fixtures to help create test suites for Flask applications. +""" + +from flask import appcontext_tearing_down, \ + current_app, \ + message_flashed, \ + template_rendered, \ + url_for, \ + _app_ctx_stack + +import io +import json +from mongoframes import * +import os +import pytest + +from app import create_app +from commands import Drop, Init, AddAccount +from models.accounts import Account +from models.assets import Asset, Variation +from tasks import setup_tasks + +__all__ = [ + 'app', + 'celery_app', + 'client', + 'flashed', + 'template', + + # Data generators + 'test_accounts', + 'test_backends', + 'test_images', + 'test_local_account', + 'test_local_assets' + ] + + +class TemplateInfo: + """ + A class that retains information about a `render_template` call. + """ + + def __init__(self): + self.path = None + self.args = None + + +@pytest.yield_fixture(scope="session") +def app(): + """Return a test application""" + app = create_app('test') + + # Make sure the app is initialized + Init().run() + + def teardown(sender, **kwargs): + # HACK: Check that the teardown is for the last item in the app context + # stack, otherwise don't drop the app yet. + # + # I don't know enough about the internal workings of flask or werkzeug + # Context Locals to be sure this is a safe solution. Any additional + # input/advice on the matter would be welcome. + # + # Anthony Blackshaw + if len(getattr(_app_ctx_stack._local, 'stack', [])) > 1: + return + + # Drop the application + Drop().run() + + with appcontext_tearing_down.connected_to(teardown, app): + yield app + +@pytest.yield_fixture +def celery_app(app): + """Return a test celery application""" + setup_tasks(app.celery) + + yield app.celery + +@pytest.yield_fixture +def client(app): + """Return a test client for the app""" + with app.test_client() as client: + yield client + +@pytest.yield_fixture +def flashed(app): + """Return information about messages flashed""" + with app.test_client() as client: + + flashes = [] + + def on_message_flashed(sender, category, message): + flashes.append((message, category)) + + message_flashed.connect(on_message_flashed) + + yield flashes + +@pytest.yield_fixture +def template(app): + """Return information about templates rendered""" + + with app.test_client() as client: + + template_info = TemplateInfo() + + def on_template_rendered(sender, template, **kwargs): + template_info.path = template.name + template_info.args = kwargs['context'] + + template_rendered.connect(on_template_rendered) + + yield template_info + + +# Data generators + +@pytest.yield_fixture +def test_accounts(app): + """Load test accounts""" + + # Load the test account information + with open('tests/data/accounts.json') as f: + data = json.load(f) + + # Add the test accounts + accounts = [] + for account_data in data: + + with open('tests/data/' + account_data['config_filepath']) as f: + config = json.load(f) + + account = Account( + name=account_data['name'], + backend=config + ) + account.insert() + accounts.append(account) + + yield accounts + + # Purge the accounts + for account in accounts: + account.purge() + +@pytest.yield_fixture +def test_backends(app): + """Create accounts to support each backend""" + + # Add the test accounts for each backend + accounts = [] + for backend in ['local', 's3']: + + with open('tests/data/{backend}.cfg'.format(backend=backend)) as f: + config = json.load(f) + + account = Account(name=backend, backend=config) + account.insert() + accounts.append(account) + + yield accounts + + # Purge the accounts + for account in accounts: + account.purge() + +@pytest.yield_fixture +def test_images(app, client, test_backends): + """Create a test image asset for all backends""" + for account in test_backends: + + # Load the file to upload + with open('tests/data/assets/uploads/image.jpg', 'rb') as f: + file_stream = io.BytesIO(f.read()) + + # Upload the file + response = client.post( + url_for('api.upload'), + data=dict( + api_key=account.api_key, + asset=(file_stream, 'image.jpg') + ) + ) + + assets = Asset.many() + yield assets + + # Purge the assets + for asset in assets: + + # Reload the asset to make sure it still exists before we attempt to + # purge it. + asset = asset.by_id(asset._id) + if asset: + asset.purge() + +@pytest.yield_fixture +def test_local_account(app): + """Create a local account""" + + # Add the local account + with open('tests/data/local.cfg') as f: + config = json.load(f) + + account = Account(name='local', backend=config) + account.insert() + + yield account + + # Purge the account + account.purge() + +@pytest.yield_fixture +def test_local_assets(client, test_local_account): + """Create a set of test assets""" + account = test_local_account + + # Upload all test assets + filepath = 'tests/data/assets/uploads' + for filename in os.listdir(filepath): + + # Load the file to upload + with open(os.path.join(filepath, filename), 'rb') as f: + file_stream = io.BytesIO(f.read()) + + # Upload the file + response = client.post( + url_for('api.upload'), + data=dict( + api_key=account.api_key, + asset=(file_stream, filename), + ) + ) + + # Generate a variation for the `image.jpg` asset + if filename == 'image.jpg': + variations = { + 'test': [ + ['fit', [100, 100]], + ['output', {'format': 'webp', 'quality': 50}] + ] + } + + client.post( + url_for('api.generate_variations'), + data=dict( + api_key=account.api_key, + uid=response.json['payload']['uid'], + variations=json.dumps(variations) + ) + ) + + assets = Asset.many() + yield assets + + # Purge the assets + for asset in assets: + + # Reload the asset to make sure it still exists before we attempt to + # purge it. + asset = asset.by_id(asset._id) + if asset: + asset.purge() \ No newline at end of file diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/api/test_api.py b/tests/api/test_api.py new file mode 100644 index 0000000..e22ce83 --- /dev/null +++ b/tests/api/test_api.py @@ -0,0 +1,437 @@ +from datetime import datetime, timedelta, timezone +from flask import current_app, g, url_for +import io +import json +from mongoframes import * +import time + +from models.accounts import Account +from models.assets import Asset +from tests import * + + +def test_list(client, test_local_account, test_local_assets): + account = test_local_account + + # Get all assets + response = client.get( + url_for('api.list'), + data=dict( + api_key=account.api_key + ) + ) + assert response.json['status'] == 'success' + + # Check the response is correct + payload = response.json['payload'] + assert payload['total_assets'] == 9 + assert payload['total_pages'] == 1 + assert payload['assets'][0]['store_key'].startswith('cdrs') + assert payload['assets'][1]['store_key'].startswith('dribbbleshot') + assert payload['assets'][2]['store_key'].startswith('file') + assert payload['assets'][3]['store_key'].startswith('frameless') + assert payload['assets'][4]['store_key'].startswith('gloop') + assert payload['assets'][5]['store_key'].startswith('image-no-ext') + assert payload['assets'][6]['store_key'].startswith('image') + assert payload['assets'][7]['store_key'].startswith('navicons') + assert payload['assets'][8]['store_key'].startswith('navigation-ui') + + # Get all file assets + response = client.get( + url_for('api.list'), + data=dict( + api_key=account.api_key, + type='file' + ) + ) + assert response.json['status'] == 'success' + + # Check the response is correct + payload = response.json['payload'] + assert payload['total_assets'] == 2 + assert payload['total_pages'] == 1 + assert payload['assets'][0]['store_key'].startswith('file') + assert payload['assets'][1]['store_key'].startswith('frameless') + + # Get all image assets + response = client.get( + url_for('api.list'), + data=dict( + api_key=account.api_key, + type='image' + ) + ) + assert response.json['status'] == 'success' + + # Check the response is correct + payload = response.json['payload'] + assert payload['total_assets'] == 7 + assert payload['total_pages'] == 1 + assert payload['assets'][0]['store_key'].startswith('cdrs') + assert payload['assets'][1]['store_key'].startswith('dribbbleshot') + assert payload['assets'][2]['store_key'].startswith('gloop') + assert payload['assets'][3]['store_key'].startswith('image-no-ext') + assert payload['assets'][4]['store_key'].startswith('image') + assert payload['assets'][5]['store_key'].startswith('navicons') + assert payload['assets'][6]['store_key'].startswith('navigation-ui') + + # Get all jpg assets + response = client.get( + url_for('api.list'), + data=dict( + api_key=account.api_key, + type='image', + q='*.jpg' + ) + ) + assert response.json['status'] == 'success' + + # Check the response is correct + payload = response.json['payload'] + assert payload['total_assets'] == 3 + assert payload['total_pages'] == 1 + assert payload['assets'][0]['store_key'].startswith('gloop') + assert payload['assets'][1]['store_key'].startswith('image') + assert payload['assets'][2]['store_key'].startswith('navigation-ui') + + # Get all assets starting with 'f' + response = client.get( + url_for('api.list'), + data=dict( + api_key=account.api_key, + q='f*' + ) + ) + assert response.json['status'] == 'success' + + # Check the response is correct + payload = response.json['payload'] + assert payload['total_assets'] == 2 + assert payload['total_pages'] == 1 + assert payload['assets'][0]['store_key'].startswith('file') + assert payload['assets'][1]['store_key'].startswith('frameless') + +def test_generate_variations(client, test_backends, test_images): + # Define a set of variations for the image + variations = { + 'test1': [ + ['fit', [200, 200]], + ['crop', [0, 0.5, 0.5, 0]], + ['rotate', 90], + ['output', {'format': 'jpg', 'quality': 50}] + ], + 'test2': [ + ['fit', [100, 100]], + ['rotate', 90], + ['crop', [0, 0.5, 0.5, 0]], + ['rotate', 180], + ['output', {'format': 'webp', 'quality': 50}] + ] + } + + # Test each backend + for account in test_backends: + asset = Asset.one(And(Q.account == account, Q.name == 'image')) + response = client.post( + url_for('api.generate_variations'), + data=dict( + api_key=account.api_key, + uid=asset.uid, + variations=json.dumps(variations), + on_delivery='wait' + ) + ) + assert response.json['status'] == 'success' + + # Check the response is correct + payload = response.json['payload'] + assert len(payload.keys()) == 2 + + # Test variation 1 + assert 'test1' in payload + assert payload['test1']['ext'] == 'jpg' + assert payload['test1']['name'] == 'test1' + key = 'image.{uid}.test1.{version}.jpg'.format( + uid=asset.uid, + version=payload['test1']['version'] + ) + assert payload['test1']['store_key'] == key + assert payload['test1']['meta']['image'] == { + 'mode': 'RGB', + 'size': [200, 150] + } + + # Test variation 2 + assert 'test2' in payload + assert payload['test2']['ext'] == 'webp' + assert payload['test2']['name'] == 'test2' + key = 'image.{uid}.test2.{version}.webp'.format( + uid=asset.uid, + version=payload['test2']['version'] + ) + assert payload['test2']['store_key'] == key + assert payload['test2']['meta']['image'] == { + 'mode': 'RGBA', + 'size': [100, 75] + } + +def test_get(client, test_local_account, test_local_assets): + account = test_local_account + + # Find a file and image asset to get + file_asset = Asset.one(Q.name == 'file') + image_asset = Asset.one(Q.name =='image') + + # Get the details for a file + response = client.get( + url_for('api.get'), + data=dict( + api_key=account.api_key, + uid=file_asset.uid + ) + ) + assert response.json['status'] == 'success' + + # Check that the asset information returned is correct + payload = response.json['payload'] + assert payload.get('created') is not None + assert payload['ext'] == 'zip' + assert payload['meta']['filename'] == 'file.zip' + assert payload.get('modified') is not None + assert payload['name'] == 'file' + assert payload['type'] == 'file' + assert payload.get('uid') is not None + assert payload['store_key'] == 'file.' + payload['uid'] + '.zip' + + # Get the details for an image + response = client.get( + url_for('api.get'), + data=dict( + api_key=account.api_key, + uid=image_asset.uid + ) + ) + assert response.json['status'] == 'success' + + # Check that the asset information returned is correct + payload = response.json['payload'] + + assert payload.get('created') is not None + assert payload['ext'] == 'jpg' + assert payload['meta']['filename'] == 'image.jpg' + assert payload['meta']['image'] == { + 'size': [720, 960], + 'mode': 'RGB' + } + assert payload.get('modified') is not None + assert payload['name'] == 'image' + assert payload['type'] == 'image' + assert payload.get('uid') is not None + assert payload['store_key'] == 'image.' + payload['uid'] + '.jpg' + assert len(payload['variations']) == 1 + + variation = payload['variations'][0] + assert variation['name'] == 'test' + assert variation['ext'] == 'webp' + key = 'image.{uid}.test.{version}.webp'.format( + uid=payload['uid'], + version=variation['version'] + ) + assert variation['store_key'] == key + assert variation.get('version') is not None + assert variation['meta']['image'] == { + 'size': [75, 100], + 'mode': 'RGBA' + } + +def test_download(client, test_local_account, test_local_assets): + account = test_local_account + + # Find an asset to download + file_asset = Asset.one(Q.name == 'file') + + # Download the file + response = client.get( + url_for('api.download'), + data=dict( + api_key=account.api_key, + uid=file_asset.uid + ) + ) + assert response.content_type == 'application/zip' + content_disposition = 'attachment; filename=' + file_asset.store_key + assert response.headers['Content-Disposition'] == content_disposition + assert len(response.data) == file_asset.meta['length'] + +def test_set_expires(client, test_local_account): + account = test_local_account + + # Load a file to upload + with open('tests/data/assets/uploads/file.zip', 'rb') as f: + file_stream = io.BytesIO(f.read()) + + # Create an asset + response = client.post( + url_for('api.upload'), + data=dict( + api_key=account.api_key, + asset=(file_stream, 'file.zip'), + name='files/test' + ) + ) + + # Get the asset we uploaded + asset = Asset.one(And( + Q.account == account, + Q.uid == response.json['payload']['uid'] + )) + + # Set an expiry date 1 hour from now + expires = datetime.now(timezone.utc) + timedelta(seconds=3600) + expires = time.mktime(expires.timetuple()) + + response = client.post( + url_for('api.set_expires'), + data=dict( + api_key=account.api_key, + uid=asset.uid, + expires=str(expires) + ) + ) + assert response.json['status'] == 'success' + + # Reload the asset and check the expires has been correctly set + asset.reload() + assert asset.expires == expires + + # Unset the expiry date + response = client.post( + url_for('api.set_expires'), + data=dict( + api_key=account.api_key, + uid=asset.uid + ) + ) + assert response.json['status'] == 'success' + + # Reload the asset and check the expires has been correctly set + asset.reload() + assert asset.expires == None + +def test_upload_file(client, test_backends): + + # Test each backend + for account in test_backends: + + # Load a file to upload + with open('tests/data/assets/uploads/file.zip', 'rb') as f: + file_stream = io.BytesIO(f.read()) + + # Set an expiry date 1 hour from now + expires = datetime.now(timezone.utc) + timedelta(seconds=3600) + expires = time.mktime(expires.timetuple()) + + # Upload a file (non-image) asset + response = client.post( + url_for('api.upload'), + data=dict( + api_key=account.api_key, + asset=(file_stream, 'file.zip'), + name='files/test', + expires=str(expires) + ) + ) + assert response.json['status'] == 'success' + + # Validate the payload + payload = response.json['payload'] + assert payload.get('created') is not None + assert payload['expires'] == expires + assert payload['ext'] == 'zip' + assert payload['meta']['filename'] == 'file.zip' + assert payload.get('modified') is not None + assert payload['name'] == 'files/test' + assert payload['type'] == 'file' + assert payload.get('uid') is not None + assert payload['store_key'] == 'files/test.' + payload['uid'] + '.zip' + +def test_upload_image(client, test_backends): + + # Test each backend + for account in test_backends: + + # Load an image to upload + with open('tests/data/assets/uploads/image.jpg', 'rb') as f: + image_stream = io.BytesIO(f.read()) + + # Set an expiry date 1 hour from now + expires = datetime.now(timezone.utc) + timedelta(seconds=3600) + expires = time.mktime(expires.timetuple()) + + # Upload an image asset + response = client.post( + url_for('api.upload'), + data=dict( + api_key=account.api_key, + asset=(image_stream, 'image.jpg'), + name='images/test', + expires=str(expires) + ) + ) + assert response.json['status'] == 'success' + + # Validate the payload + payload = response.json['payload'] + assert payload.get('created') is not None + assert payload['expires'] == expires + assert payload['ext'] == 'jpg' + assert payload['meta']['filename'] == 'image.jpg' + assert payload['meta']['image'] == { + 'size': [720, 960], + 'mode': 'RGB' + } + assert payload.get('modified') is not None + assert payload['name'] == 'images/test' + assert payload['type'] == 'image' + assert payload.get('uid') is not None + assert payload['store_key'] == 'images/test.' + payload['uid'] + '.jpg' + +def test_upload_image_without_ext(client, test_local_account): + + account = test_local_account + + # Load an image to upload + with open('tests/data/assets/uploads/image_no_ext', 'rb') as f: + image_stream = io.BytesIO(f.read()) + + # Set an expiry date 1 hour from now + expires = datetime.now(timezone.utc) + timedelta(seconds=3600) + expires = time.mktime(expires.timetuple()) + + # Upload an image asset without an extension + response = client.post( + url_for('api.upload'), + data=dict( + api_key=account.api_key, + asset=(image_stream, 'image_no_ext'), + name='images/test', + expires=str(expires) + ) + ) + assert response.json['status'] == 'success' + + # Validate the payload + payload = response.json['payload'] + assert payload.get('created') is not None + assert payload['expires'] == expires + assert payload['ext'] == 'png' + assert payload['meta']['filename'] == 'image_no_ext' + assert payload['meta']['image'] == { + 'size': [800, 600], + 'mode': 'RGB' + } + assert payload.get('modified') is not None + assert payload['name'] == 'images/test' + assert payload['type'] == 'image' + assert payload.get('uid') is not None + assert payload['store_key'] == 'images/test.' + payload['uid'] + '.png' diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/commands/test_accounts.py b/tests/commands/test_accounts.py new file mode 100644 index 0000000..88c1150 --- /dev/null +++ b/tests/commands/test_accounts.py @@ -0,0 +1,172 @@ +import builtins +import json + +from blessings import Terminal +from flask import current_app +import mock +from mongoframes import * +from pytest import * + +from commands.accounts import * +from models.accounts import Account +from tests import * + + +def mock_input(responses): + """Return a function that will mock user input""" + + response_index = {'index': 0} + + def _input(question): + response = responses[response_index['index']] + response_index['index'] += 1 + return response + + return _input + +def test_add_account(capsys, app): + # Add a new account + responses = ['tests/data/assets'] + with mock.patch.object(builtins, 'input', mock_input(responses)): + AddAccount().run('test', 'local') + + # Check the output is as expected + account = Account.one() + assert 'Account added: {0}'.format(account.api_key) == \ + capsys.readouterr()[0].strip().split('\n')[-1] + + # Check the account was created correctly + assert account.name == 'test' + assert account.backend == json.load(open('tests/data/local.cfg')) + assert account.api_key + +def test_config_account(capsys, app, test_accounts): + # Configure an account + responses = [ + 'AKIAIW5ILOAT5ZJ5XWJQ', + 'y80Io/ukJhZxaiHd4ngEVxIC7v96D+z+tJOFOoY2', + 'hangar51test' + ] + with mock.patch.object(builtins, 'input', mock_input(responses)): + ConfigAccount().run('hangar51') + + # Check the output is as expected + assert 'Account configured' == \ + capsys.readouterr()[0].strip().split('\n')[-1] + + # Check the account was configured correctly + account = Account.one(Q.name == 'hangar51') + assert account.backend['access_key'] == 'AKIAIW5ILOAT5ZJ5XWJQ' + assert account.backend['secret_key'] == \ + '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' + assert account.backend['bucket'] == 'hangar51test' + +def test_delete_account(capsys, app, test_accounts): + # Delete an account + DeleteAccount().run('getme') + + # Check the output is as expected + assert 'Account deleted' == capsys.readouterr()[0].strip() + + # Check there is no longer an account for 'getme' + getme = Account.one(Q.name == 'getme') + assert getme is None + +def test_generate_new_api_key(capsys, app, test_accounts): + # Find an existing account change the API key for + old_api_key = Account.one(Q.name == 'getme').api_key + + # Generate a new API key for an account + GenerateNewAPIKey().run('getme') + + # Check a new API key has been generated + new_api_key = Account.one(Q.name == 'getme').api_key + assert new_api_key + assert new_api_key != old_api_key + + # Check the output is as expected + assert 'New key generated: {0}'.format(new_api_key) \ + == capsys.readouterr()[0].strip() + +def test_list_accounts(capsys, app, test_accounts): + # Get a list of *all* accounts + ListAccounts().run() + + # Check output is as expected + expected_out = [ + 'Accounts (9):', + '- burst (using local)', + '- deploycms (using local)', + '- geocode (using local)', + '- getcontenttools (using local)', + '- getme (using local)', + '- glitch (using s3)', + '- hangar51 (using s3)', + '- lupin (using s3)', + '- mongoframes (using s3)' + ] + expected_out = '\n'.join(expected_out) + out = capsys.readouterr()[0].strip() + assert out == expected_out + + # Get a list of accounts containing the string 'ge' + ListAccounts().run('ge') + expected_out = [ + "Accounts matching 'ge' (3):", + '- geocode (using local)', + '- getcontenttools (using local)', + '- getme (using local)' + ] + expected_out = '\n'.join(expected_out) + out = capsys.readouterr()[0].strip() + assert out == expected_out + +def test_list_backends(capsys, app): + # Get a list of supported backends + ListBackends().run() + + # Check the output is as expected + expected_out = [ + 'Backends (2):', + '- local', + '- s3' + ] + expected_out = '\n'.join(expected_out) + out = capsys.readouterr()[0].strip() + + assert out == expected_out + +def test_rename_account(capsys, app, test_accounts): + getme = Account.one(Q.name == 'getme').api_key + + # Generate a new API key for an account + RenameAccount().run('getme', 'new_getme') + + # Check the account has been renamed + new_getme = Account.one(Q.name == 'new_getme').api_key + assert new_getme == getme + + # Check the output is as expected + assert 'Account renamed: new_getme' == capsys.readouterr()[0].strip() + +def test_view_account(capsys, app, test_accounts): + # View the details for an account + ViewAccount().run('getme') + + # Find the account in question as some details are generate when the account + # is created. + getme = Account.one(Q.name == 'getme') + + # Check output is as expected + expected_out = [ + "About 'getme':", + '- created------- ' + str(getme.created), + '- modified------ ' + str(getme.modified), + '- assets-------- 0', + '- api_key------- ' + getme.api_key, + '- backend------- local', + '- > asset_root-- tests/data/assets' + ] + expected_out = '\n'.join(expected_out) + out = capsys.readouterr()[0].strip() + assert out == expected_out \ No newline at end of file diff --git a/tests/commands/test_app.py b/tests/commands/test_app.py new file mode 100644 index 0000000..1292e07 --- /dev/null +++ b/tests/commands/test_app.py @@ -0,0 +1,24 @@ +from flask import current_app + +from commands import Drop +from tests import * + + +def test_init(app): + # The application is initialized as part of the test set up + + # Check the correct list of collections has been initialized + expected_collection = { + 'Account', + 'Asset' + } + assert set(current_app.db.collection_names(False)) == expected_collection + + +def test_drop(app): + + # Drop the application + Drop().run() + + # Check all collections have been dropped + assert set(current_app.db.collection_names(False)) == set() \ No newline at end of file diff --git a/tests/data/accounts.json b/tests/data/accounts.json new file mode 100644 index 0000000..3c0560b --- /dev/null +++ b/tests/data/accounts.json @@ -0,0 +1,11 @@ +[ + {"name": "burst", "config_filepath": "local.cfg"}, + {"name": "deploycms", "config_filepath": "local.cfg"}, + {"name": "geocode", "config_filepath": "local.cfg"}, + {"name": "getcontenttools", "config_filepath": "local.cfg"}, + {"name": "getme", "config_filepath": "local.cfg"}, + {"name": "glitch", "config_filepath": "s3.cfg"}, + {"name": "hangar51", "config_filepath": "invalid_s3.cfg"}, + {"name": "lupin", "config_filepath": "s3.cfg"}, + {"name": "mongoframes", "config_filepath": "s3.cfg"} +] \ No newline at end of file diff --git a/tests/data/invalid_s3.cfg b/tests/data/invalid_s3.cfg new file mode 100644 index 0000000..8991481 --- /dev/null +++ b/tests/data/invalid_s3.cfg @@ -0,0 +1,6 @@ +{ + "backend": "s3", + "access_key": "AKIAIW5ILOAT5ZJ5XWJQ", + "secret_key": "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", + "bucket": "hangar51test" +} \ No newline at end of file diff --git a/tests/data/local.cfg b/tests/data/local.cfg new file mode 100644 index 0000000..2af7f68 --- /dev/null +++ b/tests/data/local.cfg @@ -0,0 +1,4 @@ +{ + "backend": "local", + "asset_root": "tests/data/assets" +} \ No newline at end of file diff --git a/tests/test_tasks.py b/tests/test_tasks.py new file mode 100644 index 0000000..7288d7f --- /dev/null +++ b/tests/test_tasks.py @@ -0,0 +1,58 @@ +from datetime import datetime, timedelta, timezone +from mongoframes import * +import time + +from models.accounts import Account +from models.assets import Asset +from tests import * + + +def test_generate_varations(celery_app, test_images): + asset = test_images[0] + + # Define the variation to generate + variations = { + 'test': [ + ['fit', [200, 200]], + ['crop', [0, 0.5, 0.5, 0]], + ['rotate', 90], + ['output', {'format': 'jpg', 'quality': 50}] + ] + } + + # Call the `generate_variation` task + task = celery_app.tasks['generate_variations'] + task.apply([asset.account, asset.uid, variations]) + + # Check the variation was generated + asset.reload() + assert len(asset.variations) == 1 + + variation = asset.variations[0] + assert variation['ext'] == 'jpg' + assert variation['name'] == 'test' + key = 'image.{uid}.test.{version}.jpg'.format( + uid=asset.uid, + version=variation['version'] + ) + assert variation['store_key'] == key + assert variation['meta']['image'] == { + 'mode': 'RGB', + 'size': [200, 150] + } + +def test_purge_expired_assets(celery_app, test_images): + # Set the expiry date for all assets to an hour ago + expires = datetime.now(timezone.utc) - timedelta(seconds=3600) + expires = time.mktime(expires.timetuple()) + assets = Asset.many() + for asset in assets: + asset.expires = expires + asset.update('modified', 'expires') + + # Call the `purge_expired_assets` task + task = celery_app.tasks['purge_expired_assets'] + task.apply() + + # Check all the assets where purged + assert Asset.count() == 0 diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..c9b2dfc --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1,24 @@ +""" +Useful functions used across more than one module. +""" + +import os +import shortuuid + +__all__ = [ + 'get_file_length', + 'generate_uid' + ] + + +def get_file_length(f): + """Return the length of a file storage object""" + f.seek(0, os.SEEK_END) + length = f.tell() + f.seek(0) + return length + +def generate_uid(length): + """Generate a uid of a given length""" + su = shortuuid.ShortUUID(alphabet='abcdefghijklmnopqrstuvwxyz0123456789') + return su.uuid()[:length] \ No newline at end of file diff --git a/utils/forms.py b/utils/forms.py new file mode 100644 index 0000000..e90e92a --- /dev/null +++ b/utils/forms.py @@ -0,0 +1,52 @@ +""" +Utils for common form operations. +""" + +__all__ = ['FormData'] + + +class FormData: + """ + A wrapper class that converts a dictionary into a request like object that + can be used as the `formdata` argument when initializing a `WTForm` + instance, for example: + + ``` + form = MyWTForm(FormData({...})) + ``` + """ + + def __init__(self, data): + self._data = {} + for key, value in data.items(): + + # Fields named `session_token` are not allowed in form data + if key == 'session_token': + continue + + if key not in self._data: + self._data[key] = [] + + if isinstance(value, list): + self._data[key] += value + else: + self._data[key].append(value) + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) + + def __contains__(self, name): + return (name in self._data) + + # Methods + + def get(self, key, default=None): + if key in self._data: + return self._data[key][0] + return default + + def getlist(self, key): + return self._data.get(key, []) \ No newline at end of file