Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add CachedStaticSwiftStorage and ManifestStaticSwiftStorage #94

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add CachedStaticSwiftStorage and ManifestStaticSwiftStorage #94

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions swift/storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import mimetypes
import os
import posixpath
import re
from datetime import datetime
from functools import wraps
from io import BytesIO
from time import time
import magic
from django.conf import settings
from django.contrib.staticfiles.storage import CachedFilesMixin, ManifestFilesMixin
from django.core.exceptions import ImproperlyConfigured
from django.core.files import File
from django.core.files.storage import Storage
Expand All @@ -29,6 +31,17 @@ def setting(name, default=None):
return getattr(settings, name, default)


def safe_normpath(path):
"""
Avoid doing normpath on empty path since:
- posixpath.normpath('') -> '.'
- we should avoid relative paths with swift
"""
if path:
return posixpath.normpath(path)
return path


def validate_settings(backend):
# Check mandatory parameters
if not backend.api_auth_url:
Expand Down Expand Up @@ -108,7 +121,7 @@ def prepend_name_prefix(func):
"""
@wraps(func)
def prepend_prefix(self, name, *args, **kwargs):
name = self.name_prefix + name
name = self.name_prefix + safe_normpath(name)
return func(self, name, *args, **kwargs)
return prepend_prefix

Expand Down Expand Up @@ -241,7 +254,7 @@ def set_token(self, new_token):

def _open(self, name, mode='rb'):
original_name = name
name = self.name_prefix + name
name = self.name_prefix + safe_normpath(name)

headers, content = swiftclient.get_object(self.storage_url,
self.token,
Expand All @@ -255,7 +268,7 @@ def _open(self, name, mode='rb'):

def _save(self, name, content, headers=None):
original_name = name
name = self.name_prefix + name
name = self.name_prefix + safe_normpath(name)

if self.content_type_from_fd:
content_type = magic.from_buffer(content.read(1024), mime=True)
Expand Down Expand Up @@ -426,3 +439,23 @@ def get_available_name(self, name, max_length=None):
overwrite it.
"""
return name


class CachedStaticSwiftStorage(CachedFilesMixin, StaticSwiftStorage):
"""
A static file system storage backend which also saves
hashed copies of the files it saves.
"""
pass


class ManifestStaticSwiftStorage(ManifestFilesMixin, StaticSwiftStorage):
"""
A static file system storage backend which also saves
hashed copies of the files it saves.
"""
def read_manifest(self):
try:
super(ManifestStaticSwiftStorage, self).read_manifest()
except swiftclient.ClientException:
return None
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of silently swallowing the exception I would suggest to wrap it in IOError instead, like this (and maybe in the StaticSwiftStorage._open method instead of ManifestStaticSwiftStorage.read_manifest ):

except swiftclient.exceptions.ClientException as e:
    # django.contrib.staticfiles.storage.ManifestFilesMixin#read_manifest
    # When manifest file doesn't exist, the raised error must be an instance of IOError.
    raise IOError(e)

If you want to ignore the missing manifest error, it's possible by setting ManifestStaticSwiftStorage.manifest_strict property to False. So something like this will allow to make the ignoring of the error configurable:

class ManifestStaticSwiftStorage(ManifestFilesMixin, StaticSwiftStorage):
    manifest_strict = setting('SWIFT_MANIFEST_STRICT', True)