"""
Incorporated from django-storages, copyright all of those listed in:
http://code.welldev.org/django-storages/src/tip/AUTHORS
"""
import urllib
import os
import mimetypes
import re
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
from django.conf import settings
from django.core.files.base import File
from django.core.files.storage import Storage
from django.core.exceptions import ImproperlyConfigured
try:
from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from boto.s3.key import Key
except ImportError:
raise ImproperlyConfigured(
"Could not load boto's S3 bindings. Please install boto."
)
AWS_REGIONS = [
'eu-west-1',
'us-east-1',
'us-west-1',
'us-west-2',
'sa-east-1',
'ap-northeast-1',
'ap-southeast-1',
]
REGION_RE = re.compile(r's3-(.+).amazonaws.com')
ACCESS_KEY_NAME = getattr(settings, 'AWS_ACCESS_KEY_ID', None)
SECRET_KEY_NAME = getattr(settings, 'AWS_SECRET_ACCESS_KEY', None)
HEADERS = getattr(settings, 'AWS_HEADERS', {})
STORAGE_BUCKET_NAME = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', None)
STORAGE_BUCKET_CNAME = getattr(settings, 'AWS_STORAGE_BUCKET_CNAME', None)
AWS_REGION = getattr(settings, 'AWS_REGION', 'us-east-1')
AUTO_CREATE_BUCKET = getattr(settings, 'AWS_AUTO_CREATE_BUCKET', True)
DEFAULT_ACL = getattr(settings, 'AWS_DEFAULT_ACL', 'public-read')
QUERYSTRING_AUTH = getattr(settings, 'AWS_QUERYSTRING_AUTH', True)
QUERYSTRING_EXPIRE = getattr(settings, 'AWS_QUERYSTRING_EXPIRE', 3600)
IS_GZIPPED = getattr(settings, 'AWS_IS_GZIPPED', False)
GZIP_CONTENT_TYPES = getattr(settings, 'GZIP_CONTENT_TYPES', (
'text/css',
'application/javascript',
'application/x-javascript'
))
if IS_GZIPPED:
from gzip import GzipFile
class S3BotoStorage(Storage):
"""Amazon Simple Storage Service using Boto"""
def __init__(self, bucket=STORAGE_BUCKET_NAME,
bucket_cname=STORAGE_BUCKET_CNAME,
region=AWS_REGION, access_key=None,
secret_key=None, acl=DEFAULT_ACL,
headers=HEADERS, gzip=IS_GZIPPED,
gzip_content_types=GZIP_CONTENT_TYPES,
querystring_auth=QUERYSTRING_AUTH,
force_no_ssl=False):
self.bucket_name = bucket
self.bucket_cname = bucket_cname
self.host = self._get_host(region)
self.acl = acl
self.headers = headers
self.gzip = gzip
self.gzip_content_types = gzip_content_types
self.querystring_auth = querystring_auth
self.force_no_ssl = force_no_ssl
# This is called as chunks are uploaded to S3. Useful for getting
# around limitations in eventlet for things like gunicorn.
self.s3_callback_during_upload = None
if not access_key and not secret_key:
access_key, secret_key = self._get_access_keys()
self.connection = S3Connection(
access_key, secret_key, host=self.host,
)
@property
def bucket(self):
if not hasattr(self, '_bucket'):
self._bucket = self._get_or_create_bucket(self.bucket_name)
return self._bucket
def _get_access_keys(self):
access_key = ACCESS_KEY_NAME
secret_key = SECRET_KEY_NAME
if (access_key or secret_key) and (not access_key or not secret_key):
access_key = os.environ.get(ACCESS_KEY_NAME)
secret_key = os.environ.get(SECRET_KEY_NAME)
if access_key and secret_key:
# Both were provided, so use them
return access_key, secret_key
return None, None
def _get_host(self, region):
"""
Returns correctly formatted host. Accepted formats:
* simple region name, eg 'us-west-1' (see list in AWS_REGIONS)
* full host name, eg 's3-us-west-1.amazonaws.com'.
"""
if 'us-east-1' in region:
return 's3.amazonaws.com'
elif region in AWS_REGIONS:
return 's3-%s.amazonaws.com' % region
elif region and not REGION_RE.findall(region):
raise ImproperlyConfigured('AWS_REGION improperly configured!')
# can be full host or empty string, default region
return region
def _get_or_create_bucket(self, name):
"""Retrieves a bucket if it exists, otherwise creates it."""
try:
return self.connection.get_bucket(name)
except S3ResponseError, e:
if AUTO_CREATE_BUCKET:
return self.connection.create_bucket(name)
raise ImproperlyConfigured, ("Bucket specified by "
"AWS_STORAGE_BUCKET_NAME does not exist. Buckets can be "
"automatically created by setting AWS_AUTO_CREATE_BUCKET=True")
def _clean_name(self, name):
# Useful for windows' paths
return os.path.normpath(name).replace('\\', '/')
def _compress_content(self, content):
"""Gzip a given string."""
zbuf = StringIO()
zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
zfile.write(content.read())
zfile.close()
content.file = zbuf
return content
def _open(self, name, mode='rb'):
name = self._clean_name(name)
return S3BotoStorageFile(name, mode, self)
def _save(self, name, content):
name = self._clean_name(name)
if callable(self.headers):
headers = self.headers(name, content)
else:
headers = self.headers
if hasattr(content.file, 'content_type'):
content_type = content.file.content_type
else:
content_type = mimetypes.guess_type(name)[0] or "application/x-octet-stream"
if self.gzip and content_type in self.gzip_content_types:
content = self._compress_content(content)
headers.update({'Content-Encoding': 'gzip'})
headers.update({
'Content-Type': content_type,
'Content-Length' : len(content),
})
content.name = name
k = self.bucket.get_key(name)
if not k:
k = self.bucket.new_key(name)
# The callback seen here is particularly important for async WSGI
# servers. This allows us to call back to eventlet or whatever
# async support library we're using periodically to prevent timeouts.
k.set_contents_from_file(content, headers=headers, policy=self.acl,
cb=self.s3_callback_during_upload,
num_cb=-1)
return name
def delete(self, name):
name = self._clean_name(name)
self.bucket.delete_key(name)
def exists(self, name):
name = self._clean_name(name)
k = Key(self.bucket, name)
return k.exists()
def listdir(self, name):
name = self._clean_name(name)
return [l.name for l in self.bucket.list() if not len(name) or l.name[:len(name)] == name]
def size(self, name):
name = self._clean_name(name)
return self.bucket.get_key(name).size
def url(self, name):
name = self._clean_name(name)
if self.bucket.get_key(name) is None:
return ''
return self.bucket.get_key(name).generate_url(QUERYSTRING_EXPIRE,
method='GET',
query_auth=self.querystring_auth,
force_http=self.force_no_ssl)
def url_as_attachment(self, name, filename=None):
name = self._clean_name(name)
if filename:
disposition = 'attachment; filename="%s"' % filename
else:
disposition = 'attachment;'
response_headers = {
'response-content-disposition': disposition,
}
return self.connection.generate_url(QUERYSTRING_EXPIRE, 'GET',
bucket=self.bucket.name, key=name,
query_auth=True,
force_http=self.force_no_ssl,
response_headers=response_headers)
def get_available_name(self, name):
""" Overwrite existing file with the same name. """
name = self._clean_name(name)
return name
class S3BotoStorage_AllPublic(S3BotoStorage):
"""
Same as S3BotoStorage, but defaults to uploading everything with a
public acl. This has two primary beenfits:
1) Non-encrypted requests just make a lot better sense for certain things
like profile images. Much faster, no need to generate S3 auth keys.
2) Since we don't have to hit S3 for auth keys, this backend is much
faster than S3BotoStorage, as it makes no attempt to validate whether
keys exist.
WARNING: This backend makes absolutely no attempt to verify whether the
given key exists on self.url(). This is much faster, but be aware.
"""
def __init__(self, *args, **kwargs):
super(S3BotoStorage_AllPublic, self).__init__(acl='public-read',
querystring_auth=False,
force_no_ssl=True,
*args, **kwargs)
def url(self, name):
"""
Since we assume all public storage with no authorization keys, we can
just simply dump out a URL rather than having to query S3 for new keys.
"""
name = urllib.quote_plus(self._clean_name(name), safe='/')
if self.bucket_cname:
return "http://%s/%s" % (self.bucket_cname, name)
elif self.host:
return "http://%s/%s/%s" % (self.host, self.bucket_name, name)
# No host ? Then it's the default region
return "http://s3.amazonaws.com/%s/%s" % (self.bucket_name, name)
class S3BotoStorageFile(File):
def __init__(self, name, mode, storage):
self._storage = storage
self.name = name
self._mode = mode
self.key = storage.bucket.get_key(name)
self._is_dirty = False
self.file = StringIO()
@property
def size(self):
if not self.key:
raise IOError('No such S3 key: %s' % self.name)
return self.key.size
def read(self, *args, **kwargs):
self.file = StringIO()
self._is_dirty = False
if not self.key:
raise IOError('No such S3 key: %s' % self.name)
self.key.get_contents_to_file(self.file)
return self.file.getvalue()
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was opened for read-only access.")
self.file = StringIO(content)
self._is_dirty = True
def close(self):
if self._is_dirty:
if not self.key:
self.key = self._storage.bucket.new_key(key_name=self.name)
self.key.set_contents_from_string(self.file.getvalue(), headers=self._storage.headers, policy=self.storage.acl)
self.key.close()