# HG changeset patch # User Gustavo Andres Morero # Date 1525895877 10800 # Wed May 09 16:57:57 2018 -0300 # Node ID 8ef069d858587f7f93fab42ac7ddba72969b6c68 # Parent 79f5fe0a496a550e81cc13c44cf1c1fa18ae7c57 updating aws s3proc to use boto3. diff --git a/webutils/aws/s3proc.py b/webutils/aws/s3proc.py --- a/webutils/aws/s3proc.py +++ b/webutils/aws/s3proc.py @@ -2,8 +2,17 @@ Requires Boto AWS library! http://code.google.com/p/boto/ ''' -import boto -from boto.s3.key import Key +import boto3 +import logging +import botocore +import mimetypes + +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO + +log = logging.getLogger('webutils.aws.s3proc') class S3Error(Exception): @@ -44,31 +53,32 @@ return perm def connect(self): - self.conn = boto.connect_s3(self.aws_key, self.aws_secret_key) + boto3.setup_default_session( + aws_access_key_id=self.aws_key, + aws_secret_access_key=self.aws_secret_key, + ) + self.conn = boto3.resource('s3') + self.bucket = self.conn.Bucket(self.bucket_name) + self.client = self.conn.meta.client try: - self.bucket = self.conn.get_bucket( - self.bucket_name, - validate=self.bucket_validate, - ) - except boto.exception.S3ResponseError as e: - if e.status == 404: - try: - self.bucket = self.conn.create_bucket(self.bucket_name) - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) - except boto.exception.S3CreateError: - raise S3Error( - 'Unable to create bucket %s' % self.bucket_name - ) + self.client.head_bucket(Bucket=self.bucket_name) + except botocore.exceptions.ClientError as e: + error_code = int(e.response['Error']['Code']) + if error_code == 404: + self.conn.create_bucket(Bucket=self.bucket_name) return True @checkbucket def put(self, filename, data, perm=None, fail_silently=True): perm = self._get_perm(perm) - obj = Key(self.bucket) - obj.key = filename - obj.set_contents_from_string(data) - obj.set_acl(perm) + kwargs = {'Body': data, 'ACL': perm} + try: + ctype = mimetypes.guess_type(filename)[0] + if ctype: + kwargs['ContentType'] = ctype + except IndexError: + pass + self.bucket.Object(filename).put(**kwargs) return True @checkbucket @@ -77,34 +87,47 @@ ''' Upload file from disk. If you want a different remote filename, specify in remote_filename ''' - perm = self._get_perm(perm) - obj = Key(self.bucket) - obj.key = remote_filename or filename - obj.set_contents_from_filename(filename) - obj.set_acl(perm) - return True + try: + fp = open(filename, 'rb') + r_path = remote_filename if remote_filename else filename + self.put(r_path, fp, perm) + fp.close() + return True + except Exception as e: + log.error('Failed file upload: {0}'.format(str(e))) + pass + return False + + @checkbucket + def get_file_key(self, filename): + ''' Returns a Key object. Use key['Body'].read() + ''' + try: + return self.bucket.Object(filename).get() + except botocore.exceptions.ClientError as error: + raise S3Error('Error: {0}: {1}'.format( + error.response['ResponseMetadata']['HTTPStatusCode'], + error.response['Error']['Message'])) @checkbucket def get(self, filename): ''' Returns file data ''' - obj = Key(self.bucket) - obj.key = filename - try: - return obj.get_contents_as_string() - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) + key = self.get_file_key(filename) + _file = StringIO(key['Body'].read()) + return _file @checkbucket def get_to_file(self, filename, local_filename=None): ''' Save file to local filename. ''' - obj = Key(self.bucket) - obj.key = filename try: - obj.get_contents_to_filename(local_filename or filename) - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) + self.bucket.Object(filename).download_file( + local_filename if local_filename else filename) + except botocore.exceptions.ClientError as error: + raise S3Error('Error: {0}: {1}'.format( + error.response['ResponseMetadata']['HTTPStatusCode'], + error.response['Error']['Message'])) return True @checkbucket @@ -112,48 +135,64 @@ ''' Returns true/false on delete Raises S3Error if fail_silently is set to False ''' - self.bucket.delete_key(filename) + self.bucket.Object(filename).delete() return True @checkbucket def get_perm(self, filename): ''' Returns permissions for set file. ''' - obj = Key(self.bucket) - obj.key = filename try: - return obj.get_acl() - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) + return self.conn.ObjectAcl(self.bucket_name, filename) + except botocore.exceptions.ClientError as error: + raise S3Error('Error: {0}: {1}'.format( + error.response['ResponseMetadata']['HTTPStatusCode'], + error.response['Error']['Message'])) @checkbucket def set_perm(self, filename, perm, fail_silently=True): ''' Sets permissions on file. ''' perm = self._get_perm(perm) - obj = Key(self.bucket) - obj.key = filename try: - obj.set_acl(perm) - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) + object_acl = self.conn.ObjectAcl(self.bucket_name, filename) + object_acl.put(ACL=perm) + except botocore.exceptions.ClientError as error: + raise S3Error('Error: {0}: {1}'.format( + error.response['ResponseMetadata']['HTTPStatusCode'], + error.response['Error']['Message'])) return True @checkbucket def set_metadata(self, filename, meta_key, meta_value): - obj = Key(self.bucket) - obj.key = filename try: - obj.set_metadata(meta_key, meta_value) - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) + k = self.client.head_object(Bucket=self.bucket_name, Key=filename) + m = k["Metadata"] + m[meta_key] = meta_value + copy_source = '{0}/{1}'.format(self.bucket_name, filename) + self.client.copy_object( + Bucket=self.bucket_name, + Key=filename, + CopySource=copy_source, + Metadata=m, + MetadataDirective='REPLACE', + ) + except botocore.exceptions.ClientError as error: + raise S3Error('Error: {0}: {1}'.format( + error.response['ResponseMetadata']['HTTPStatusCode'], + error.response['Error']['Message'])) return True @checkbucket def get_metadata(self, filename, meta_key): - obj = Key(self.bucket) - obj.key = filename try: - return obj.get_metadata(meta_key) - except boto.exception.S3ResponseError as err: - raise S3Error('Error: %i: %s' % (err.status, err.reason)) + k = self.client.head_object(Bucket=self.bucket_name, Key=filename) + except botocore.exceptions.ClientError as error: + raise S3Error('Error: {0}: {1}'.format( + error.response['ResponseMetadata']['HTTPStatusCode'], + error.response['Error']['Message'])) + try: + m = k["Metadata"] + return m[meta_key] + except KeyError: + raise S3Error('Invalid meta key')