Merge pull request #258 from harshavardhana/pr_out_add_new_presignurl…

…_api

    Add new presignURL api, tests and examples, overall cleanup

examples/presigned_get_object.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import hashlib
+
+from minio import Minio
+
+__author__ = 'minio'
+
+# find out your s3 end point here:
+# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+
+client = Minio('https://<your-s3-endpoint>',
+ access_key='YOUR-ACCESSKEYID',
+ secret_key='YOUR-SECRETACCESSKEY')
+
+print client.presigned_get_object('mybucket', 'myobject')

minio/minio.py

@@ -37,7 +37,7 @@
  parse_new_multipart_upload)
 from .error import ResponseError
 from .definitions import Object
-from .signer import sign_v4
+from .signer import sign_v4, presign_v4
 from .xml_requests import bucket_constraint, get_complete_multipart_upload
 
 class Minio(object):
@@ -329,6 +329,40 @@ def drop_all_incomplete_uploads(self, bucket):
  for upload in uploads:
  self._drop_incomplete_upload(bucket, upload.key, upload.upload_id)
 
+ def presigned_get_object(self, bucket, key, expires=None):
+ """
+ Presigns a get object request and provides a url
+ """
+ return self.presigned_get_partial_object(bucket, key, expires)
+
+ def presigned_get_partial_object(self, bucket, key, expires=None, offset=0, length=0):
+ """
+ """
+ is_valid_bucket_name(bucket)
+ is_non_empty_string(key)
+
+ request_range = ''
+ if offset is not 0 and length is not 0:
+ request_range = str(offset) + "-" + str(offset + length - 1)
+ if offset is not 0 and length is 0:
+ request_range = str(offset) + "-"
+ if offset is 0 and length is not 0:
+ request_range = "0-" + str(length - 1)
+
+ method = 'GET'
+ url = get_target_url(self._endpoint_url, bucket=bucket, key=key)
+ headers = {}
+
+ if request_range:
+ headers['Range'] = 'bytes=' + request_range
+
+ method = 'GET'
+ presign_url = presign_v4(method=method, url=url, headers=headers,
+ access_key=self._access_key,
+ secret_key=self._secret_key)
+
+ return presign_url
+
  def get_object(self, bucket, key):
  """
  Retrieves an object from a bucket.

minio/signer.py

@@ -13,14 +13,106 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import hashlib
 import hmac
 import binascii
 
 from datetime import datetime
-from .compat import urlsplit, strtype
+from .error import InvalidArgumentError
+from .compat import urlsplit, strtype, urlencode
 from .helpers import get_region
 
+def presign_v4(method, url, headers=None, access_key=None, secret_key=None, expires=None):
+ if not access_key or not secret_key:
+ raise InvalidArgumentError('invalid access/secret id')
+
+ # verify only if 'None' not on expires with 0 value which should
+ # be an InvalidArgument is handled later below
+ if expires is None:
+ expires = 604800
+
+ if expires < 1 or expires > 604800:
+ raise InvalidArgumentError('expires param valid values are between 1 secs to 604800 secs')
+
+ if headers is None:
+ headers = {}
+
+ parsed_url = urlsplit(url)
+ content_hash_hex = 'UNSIGNED-PAYLOAD'
+ host = parsed_url.netloc
+ headers['host'] = host
+ date = datetime.utcnow()
+ iso8601Date = date.strftime("%Y%m%dT%H%M%SZ")
+ region = get_region(parsed_url.hostname)
+
+ headers_to_sign = dict(headers)
+ ignored_headers = ['Authorization', 'Content-Length', 'Content-Type',
+ 'User-Agent']
+
+ for ignored_header in ignored_headers:
+ if ignored_header in headers_to_sign:
+ del headers_to_sign[ignored_header]
+
+ query = {}
+ query['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256'
+ query['X-Amz-Credential'] = generate_credential_string(access_key, date, region)
+ query['X-Amz-Date'] = iso8601Date
+ query['X-Amz-Expires'] = expires
+ query['X-Amz-SignedHeaders'] = ';'.join(get_signed_headers(headers_to_sign))
+
+ url_components = [parsed_url.geturl()]
+ if query is not None:
+ ordered_query = collections.OrderedDict(sorted(query.items()))
+ query_components = []
+ for component_key in ordered_query:
+ single_component = [component_key]
+ if ordered_query[component_key] is not None:
+ single_component.append('=')
+ single_component.append(
+ urlencode(str(ordered_query[component_key])).replace('/', '%2F'))
+ query_components.append(''.join(single_component))
+
+ query_string = '&'.join(query_components)
+ if query_string:
+ url_components.append('?')
+ url_components.append(query_string)
+ new_url = ''.join(url_components)
+ new_parsed_url = urlsplit(new_url)
+ canonical_request = generate_canonical_request(method,
+ new_parsed_url,
+ headers_to_sign,
+ content_hash_hex)
+
+ canonical_request_hasher = hashlib.sha256()
+ canonical_request_hasher.update(canonical_request.encode('utf-8'))
+ canonical_request_sha256 = canonical_request_hasher.hexdigest()
+
+ string_to_sign = generate_string_to_sign(date, region,
+ canonical_request_sha256)
+ signing_key = generate_signing_key(date, region, secret_key)
+ signature = hmac.new(signing_key, string_to_sign.encode('utf-8'),
+ hashlib.sha256).hexdigest()
+
+ new_parsed_url = urlsplit(new_url + "&X-Amz-Signature="+signature)
+ return new_parsed_url.geturl()
+
+def get_signed_headers(headers):
+ headers_to_sign = dict(headers)
+ ignored_headers = ['Authorization', 'Content-Length', 'Content-Type',
+ 'User-Agent']
+
+ for ignored_header in ignored_headers:
+ if ignored_header in headers_to_sign:
+ del headers_to_sign[ignored_header]
+
+ signed_headers = []
+ for header in headers:
+ signed_headers.append(header)
+ signed_headers.sort()
+
+ return signed_headers
+
 def sign_v4(method, url, headers=None, access_key=None, secret_key=None,
  content_hash=None):
  if not access_key or not secret_key:
@@ -36,7 +128,10 @@ def sign_v4(method, url, headers=None, access_key=None, secret_key=None,
 
  host = parsed_url.netloc
  headers['host'] = host
- headers['x-amz-date'] = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
+ region = get_region(parsed_url.hostname)
+
+ date = datetime.utcnow()
+ headers['x-amz-date'] = date.strftime("%Y%m%dT%H%M%SZ")
  headers['x-amz-content-sha256'] = content_hash_hex
 
  headers_to_sign = dict(headers)
@@ -75,14 +170,11 @@ def sign_v4(method, url, headers=None, access_key=None, secret_key=None,
  if ignored_header in headers_to_sign:
  del headers_to_sign[ignored_header]
 
- canonical_request, signed_headers = generate_canonical_request(method,
- parsed_url,
- headers_to_sign,
- content_hash_hex)
-
- region = get_region(parsed_url.hostname)
-
- date = datetime.utcnow()
+ signed_headers = get_signed_headers(headers_to_sign)
+ canonical_request = generate_canonical_request(method,
+ parsed_url,
+ headers_to_sign,
+ content_hash_hex)
 
  canonical_request_hasher = hashlib.sha256()
  canonical_request_hasher.update(canonical_request.encode('utf-8'))
@@ -91,12 +183,12 @@ def sign_v4(method, url, headers=None, access_key=None, secret_key=None,
  string_to_sign = generate_string_to_sign(date, region,
  canonical_request_sha256)
  signing_key = generate_signing_key(date, region, secret_key)
- signed_request = hmac.new(signing_key, string_to_sign.encode('utf-8'),
-  hashlib.sha256).hexdigest()
+ signature = hmac.new(signing_key, string_to_sign.encode('utf-8'),
+ hashlib.sha256).hexdigest()
 
  authorization_header = generate_authorization_header(access_key, date, region,
  signed_headers,
- signed_request)
+ signature)
 
  headers['authorization'] = authorization_header
 
@@ -134,21 +226,15 @@ def generate_canonical_request(method, parsed_url, headers, content_hash_hex):
  lines.append(';'.join(signed_headers))
  lines.append(str(content_hash_hex))
 
- return '\n'.join(lines), signed_headers
+ return '\n'.join(lines)
 
 
 def generate_string_to_sign(date, region, request_hash):
  formatted_date_time = date.strftime("%Y%m%dT%H%M%SZ")
- formatted_date = date.strftime("%Y%m%d")
-
- scope = '/'.join([formatted_date,
- region,
- 's3',
- 'aws4_request'])
 
  return '\n'.join(['AWS4-HMAC-SHA256',
  formatted_date_time,
- scope,
+ generate_scope_string(date, region),
  request_hash])
 
 
@@ -165,15 +251,21 @@ def generate_signing_key(date, region, secret):
  return hmac.new(key4, 'aws4_request'.encode('utf-8'),
  hashlib.sha256).digest()
 
+def generate_scope_string(date, region):
+ formatted_date = date.strftime("%Y%m%d")
+ scope = '/'.join([formatted_date,
+ region,
+ 's3',
+ 'aws4_request'])
+ return scope
+
+def generate_credential_string(access_key, date, region):
+ return access_key + '/' + generate_scope_string(date, region)
 
 def generate_authorization_header(access_key, date, region, signed_headers,
- signed_request):
- formatted_date = date.strftime("%Y%m%d")
+ signature):
  signed_headers_string = ';'.join(signed_headers)
- auth_header = "AWS4-HMAC-SHA256 Credential=" + access_key + "/" + \
- formatted_date + "/" + region + \
- "/s3/aws4_request, SignedHeaders=" + \
- signed_headers_string + \
- ", Signature=" + \
- signed_request
+ credential = generate_credential_string(access_key, date, region)
+ auth_header = "AWS4-HMAC-SHA256 Credential=" + credential + ", SignedHeaders=" + \
+ signed_headers_string + ", Signature=" + signature
  return auth_header

tests/unit/compat.py

@@ -16,9 +16,9 @@
 import sys
 
 try:
- from urllib.parse import urlparse as compat_urllib_parse
+ from urllib.parse import urlparse as urlsplit
 except ImportError: # python 2
- from urlparse import urlparse as compat_urllib_parse
+ from urlparse import urlparse as urlsplit
 
 strtype = None
 if sys.version_info < (3, 0):