upload 3.20.3

huaweicloud · Jun 19, 2020 · cdf7239 · cdf7239
1 parent 350fb94
commit cdf7239
Show file tree

Hide file tree

Showing 9 changed files with 249 additions and 12 deletions.
diff --git a/release/huaweicloud-obs-sdk-python_3.20.3.tar.gz b/release/huaweicloud-obs-sdk-python_3.20.3.tar.gz
diff --git a/release/huaweicloud-obs-sdk-python_3.20.3.tar.gz.sha256 b/release/huaweicloud-obs-sdk-python_3.20.3.tar.gz.sha256
@@ -0,0 +1 @@
+a8c877e608a19cee9c847df5ea632fb331f05bbd6a5d80382ec10ed3f37b1095 *huaweicloud-obs-sdk-python_3.20.3.tar.gz
diff --git a/src/obs/__init__.py b/src/obs/__init__.py
@@ -15,7 +15,7 @@
 from obs.ilog import LogConf
 from obs.client import ObsClient
 from obs.model import *
-
+from obs.model import ExtensionHeader, FetchStatus
 
 __all__ = [
     'LogConf',
@@ -74,6 +74,7 @@
     'GetObjectRequest',
     'UploadFileHeader',
     'Payer',
-    'ExtensionHeader'
+    'ExtensionHeader',
+    'FetchStatus'
 ]
 
diff --git a/src/obs/bucket.py b/src/obs/bucket.py
@@ -78,7 +78,12 @@ class BucketClient(object):
         'deleteBucketEncryption',
         'headObject',
         'setBucketRequestPayment',
-        'getBucketRequestPayment'
+        'getBucketRequestPayment',
+        'setBucketFetchPolicy',
+        'getBucketFetchPolicy',
+        'deleteBucketFetchPolicy',
+        'setBucketFetchJob',
+        'getBucketFetchJob'
     ]
 
     def __init__(self, obsClient, bucketName):

diff --git a/src/obs/client.py b/src/obs/client.py
@@ -45,6 +45,7 @@
 from obs.bucket import BucketClient
 from obs import loadtoken
 from inspect import isfunction
+from obs.model import FetchPolicy, _FetchJob
 
 if const.IS_PYTHON2:
     from urlparse import urlparse
@@ -858,7 +859,10 @@ def _parse_xml_internal(self, result, methodName=None, chuckSize=65536, readable
             try:
                 search = self.pattern.search(xml)
                 xml = xml if search is None else xml.replace(search.group(), '')
-                code, message, requestId, hostId, resource = self.convertor.parseErrorResult(xml)
+                if headers.get("content-type", "not exsited") == const.MIME_TYPES.get("json"):
+                    code, message, requestId = self.convertor.parseJsonErrorResult(xml)
+                else:
+                    code, message, requestId, hostId, resource = self.convertor.parseErrorResult(xml)
             except Exception as ee:
                 self.log_client.log(ERROR, util.to_string(ee))
                 self.log_client.log(ERROR, traceback.format_exc())
@@ -1876,6 +1880,49 @@ def downloadFiles(self, bucketName, prefix, downloadFolder=None, taskNum=const.D
                       threshold=const.DEFAULT_MAXIMUM_SIZE, partSize=5*1024*1024, subTaskNum=1, enableCheckpoint=False, checkpointFile=None, extensionHeaders=None):
         return _download_files(self, bucketName, prefix, downloadFolder, taskNum, taskQueueSize, headers, imageProcess, 
                                interval, taskCallback, progressCallback, threshold, partSize, subTaskNum, enableCheckpoint, checkpointFile, extensionHeaders=extensionHeaders)
+
+    #OEF interface
+
+    @funcCache
+    def setBucketFetchPolicy(self, bucketName, status, agency, extensionHeaders=None):
+        self._assert_not_null(status, "status is empty")
+        self._assert_not_null(agency, "agency is empty")
+        fetchPolicy = FetchPolicy(status, agency)
+        return self._make_put_request(bucketName, const.FETCH_POLICY_KEY, extensionHeaders=extensionHeaders,
+                                      **self.convertor.trans_set_bucket_fetch_policy(fetchPolicy))
+
+    @funcCache
+    def getBucketFetchPolicy(self, bucketName, extensionHeaders=None):
+        headers = {self.ha.oef_marker_header(): "yes"}
+        return self._make_get_request(bucketName, const.FETCH_POLICY_KEY, methodName="getBucketFetchPolicy",
+                                      headers=headers, extensionHeaders=extensionHeaders)
+
+    @funcCache
+    def deleteBucketFetchPolicy(self, bucketName, extensionHeaders=None):
+        headers = {self.ha.oef_marker_header(): "yes"}
+        return self._make_delete_request(bucketName, const.FETCH_POLICY_KEY, headers=headers,
+                                         extensionHeaders=extensionHeaders)
+
+    @funcCache
+    def setBucketFetchJob(self, bucketName, url, host=None, key=None, md5=None, callBackUrl=None,
+                          callBackBody=None, callBackBodyType=None, callBackHost=None, fileType=None,
+                          ignoreSameKey=False, objectHeaders=None, etag=None, trustName=None, extensionHeaders=None):
+        self._assert_not_null(url, "url is empty")
+        fetchJob = _FetchJob(url=url, host=host, bucket=bucketName, key=key, md5=md5, callBackUrl=callBackUrl,
+                             callBackBody=callBackBody, callBackBodyType=callBackBodyType, callBackHost=callBackHost,
+                             fileType=fileType, ignoreSameKey=ignoreSameKey, objectHeaders=objectHeaders, etag=etag,
+                             trustName=trustName)
+        return self._make_post_request(bucketName, const.FETCH_JOB_KEY, methodName="setBucketFetchJob",
+                                       extensionHeaders=extensionHeaders,
+                                       **self.convertor.trans_set_bucket_fetch_job(fetchJob))
+
+    @funcCache
+    def getBucketFetchJob(self, bucketName, jobId, extensionHeaders=None):
+        self._assert_not_null(jobId, "jobId is empty")
+        headers = {self.ha.oef_marker_header(): "yes"}
+        key = const.FETCH_JOB_KEY + "/" + jobId
+        return self._make_get_request(bucketName, key, methodName="getBucketFetchJob", headers=headers,
+                                      extensionHeaders=extensionHeaders)
 
 
 

diff --git a/src/obs/const.py b/src/obs/const.py
@@ -87,7 +87,7 @@
 DEFAULT_TASK_QUEUE_SIZE = 20000
 
 
-OBS_SDK_VERSION = '3.20.1'
+OBS_SDK_VERSION = '3.20.3'
 
 V2_META_HEADER_PREFIX = 'x-amz-meta-'
 V2_HEADER_PREFIX = 'x-amz-'
@@ -108,6 +108,9 @@
 V2_SIGNATURE = 'v2'
 OBS_SIGNATURE = 'obs'
 
+FETCH_POLICY_KEY = "v1/extension_policy"
+FETCH_JOB_KEY = "v1/async-fetch/jobs"
+
 ALLOWED_RESOURCE_PARAMTER_NAMES = (
         'acl',
         'backtosource',
@@ -212,7 +215,8 @@
         'access-control-allow-methods',
         'access-control-expose-headers',
         'connection',
-        'x-reserved-indicator'
+        'x-reserved-indicator',
+        'x-oef-request-id'
 )
 
 

diff --git a/src/obs/convertor.py b/src/obs/convertor.py
@@ -16,9 +16,11 @@
     import xml.etree.cElementTree as ET
 except:
     import xml.etree.ElementTree as ET
+import json
 from obs.model import *
 from obs import util
 from obs import const
+from obs.model import FetchPolicy, SetBucketFetchJobResponse, GetBucketFetchJobResponse, FetchJobResponse
 
 class Adapter(object):
 
@@ -167,7 +169,10 @@ def object_type_header(self):
 
     def request_payer_header(self):
         return self._get_header_prefix() + 'request-payer'
-
+
+    def oef_marker_header(self):
+        return self._get_header_prefix() + 'oef-marker'
+
     def adapt_group(self, group):
         if self.is_obs:
             return group if group in self.OBS_ALLOWED_GROUP else 'Everyone' if group == 'http://acs.amazonaws.com/groups/global/AllUsers' or group == 'AllUsers' else None
@@ -921,7 +926,33 @@ def trans_get_extension_headers(self, headers):
         _headers = {}
         if headers is not None and len(headers) > 0:
             self._put_key_value(_headers, self.ha.request_payer_header(), (headers.get('requesterPayer')))
-        return  _headers
+        return _headers
+
+    # OEF trans func
+    def trans_set_bucket_fetch_policy(self, fetchPolicy):
+        headers = {}
+        self._put_key_value(headers, const.CONTENT_TYPE_HEADER, const.MIME_TYPES.get("json"))
+        self._put_key_value(headers, self.ha.oef_marker_header(), "yes")
+        jsonPolicy = {"fetch": fetchPolicy}
+        entity = json.dumps(jsonPolicy, ensure_ascii=False)
+        return {'headers': headers, 'entity': entity}
+
+    def trans_set_bucket_fetch_job(self, fetchJob):
+        headers = {}
+        self._put_key_value(headers, const.CONTENT_TYPE_HEADER, const.MIME_TYPES.get("json"))
+        self._put_key_value(headers, self.ha.oef_marker_header(), "yes")
+        if fetchJob.get("objectheaders") is not None:
+            for k in list(fetchJob["objectheaders"].keys()):
+                v = fetchJob["objectheaders"].get(k)
+                if v is not None:
+                    if not util.to_string(k).lower().startswith(self.ha._get_header_prefix()):
+                        del fetchJob["objectheaders"][k]
+                        k = self.ha._get_meta_header_prefix() + k
+                    fetchJob["objectheaders"][k] = v
+                else:
+                    del fetchJob["objectheaders"][k]
+        entity = json.dumps(fetchJob, ensure_ascii=False)
+        return {'headers': headers, 'entity': entity}
 
     def _find_item(self, root, itemname):
         result = root.find(itemname)
@@ -1646,4 +1677,68 @@ def parseGetBucketRequestPayment(self, xml, headers=None):
         root = ET.fromstring(xml)
         payer =self._find_item(root, 'Payer')
         payment = GetBucketRequestPaymentResponse(payer=payer)
-        return payment
+        return payment
+
+    #OEF parse func
+    def _find_json_item(self, value, itemname):
+        result = value.get(itemname)
+        if result is None:
+            return None
+        if const.IS_PYTHON2:
+            result = util.safe_encode(result)
+        return util.to_string(result)
+
+    def parseJsonErrorResult(self, jsons):
+        result = json.loads(jsons)
+        code = self._find_json_item(result, "code")
+        message = self._find_json_item(result, "message")
+        requestId = self._find_json_item(result, "request_id")
+        return code, message, requestId
+
+    def parseGetBucketFetchPolicy(self, jsons, headers=None):
+        result = json.loads(jsons)
+        status = None
+        agency = None
+        fetchResult = result.get("fetch")
+        if fetchResult is not None:
+            status = self._find_json_item(fetchResult, "status")
+            agency = self._find_json_item(fetchResult, "agency")
+        policy = FetchPolicy(status=status, agency=agency)
+        return policy
+
+    def parseSetBucketFetchJob(self, jsons, headers=None):
+        result = json.loads(jsons)
+        ID = self._find_json_item(result, "id")
+        wait = result.get("Wait")
+        response = SetBucketFetchJobResponse(id=ID, wait=wait)
+        return response
+
+    def parseGetBucketFetchJob(self, jsons, header=None):
+        result = json.loads(jsons)
+        err = self._find_json_item(result, "err")
+        code = self._find_json_item(result, "code")
+        status = self._find_json_item(result, "status")
+
+        job = result.get("job")
+        if job is None:
+            response = GetBucketFetchJobResponse(code=code, status=status, job=None, err=err)
+            return response
+
+        url = self._find_json_item(job, "url")
+        host = self._find_json_item(job, "host")
+        bucket = self._find_json_item(job, "bucket")
+        key = self._find_json_item(job, "key")
+        md5 = self._find_json_item(job, "md5")
+        fileType = self._find_json_item(job, "file_type")
+        ignoreSameKey = self._find_json_item(job, "ignore_same_key")
+        callBackUrl = self._find_json_item(job, "callbackurl")
+        callBackBody = self._find_json_item(job, "callbackbody")
+        callBackHost = self._find_json_item(job, "callbackhost")
+        callBackBodyType = self._find_json_item(job, "callbackbodytype")
+        fetchJobResponse = FetchJobResponse(url=url, host=host, bucket=bucket, key=key, md5=md5, fileType=fileType,
+                                            ignoreSameKey=ignoreSameKey, callBackUrl=callBackUrl,
+                                            callBackBody=callBackBody,
+                                            callBackHost=callBackHost, callBackBodyType=callBackBodyType)
+
+        response = GetBucketFetchJobResponse(code=code, status=status, job=fetchJobResponse, err=err)
+        return response
diff --git a/src/obs/model.py b/src/obs/model.py
@@ -114,7 +114,13 @@
     'UploadFileHeader',
     'GetBucketRequestPaymentResponse',
     'Payer',
-    'ExtensionHeader'
+    'ExtensionHeader',
+    'FetchStatus',
+    'FetchPolicy',
+    '_FetchJob',
+    'FetchJobResponse',
+    'SetBucketFetchJobResponse',
+    'GetBucketFetchJobResponse'
 ]
 
 
@@ -1245,4 +1251,82 @@ class ExtensionHeader(BaseModel):
     allowedAttr = {'requesterPayer': BASESTRING}
 
     def __init__(self, requesterPayer=None):
-        self.requesterPayer = requesterPayer
+        self.requesterPayer = requesterPayer
+
+
+#OEF Model
+class FetchStatus(object):
+    OPEN = "open"
+    CLOSED = "closed"
+
+
+class FetchPolicy(BaseModel):
+    allowedAttr = {'status': BASESTRING, 'agency': BASESTRING}
+
+    def __init__(self, status=None, agency=None):
+        self.status = status
+        self.agency = agency
+
+
+class _FetchJob(BaseModel):
+    allowedAttr = {'url': BASESTRING, 'host': BASESTRING, 'bucket': BASESTRING, 'key': BASESTRING, 'md5': BASESTRING,
+                   'callbackurl': BASESTRING, 'callbackbody': BASESTRING, 'callbackbodytype': BASESTRING,
+                   'callbackhost': BASESTRING, 'file_type': BASESTRING, 'ignore_same_key': bool, 'objectheaders': dict,
+                   'etag': BASESTRING, 'trustname': BASESTRING}
+
+    def __init__(self, url=None, host=None, bucket=None, key=None, md5=None, callBackUrl=None,
+                 callBackBody=None, callBackBodyType=None, callBackHost=None, fileType=None,
+                 ignoreSameKey=False, objectHeaders=None, etag=None, trustName=None):
+        self.url = url
+        self.host = host
+        self.bucket = bucket
+        self.key = key
+        self.md5 = md5
+        self.callbackurl = callBackUrl
+        self.callbackbody = callBackBody
+        self.callbackbodytype = callBackBodyType
+        self.callbackhost = callBackHost
+        self.file_type = fileType
+        self.ignore_same_key = ignoreSameKey
+        self.objectheaders = objectHeaders
+        self.etag = etag
+        self.trustname = trustName
+
+
+class SetBucketFetchJobResponse(BaseModel):
+    allowedAttr = {"id": BASESTRING, "wait": int}
+
+    def __init__(self, id=None, wait=None):
+        self.id = id
+        self.wait = wait
+
+
+class FetchJobResponse(BaseModel):
+    allowedAttr = {'url': BASESTRING, 'host': BASESTRING, 'bucket': BASESTRING, 'key': BASESTRING, 'md5': BASESTRING,
+                   'callBackUrl': BASESTRING, 'callBackBody': BASESTRING, 'callBackBodyType': BASESTRING,
+                   'callBackHost': BASESTRING, 'fileType': BASESTRING, 'ignoreSameKey': bool}
+
+    def __init__(self, url=None, host=None, bucket=None, key=None, md5=None, callBackUrl=None,
+                 callBackBody=None, callBackBodyType=None, callBackHost=None, fileType=None,
+                 ignoreSameKey=False):
+        self.url = url
+        self.host = host
+        self.bucket = bucket
+        self.key = key
+        self.md5 = md5
+        self.callBackUrl = callBackUrl
+        self.callBackBody = callBackBody
+        self.callBackBodyType = callBackBodyType
+        self.callBackHost = callBackHost
+        self.fileType = fileType
+        self.ignoreSameKey = ignoreSameKey
+
+
+class GetBucketFetchJobResponse(BaseModel):
+    allowedAttr = {"code": BASESTRING, "err": BASESTRING, "status": BASESTRING, "job": FetchJobResponse}
+
+    def __init__(self, code=None, status=None, job=None, err=None):
+        self.code = code
+        self.status = status
+        self.job = job
+        self.err = err
diff --git a/src/setup.py b/src/setup.py
@@ -23,7 +23,7 @@
 
 setup(
     name='esdk-obs-python',
-    version='3.20.1',
+    version='3.20.3',
     packages=find_packages(),
     zip_safe=False,
     description='OBS Python SDK',