bump to 7.14.0

.github/workflows/ci-test.yml

@@ -55,6 +55,7 @@ jobs:
           QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
           QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
           QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
+          QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
           QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
           QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
           QINIU_TEST_ENV: "travis"

examples/upload.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
+# import hashlib
 
-from qiniu import Auth, put_file, etag, urlsafe_base64_encode
+from qiniu import Auth, put_file, urlsafe_base64_encode
 import qiniu.config
 from qiniu.compat import is_py2, is_py3
 
@@ -24,13 +25,28 @@
 # 要上传文件的本地路径
 localfile = '/Users/jemy/Documents/qiniu.png'
 
-ret, info = put_file(token, key, localfile)
+# 上传时，sdk 会自动计算文件 hash 作为参数传递给服务端确保上传完整性
+# （若不一致，服务端会拒绝完成上传）
+# 但在访问文件时，服务端可能不会提供 MD5 或者编码格式不是期望的
+# 因此若有需有，请通过元数据功能自定义 MD5 或其他 hash 字段
+# hasher = hashlib.md5()
+# with open(localfile, 'rb') as f:
+#     for d in f:
+#         hasher.update(d)
+# object_metadata = {
+#     'x-qn-meta-md5': hasher.hexdigest()
+# }
+
+ret, info = put_file(
+    token,
+    key,
+    localfile
+    # metadata=object_metadata
+)
 print(ret)
 print(info)
 
 if is_py2:
     assert ret['key'].encode('utf-8') == key
 elif is_py3:
     assert ret['key'] == key
-
-assert ret['hash'] == etag(localfile)

examples/upload_callback.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
 
-from qiniu import Auth, put_file, etag
+from qiniu import Auth, put_file
 
 access_key = '...'
 secret_key = '...'
@@ -25,4 +25,3 @@
 ret, info = put_file(token, key, localfile)
 print(info)
 assert ret['key'] == key
-assert ret['hash'] == etag(localfile)

examples/upload_pfops.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
-from qiniu import Auth, put_file, etag, urlsafe_base64_encode
+from qiniu import Auth, put_file, urlsafe_base64_encode
 
 access_key = '...'
 secret_key = '...'
@@ -36,4 +36,3 @@
 ret, info = put_file(token, key, localfile)
 print(info)
 assert ret['key'] == key
-assert ret['hash'] == etag(localfile)

examples/upload_with_qvmzone.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
 
-from qiniu import Auth, put_file, etag, urlsafe_base64_encode
+from qiniu import Auth, put_file, urlsafe_base64_encode
 import qiniu.config
 from qiniu import Zone, set_default
 
@@ -37,4 +37,3 @@
 ret, info = put_file(token, key, localfile)
 print(info)
 assert ret['key'] == key
-assert ret['hash'] == etag(localfile)

qiniu/__init__.py

@@ -15,10 +15,10 @@
 
 from .config import set_default
 from .zone import Zone
-from .region import Region
+from .region import LegacyRegion as Region
 
 from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \
-    build_batch_stat, build_batch_delete, build_batch_restoreAr
+    build_batch_stat, build_batch_delete, build_batch_restoreAr, build_batch_restore_ar
 from .services.storage.uploader import put_data, put_file, put_stream
 from .services.storage.upload_progress_recorder import UploadProgressRecorder
 from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager

qiniu/auth.py

@@ -37,6 +37,7 @@
     str('persistentOps'),  # 持久化处理操作
     str('persistentNotifyUrl'),  # 持久化处理结果通知URL
     str('persistentPipeline'),  # 持久化处理独享队列
+    str('persistentType'),  # 指定是否开始闲时任务
     str('deleteAfterDays'),  # 文件多少天后自动删除
     str('fileType'),  # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储
     str('isPrefixalScope'),  # 指定上传文件必须使用的前缀

qiniu/config.py

@@ -1,26 +1,25 @@
 # -*- coding: utf-8 -*-
-from qiniu import region
-
 RS_HOST = 'http://rs.qiniu.com'  # 管理操作Host
 RSF_HOST = 'http://rsf.qbox.me'  # 列举操作Host
 API_HOST = 'http://api.qiniuapi.com'  # 数据处理操作Host
-UC_HOST = region.UC_HOST  # 获取空间信息Host
-QUERY_REGION_HOST = 'https://kodo-config.qiniuapi.com'
+UC_HOST = 'https://uc.qbox.me'  # 获取空间信息Host
+QUERY_REGION_HOST = 'https://uc.qiniuapi.com'
+QUERY_REGION_BACKUP_HOSTS = [
+    'kodo-config.qiniuapi.com',
+    'uc.qbox.me'
+]
 
 _BLOCK_SIZE = 1024 * 1024 * 4  # 断点续传分块大小，该参数为接口规格，暂不支持修改
 
 _config = {
-    'default_zone': region.Region(),
+    'default_zone': None,
     'default_rs_host': RS_HOST,
     'default_rsf_host': RSF_HOST,
     'default_api_host': API_HOST,
     'default_uc_host': UC_HOST,
     'default_query_region_host': QUERY_REGION_HOST,
-    'default_query_region_backup_hosts': [
-        'uc.qbox.me',
-        'api.qiniu.com'
-    ],
-    'default_backup_hosts_retry_times': 2,
+    'default_query_region_backup_hosts': QUERY_REGION_BACKUP_HOSTS,
+    'default_backup_hosts_retry_times': 3,  # 仅控制旧区域 LegacyRegion 查询 Hosts 的重试次数
     'connection_timeout': 30,  # 链接超时为时间为30s
     'connection_retries': 3,  # 链接重试次数为3次
     'connection_pool': 10,  # 链接池个数为10
@@ -48,6 +47,10 @@ def is_customized_default(key):
 
 
 def get_default(key):
+    if key == 'default_zone' and not _is_customized_default[key]:
+        # prevent circle import
+        from .region import LegacyRegion
+        return LegacyRegion()
     return _config[key]
 
 

qiniu/http/__init__.py

@@ -1,39 +1,15 @@
 # -*- coding: utf-8 -*-
 import logging
 import platform
-import functools
 
 import requests
-from requests.adapters import HTTPAdapter
 from requests.auth import AuthBase
 
 from qiniu import config, __version__
 import qiniu.auth
 
-from .client import HTTPClient
 from .response import ResponseInfo
-from .middleware import UserAgentMiddleware
-
-
-qn_http_client = HTTPClient(
-    middlewares=[
-        UserAgentMiddleware(__version__)
-    ]
-)
-
-
-# compatibility with some config from qiniu.config
-def _before_send(func):
-    @functools.wraps(func)
-    def wrapper(self, *args, **kwargs):
-        if _session is None:
-            _init()
-        return func(self, *args, **kwargs)
-
-    return wrapper
-
-
-qn_http_client.send_request = _before_send(qn_http_client.send_request)
+from .default_client import qn_http_client, _init_http_adapter
 
 _sys_info = '{0}; {1}'.format(platform.system(), platform.machine())
 _python_ver = platform.python_version()
@@ -61,12 +37,7 @@ def _init():
     global _session
     if _session is None:
         _session = qn_http_client.session
-
-    adapter = HTTPAdapter(
-        pool_connections=config.get_default('connection_pool'),
-        pool_maxsize=config.get_default('connection_pool'),
-        max_retries=config.get_default('connection_retries'))
-    _session.mount('http://', adapter)
+    _init_http_adapter()
 
 
 def _post(url, data, files, auth, headers=None):

qiniu/http/default_client.py

@@ -0,0 +1,37 @@
+import functools
+
+from requests.adapters import HTTPAdapter
+
+from qiniu import config, __version__
+
+from .client import HTTPClient
+from .middleware import UserAgentMiddleware
+
+qn_http_client = HTTPClient(
+    middlewares=[
+        UserAgentMiddleware(__version__)
+    ]
+)
+
+
+# compatibility with some config from qiniu.config
+def _before_send(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        _init_http_adapter()
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+qn_http_client.send_request = _before_send(qn_http_client.send_request)
+
+
+def _init_http_adapter():
+    # may be optimized:
+    # only called when config changed, not every time before send request
+    adapter = HTTPAdapter(
+        pool_connections=config.get_default('connection_pool'),
+        pool_maxsize=config.get_default('connection_pool'),
+        max_retries=config.get_default('connection_retries'))
+    qn_http_client.session.mount('http://', adapter)

qiniu/http/endpoint.py

@@ -0,0 +1,68 @@
+class Endpoint:
+    @staticmethod
+    def from_host(host):
+        """
+        Autodetect scheme from host string
+
+        Parameters
+        ----------
+        host: str
+
+        Returns
+        -------
+        Endpoint
+        """
+        if '://' in host:
+            scheme, host = host.split('://')
+            return Endpoint(host=host, default_scheme=scheme)
+        else:
+            return Endpoint(host=host)
+
+    def __init__(self, host, default_scheme='https'):
+        """
+        Parameters
+        ----------
+        host: str
+        default_scheme: str
+        """
+        self.host = host
+        self.default_scheme = default_scheme
+
+    def __str__(self):
+        return 'Endpoint(host:\'{0}\',default_scheme:\'{1}\')'.format(
+            self.host,
+            self.default_scheme
+        )
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __eq__(self, other):
+        if not isinstance(other, Endpoint):
+            raise TypeError('Cannot compare Endpoint with {0}'.format(type(other)))
+
+        return self.host == other.host and self.default_scheme == other.default_scheme
+
+    def get_value(self, scheme=None):
+        """
+        Parameters
+        ----------
+        scheme: str
+
+        Returns
+        -------
+        str
+        """
+        scheme = scheme if scheme is not None else self.default_scheme
+        return ''.join([scheme, '://', self.host])
+
+    def clone(self):
+        """
+        Returns
+        -------
+        Endpoint
+        """
+        return Endpoint(
+            host=self.host,
+            default_scheme=self.default_scheme
+        )

qiniu/http/endpoints_provider.py

@@ -0,0 +1,13 @@
+import abc
+
+
+class EndpointsProvider:
+    __metaclass__ = abc.ABCMeta
+
+    @abc.abstractmethod
+    def __iter__(self):
+        """
+        Returns
+        -------
+        list[Endpoint]
+        """

qiniu/http/endpoints_retry_policy.py

@@ -0,0 +1,56 @@
+from qiniu.retry.abc import RetryPolicy
+
+
+class EndpointsRetryPolicy(RetryPolicy):
+    def __init__(self, endpoints_provider=None, skip_init_context=False):
+        """
+        Parameters
+        ----------
+        endpoints_provider: Iterable[Endpoint]
+        skip_init_context: bool
+        """
+        self.endpoints_provider = endpoints_provider if endpoints_provider else []
+        self.skip_init_context = skip_init_context
+
+    def init_context(self, context):
+        """
+        Parameters
+        ----------
+        context: dict
+
+        Returns
+        -------
+        None
+        """
+        if self.skip_init_context:
+            return
+        context['alternative_endpoints'] = list(self.endpoints_provider)
+        if not context['alternative_endpoints']:
+            raise ValueError('There isn\'t available endpoint')
+        context['endpoint'] = context['alternative_endpoints'].pop(0)
+
+    def should_retry(self, attempt):
+        """
+        Parameters
+        ----------
+        attempt: qiniu.retry.Attempt
+
+        Returns
+        -------
+        bool
+        """
+        return len(attempt.context['alternative_endpoints']) > 0
+
+    def prepare_retry(self, attempt):
+        """
+        Parameters
+        ----------
+        attempt: qiniu.retry.Attempt
+
+        Returns
+        -------
+        None
+        """
+        if not attempt.context['alternative_endpoints']:
+            raise Exception('There isn\'t available endpoint for next try')
+        attempt.context['endpoint'] = attempt.context['alternative_endpoints'].pop(0)