Bump to v7.15.0 (#452)

.github/workflows/ci-test.yml

@@ -60,14 +60,107 @@ jobs:
           QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
           QINIU_TEST_ENV: "travis"
           MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
-          PYTHONPATH: "$PYTHONPATH:."
         run: |
           flake8 --show-source --max-line-length=160 ./qiniu
-          coverage run -m pytest ./test_qiniu.py ./tests/cases
-          ocular --data-file .coverage
-          codecov
+          python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+      - name: Post Setup mock server
+        if: ${{ always() }}
+        shell: bash
+        run: |
+          set +e
           cat mock-server.pid | xargs kill
+          rm mock-server.pid
       - name: Print mock server log
         if: ${{ failure() }}
         run: |
           cat py-mock-server.log
+      - name: Upload results to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+  test-win:
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        python_version: ['2.7', '3.5', '3.9']
+    runs-on: windows-2019
+    # make sure only one test running,
+    # remove this when cases could run in parallel.
+    needs: test
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.ref }}
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge
+          python-version: ${{ matrix.python_version }}
+          activate-environment: qiniu-sdk
+          auto-activate-base: false
+      - name: Setup pip
+        env:
+          PYTHON_VERSION: ${{ matrix.python_version }}
+          PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip
+        run: |
+          # reinstall pip by some python(<3.7) not compatible
+          $pyversion = [Version]"$ENV:PYTHON_VERSION"
+          if ($pyversion -lt [Version]"3.7") {
+            Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py"
+            python $ENV:TEMP\get-pip.py --user
+            Remove-Item -Path "$ENV:TEMP\get-pip.py"
+          }
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -I -e ".[dev]"
+      - name: Run cases
+        env:
+          QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
+          QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
+          QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
+          QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
+          QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
+          QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
+          QINIU_TEST_ENV: "github"
+          MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
+          PYTHONPATH: "$PYTHONPATH:."
+        run: |
+          Write-Host "======== Setup Mock Server ========="
+          conda create -y -n mock-server python=3.10
+          conda activate mock-server
+          python --version
+          $processOptions = @{
+            FilePath="python"
+            ArgumentList="tests\mock_server\main.py", "--port", "9000"
+            PassThru=$true
+            RedirectStandardOutput="py-mock-server.log"
+          }
+          $mocksrvp = Start-Process @processOptions
+          $mocksrvp.Id | Out-File -FilePath "mock-server.pid"
+          conda deactivate
+          Sleep 3
+          Write-Host "======== Running Test ========="
+          python --version
+          python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
+      - name: Post Setup mock server
+        if: ${{ always() }}
+        run: |
+          Try {
+            $mocksrvpid = Get-Content -Path "mock-server.pid"
+            Stop-Process -Id $mocksrvpid
+            Remove-Item -Path "mock-server.pid"
+          } Catch {
+            Write-Host -Object $_
+          }
+      - name: Print mock server log
+        if: ${{ failure() }}
+        run: |
+          Get-Content -Path "py-mock-server.log" | Write-Host
+      - name: Upload results to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -36,6 +36,7 @@ pip-log.txt
 .coverage
 .tox
 nosetests.xml
+coverage.xml
 
 # Translations
 *.mo
@@ -45,4 +46,4 @@ nosetests.xml
 .project
 .pydevproject
 /.idea
-/.venv
+/.venv*

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+## 7.15.0
+* 对象存储，持久化处理支持工作流模版
+* 对象存储，修复 Windows 平台兼容性问题
+
 ## 7.14.0
 * 对象存储，空间管理、上传文件新增备用域名重试逻辑
 * 对象存储，调整查询区域主备域名
@@ -44,7 +48,7 @@
 ## 7.9.0（2022-07-20）
 * 对象存储，支持使用时不配置区域信息，SDK 自动获取；
 * 对象存储，新增 list_domains API 用于查询空间绑定的域名
-* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API 
+* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API
 * 解决部分已知问题
 
 ## 7.8.0（2022-06-08）
@@ -237,5 +241,3 @@
 * 代码覆盖度报告
 * policy改为dict, 便于灵活增加，并加入过期字段检查
 * 文件列表支持目录形式
-
-

codecov.yml

@@ -1,28 +1,28 @@
 codecov:
   ci:
-    - prow.qiniu.io       # prow 里面运行需添加，其他 CI 不要
-  require_ci_to_pass: no  # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。
+    - prow.qiniu.io       # prow need this. seems useless
+  require_ci_to_pass: no  # `no` means the bot will comment on the PR even before all ci passed
 
-github_checks:              #关闭github checks
+github_checks:              # close github checks
   annotations: false
 
 comment:
   layout: "reach, diff, flags, files"
-  behavior: new           # 默认是更新旧留言，改为 new，删除旧的，增加新的。
+  behavior: new           # `new` means the bot will comment a new message instead of edit the old one
   require_changes: false  # if true: only post the comment if coverage changes
   require_base: no        # [yes :: must have a base report to post]
   require_head: yes       # [yes :: must have a head report to post]
   branches:               # branch names that can post comment
       - "master"
 
 coverage:
-  status:                                 # 评判 pr 通过的标准
+  status:                                 # check coverage status to pass or fail
       patch: off
-      project:                            # project 统计所有代码x
+      project:                            # project analyze all code in the project
           default:
             # basic
-            target: 73.5%                  # 总体通过标准
-            threshold: 3%                 # 允许单次下降的幅度
+            target: 73.5%                 # the minimum coverage ratio that the commit must meet
+            threshold: 3%                 # allow the coverage to drop
             base: auto
             if_not_found: success
             if_ci_failed: error

qiniu/__init__.py

@@ -9,7 +9,7 @@
 
 # flake8: noqa
 
-__version__ = '7.14.0'
+__version__ = '7.15.0'
 
 from .auth import Auth, QiniuMacAuth
 

qiniu/auth.py

@@ -34,10 +34,11 @@
     str('fsizeMin'),  # 上传文件最少字节数
     str('keylimit'),  # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key
 
-    str('persistentOps'),  # 持久化处理操作
+    str('persistentOps'),  # 持久化处理操作，与 persistentWorkflowTemplateID 二选一
     str('persistentNotifyUrl'),  # 持久化处理结果通知URL
     str('persistentPipeline'),  # 持久化处理独享队列
     str('persistentType'),  # 为 `1` 时，开启闲时任务，必须是 int 类型
+    str('persistentWorkflowTemplateID'),  # 工作流模板 ID，与 persistentOps 二选一
 
     str('deleteAfterDays'),  # 文件多少天后自动删除
     str('fileType'),  # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储

qiniu/http/regions_provider.py

@@ -5,9 +5,10 @@
 import logging
 import tempfile
 import os
+import shutil
 
 from qiniu.compat import json, b as to_bytes
-from qiniu.utils import io_md5
+from qiniu.utils import io_md5, dt2ts
 
 from .endpoint import Endpoint
 from .region import Region, ServiceName
@@ -264,7 +265,7 @@ def _persist_region(region):
         },
         ttl=region.ttl,
         # use datetime.datetime.timestamp() when min version of python >= 3
-        createTime=int(float(region.create_time.strftime('%s.%f')) * 1000)
+        createTime=dt2ts(region.create_time)
     )._asdict()
 
 
@@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False):
 
     with open(persist_path, 'r') as f:
         for line in f:
+            if not line.strip():
+                continue
             try:
-                cache_key, regions = _parse_persisted_regions(line)
+                cache_key, regions = _parse_persisted_regions(line.strip())
                 yield cache_key, regions
             except Exception as err:
                 if not ignore_parse_error:
@@ -655,7 +658,7 @@ def __shrink_cache(self):
                         )
 
                 # rename file
-                os.rename(shrink_file_path, self._cache_scope.persist_path)
+                shutil.move(shrink_file_path, self._cache_scope.persist_path)
         except FileAlreadyLocked:
             pass
         finally:

qiniu/region.py

@@ -6,7 +6,7 @@
 
 
 from .compat import json, s as str_from_bytes
-from .utils import urlsafe_base64_decode
+from .utils import urlsafe_base64_decode, dt2ts
 from .config import UC_HOST, is_customized_default, get_default
 from .http.endpoint import Endpoint as _HTTPEndpoint
 from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider
@@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False):
 
         ttl = region.ttl if region.ttl > 0 else 24 * 3600  # 1 day
         # use datetime.datetime.timestamp() when min version of python >= 3
-        create_time = int(float(region.create_time.strftime('%s.%f')) * 1000)
+        create_time = dt2ts(region.create_time)
         bucket_hosts['deadline'] = create_time + ttl
 
         return bucket_hosts

qiniu/services/processing/pfop.py

@@ -24,36 +24,47 @@ def __init__(self, auth, bucket, pipeline=None, notify_url=None):
         self.pipeline = pipeline
         self.notify_url = notify_url
 
-    def execute(self, key, fops, force=None, persistent_type=None):
+    def execute(self, key, fops=None, force=None, persistent_type=None, workflow_template_id=None):
         """
         执行持久化处理
 
         Parameters
         ----------
         key: str
             待处理的源文件
-        fops: list[str]
+        fops: list[str], optional
             处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop
+            与 template_id 二选一
         force: int or str, optional
             强制执行持久化处理开关
         persistent_type: int or str, optional
             持久化处理类型，为 '1' 时开启闲时任务
+        template_id: str, optional
+            与 fops 二选一
         Returns
         -------
         ret: dict
             持久化处理的 persistentId，类似 {"persistentId": 5476bedf7823de4068253bae};
         resp: ResponseInfo
         """
-        ops = ';'.join(fops)
-        data = {'bucket': self.bucket, 'key': key, 'fops': ops}
+        if not fops and not workflow_template_id:
+            raise ValueError('Must provide one of fops or template_id')
+        data = {
+            'bucket': self.bucket,
+            'key': key,
+        }
         if self.pipeline:
             data['pipeline'] = self.pipeline
         if self.notify_url:
             data['notifyURL'] = self.notify_url
+        if fops:
+            data['fops'] = ';'.join(fops)
         if force == 1 or force == '1':
             data['force'] = str(force)
         if persistent_type and type(int(persistent_type)) is int:
             data['type'] = str(persistent_type)
+        if workflow_template_id:
+            data['workflowTemplateID'] = workflow_template_id
 
         url = '{0}/pfop'.format(config.get_default('default_api_host'))
         return http._post_with_auth(url, data, self.auth)

qiniu/utils.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 from hashlib import sha1, new as hashlib_new
 from base64 import urlsafe_b64encode, urlsafe_b64decode
-from datetime import datetime
+from datetime import datetime, tzinfo, timedelta
+
 from .compat import b, s
 
 try:
@@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name):
             result += ch
         upper = ch == "-"
     return result
+
+
+class _UTC_TZINFO(tzinfo):
+    def utcoffset(self, dt):
+        return timedelta(hours=0)
+
+    def tzname(self, dt):
+        return "UTC"
+
+    def dst(self, dt):
+        return timedelta(0)
+
+
+def dt2ts(dt):
+    """
+    converte datetime to timestamp
+
+    Parameters
+    ----------
+    dt: datetime.datetime
+    """
+    if not dt.tzinfo:
+        st = (dt - datetime(1970, 1, 1)).total_seconds()
+    else:
+        st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds()
+
+    return int(st)

setup.py

@@ -42,10 +42,8 @@ def find_version(*file_paths):
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
@@ -66,8 +64,6 @@ def find_version(*file_paths):
             'pytest',
             'pytest-cov',
             'freezegun',
-            'scrutinizer-ocular',
-            'codecov'
         ]
     },