Merge pull request #1537 from ITISFoundation/FREEZE_ChrigelMaurer_v1

FREEZE_ChrigelMaurer_v1

api/tests/requirements.txt

@@ -27,12 +27,12 @@ py==1.8.1                 # via pytest
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements.in
-pytest-cov==2.8.1         # via -r requirements.in
+pytest-cov==2.9.0         # via -r requirements.in
 pytest-instafail==0.4.1.post0  # via -r requirements.in
 pytest-sugar==0.9.3       # via -r requirements.in
 pytest==5.4.2             # via -r requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-sugar
 pyyaml==5.3.1             # via openapi-spec-validator
-six==1.14.0               # via isodate, jsonschema, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging
+six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging
 strict-rfc3339==0.7       # via openapi-schema-validator
 termcolor==1.1.0          # via pytest-sugar
 typing-extensions==3.7.4.2  # via aiohttp

docs/staging-instructions.md

@@ -17,7 +17,7 @@ The process of moving code from [Master](https://github.com/ITISFoundation/ospar
     git push --set-upstream origin FREEZE_${SPRINT_NAME}
     # create the log entries to be copied into the pull request
     #body=scripts/url-encoder.sh "$(git log --oneline staging..HEAD --no-decorate)"
-    body=$(scripts/url-encoder.sh "$(git log staging..HEAD --pretty="format:- %s")")
+    body=$(scripts/url-encoder.bash "$(git log origin/staging..HEAD --pretty="format:- %s")")
     # open the PR on github website
     echo "https://github.com/ITISFoundation/osparc-simcore/compare/staging...FREEZE_${SPRINT_NAME}?expand=1&title=FREEZE_${SPRINT_NAME}&body=$body"
 

packages/postgres-database/requirements/_migration.txt

@@ -17,7 +17,7 @@ psycopg2-binary==2.8.5    # via -r requirements/_base.txt, sqlalchemy
 python-dateutil==2.8.1    # via alembic
 python-editor==1.0.4      # via alembic
 requests==2.23.0          # via docker
-six==1.14.0               # via docker, python-dateutil, tenacity, websocket-client
+six==1.15.0               # via docker, python-dateutil, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.txt, alembic
 tenacity==6.2.0           # via -r requirements/_migration.in
 urllib3==1.25.9           # via -r requirements/_migration.in, requests

packages/postgres-database/requirements/_test.txt

@@ -35,7 +35,7 @@ py==1.8.1                 # via pytest
 pylint==2.5.0             # via -r requirements/_test.in
 pyparsing==2.4.7          # via packaging
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
-pytest-cov==2.8.1         # via -r requirements/_test.in
+pytest-cov==2.9.0         # via -r requirements/_test.in
 pytest-docker==0.7.2      # via -r requirements/_test.in
 pytest-instafail==0.4.1.post0  # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
@@ -44,7 +44,7 @@ python-dateutil==2.8.1    # via -r requirements/_migration.txt, alembic, faker
 python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
 pyyaml==5.3.1             # via -r requirements/_test.in
 requests==2.23.0          # via -r requirements/_migration.txt, coveralls, docker
-six==1.14.0               # via -r requirements/_migration.txt, astroid, docker, packaging, python-dateutil, tenacity, websocket-client
+six==1.15.0               # via -r requirements/_migration.txt, astroid, docker, packaging, python-dateutil, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_migration.txt, aiopg, alembic
 tenacity==6.2.0           # via -r requirements/_migration.txt
 text-unidecode==1.3       # via faker

packages/s3wrapper/requirements/_base.txt

@@ -9,5 +9,5 @@ configparser==5.0.0       # via minio
 minio==5.0.10             # via -r requirements/_base.in
 python-dateutil==2.8.1    # via minio
 pytz==2020.1              # via minio
-six==1.14.0               # via python-dateutil
+six==1.15.0               # via python-dateutil
 urllib3==1.25.9           # via -r requirements/_base.in, minio

packages/s3wrapper/requirements/_test.txt

@@ -24,14 +24,14 @@ pluggy==0.13.1            # via pytest
 py==1.8.1                 # via pytest
 pylint==2.5.0             # via -r requirements/_test.in
 pyparsing==2.4.7          # via packaging
-pytest-cov==2.8.1         # via -r requirements/_test.in
+pytest-cov==2.9.0         # via -r requirements/_test.in
 pytest-docker==0.7.2      # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest==5.4.2             # via -r requirements/_test.in, pytest-cov
 python-dateutil==2.8.1    # via -r requirements/_base.txt, minio
 pytz==2020.1              # via -r requirements/_base.txt, minio
 requests==2.23.0          # via -r requirements/_test.in, coveralls
-six==1.14.0               # via -r requirements/_base.txt, astroid, packaging, python-dateutil
+six==1.15.0               # via -r requirements/_base.txt, astroid, packaging, python-dateutil
 toml==0.10.1              # via pylint
 typed-ast==1.4.1          # via astroid
 urllib3==1.25.9           # via -r requirements/_base.txt, minio, requests

packages/service-library/requirements/_base.txt

@@ -20,11 +20,11 @@ lazy-object-proxy==1.4.3  # via openapi-core
 multidict==4.7.6          # via aiohttp, yarl
 openapi-core==0.12.0      # via -r requirements/_base.in
 openapi-spec-validator==0.2.8  # via openapi-core
-prometheus-client==0.7.1  # via -r requirements/_base.in
+prometheus-client==0.8.0  # via -r requirements/_base.in
 psycopg2-binary==2.8.5    # via -r requirements/_base.in, aiopg, sqlalchemy
 pyrsistent==0.16.0        # via jsonschema
 pyyaml==5.3.1             # via -r requirements/_base.in, openapi-spec-validator
-six==1.14.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
+six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/_base.in

packages/service-library/requirements/_test.txt

@@ -30,14 +30,14 @@ openapi-core==0.12.0      # via -r requirements/_base.txt
 openapi-spec-validator==0.2.8  # via -r requirements/_base.txt, openapi-core
 packaging==20.4           # via pytest, pytest-sugar
 pluggy==0.13.1            # via pytest
-prometheus-client==0.7.1  # via -r requirements/_base.txt
+prometheus-client==0.8.0  # via -r requirements/_base.txt
 psycopg2-binary==2.8.5    # via -r requirements/_base.txt, aiopg, sqlalchemy
 py==1.8.1                 # via pytest
 pylint==2.5.0             # via -r requirements/_test.in
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via -r requirements/_base.txt, jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
-pytest-cov==2.8.1         # via -r requirements/_test.in
+pytest-cov==2.9.0         # via -r requirements/_test.in
 pytest-docker==0.7.2      # via -r requirements/_test.in
 pytest-instafail==0.4.1.post0  # via -r requirements/_test.in
 pytest-mock==3.1.0        # via -r requirements/_test.in
@@ -46,7 +46,7 @@ pytest-sugar==0.9.3       # via -r requirements/_test.in
 pytest==5.4.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar
 pyyaml==5.3.1             # via -r requirements/_base.txt, openapi-spec-validator
 requests==2.23.0          # via coveralls
-six==1.14.0               # via -r requirements/_base.txt, astroid, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pyrsistent, tenacity
+six==1.15.0               # via -r requirements/_base.txt, astroid, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pyrsistent, tenacity
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.txt, aiopg
 strict-rfc3339==0.7       # via -r requirements/_base.txt, openapi-core
 tenacity==6.2.0           # via -r requirements/_base.txt

packages/simcore-sdk/requirements/_base.txt

@@ -19,7 +19,7 @@ networkx==2.4             # via -r requirements/_base.in
 psycopg2-binary==2.8.5    # via -r requirements/_base.in, aiopg, sqlalchemy
 pydantic==1.5.1           # via -r requirements/_base.in
 pyyaml==5.3.1             # via trafaret-config
-six==1.14.0               # via tenacity
+six==1.15.0               # via tenacity
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via aiopg
 tenacity==6.2.0           # via -r requirements/_base.in
 trafaret-config==2.0.2    # via -r requirements/_base.in

packages/simcore-sdk/requirements/_test.txt

@@ -36,7 +36,7 @@ pydantic==1.5.1           # via -r requirements/_base.txt
 pylint==2.5.0             # via -r requirements/_test.in
 pyparsing==2.4.7          # via packaging
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
-pytest-cov==2.8.1         # via -r requirements/_test.in
+pytest-cov==2.9.0         # via -r requirements/_test.in
 pytest-docker==0.7.2      # via -r requirements/_test.in
 pytest-instafail==0.4.1.post0  # via -r requirements/_test.in
 pytest-mock==3.1.0        # via -r requirements/_test.in
@@ -45,7 +45,7 @@ pytest-sugar==0.9.3       # via -r requirements/_test.in
 pytest==5.4.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-mock, pytest-sugar
 pyyaml==5.3.1             # via -r requirements/_base.txt, trafaret-config
 requests==2.23.0          # via -r requirements/_test.in, coveralls, docker
-six==1.14.0               # via -r requirements/_base.txt, astroid, docker, packaging, tenacity, websocket-client
+six==1.15.0               # via -r requirements/_base.txt, astroid, docker, packaging, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.17  # via -r requirements/_base.txt, aiopg
 tenacity==6.2.0           # via -r requirements/_base.txt
 termcolor==1.1.0          # via pytest-sugar

services/storage/src/simcore_service_storage/api/v0/openapi.yaml

@@ -1759,7 +1759,8 @@ paths:
                       example: Dabbling in temporal transitions ...
                     prjOwner:
                       type: string
-                      description: user uuid
+                      format: idn-email
+                      description: user email
                     accessRights:
                       type: object
                       description: object containing the GroupID as key and read/write/execution permissions as value
@@ -1970,7 +1971,8 @@ paths:
                       example: Dabbling in temporal transitions ...
                     prjOwner:
                       type: string
-                      description: user uuid
+                      format: idn-email
+                      description: user email
                     accessRights:
                       type: object
                       description: object containing the GroupID as key and read/write/execution permissions as value
@@ -2191,7 +2193,8 @@ paths:
                     example: Dabbling in temporal transitions ...
                   prjOwner:
                     type: string
-                    description: user uuid
+                    format: idn-email
+                    description: user email
                   accessRights:
                     type: object
                     description: object containing the GroupID as key and read/write/execution permissions as value
@@ -2512,7 +2515,8 @@ components:
           example: Dabbling in temporal transitions ...
         prjOwner:
           type: string
-          description: user uuid
+          format: idn-email
+          description: user email
         accessRights:
           type: object
           description: object containing the GroupID as key and read/write/execution permissions as value

services/storage/src/simcore_service_storage/api/v0/schemas/project-v0.0.1.json

@@ -41,7 +41,8 @@
     },
     "prjOwner": {
       "type": "string",
-      "description": "user uuid"
+      "format": "idn-email",
+      "description": "user email"
     },
     "accessRights": {
       "type": "object",
@@ -106,7 +107,7 @@
               "type": "string"
             },
             "progress": {
-              "type":"number",
+              "type": "number",
               "maximum": 100,
               "minimum": 0,
               "description": "the node progress value"
@@ -158,7 +159,10 @@
                       ],
                       "properties": {
                         "store": {
-                          "type": ["string", "integer"]
+                          "type": [
+                            "string",
+                            "integer"
+                          ]
                         },
                         "dataset": {
                           "type": "string"
@@ -181,7 +185,11 @@
               "patternProperties": {
                 "^[-_a-zA-Z0-9]+$": {
                   "type": "string",
-                  "enum": ["Invisible", "ReadOnly", "ReadAndWrite"],
+                  "enum": [
+                    "Invisible",
+                    "ReadOnly",
+                    "ReadAndWrite"
+                  ],
                   "default": "ReadAndWrite",
                   "examples": [
                     "ReadOnly"
@@ -224,7 +232,10 @@
                       ],
                       "properties": {
                         "store": {
-                          "type": ["string", "integer"]
+                          "type": [
+                            "string",
+                            "integer"
+                          ]
                         },
                         "dataset": {
                           "type": "string"
@@ -258,7 +269,10 @@
               ]
             },
             "parent": {
-              "type": ["string", "null"],
+              "type": [
+                "string",
+                "null"
+              ],
               "pattern": "^\\S+$",
               "description": "Parent's (group-nodes') node ID s.",
               "examples": [
@@ -293,4 +307,4 @@
       }
     }
   }
-}
+}

services/storage/src/simcore_service_storage/datcore.py

@@ -22,14 +22,6 @@
 DatasetMetaDataVec = List[DatasetMetaData]
 
 
-# FIXME: W0611:Unused IOAPI imported from blackfynn.api.transfers
-# from blackfynn.api.transfers import IOAPI
-
-
-# FIXME: W0212:Access to a protected member _api of a client class
-# pylint: disable=W0212
-
-
 def _get_collection_id(
     folder: BaseCollection, _collections: List[str], collection_id: str
 ) -> str:
@@ -53,6 +45,7 @@ def _get_collection_id(
 
 class DatcoreClient(object):
     def __init__(self, api_token=None, api_secret=None, host=None, streaming_host=None):
+        # WARNING: contruction raise exception if service is not available. Use datacore_wrapper for safe calls
         self.client = Blackfynn(
             profile=None,
             api_token=api_token,
@@ -310,8 +303,9 @@ def upload_file(self, destination: str, filepath: str, meta_data=None):
         if collection is None:
             return False
 
-        files = [filepath]
-        # pylint: disable = E1101
+        files = [
+            filepath,
+        ]
         self.client._api.io.upload_files(collection, files, display_progress=True)
         collection.update()
 
@@ -349,10 +343,9 @@ def download_file(self, source, filename, destination_path):
             destination__apth (str): Path on host for storing file
         """
 
-        # pylint: disable = E1101
         url = self.download_link(source, filename)
         if url:
-            _file = urllib.URLopener()
+            _file = urllib.URLopener() # nosec
             _file.retrieve(url, destination_path)
             return True
         return False
@@ -362,7 +355,7 @@ def download_link(self, destination, filename):
             returns presigned url for download, destination is a dataset or collection
         """
         collection, collection_id = self._collection_from_destination(destination)
-        # pylint: disable = E1101
+
         for item in collection:
             if isinstance(item, DataPackage):
                 if Path(item.files[0].as_dict()["content"]["s3key"]).name == filename: