dependencies: fix security issues

* Bumps dependencies. * Better verbose prints. Co-Authored-by: Peter Weber <[email protected]>
rero · Dec 20, 2023 · 1bfc55f · 1bfc55f
1 parent 4a75ca7
commit 1bfc55f
Show file tree

Hide file tree

Showing 7 changed files with 530 additions and 530 deletions.
diff --git a/.github/workflows/continuous-integration-test.yml b/.github/workflows/continuous-integration-test.yml
@@ -16,9 +16,9 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Setup node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: '14'
+        node-version: '16'
 
     - name: Docker compose up
       run: docker-compose up -d

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -21,12 +21,7 @@ python = ">= 3.9, <3.10"
 ## foo = ">3.0" # finally will upgrade to 5.0
 ## when the python code will be executed a conflict exception will be raised.
 #------------------------------------------------------------------------------
-# Default from Invenio
-lxml = ">=4.9.1"
-marshmallow = ">=3.0.0,<4.0.0"
-uwsgi = ">=2.0"
-uwsgi-tools = ">=1.1.1"
-uwsgitop = ">=0.11"
+
 ## Third party inven## Third party invenio modules used by RERO EBOOKS
 invenio-oaiharvester = { git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4" }
 invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
@@ -38,32 +33,41 @@ invenio-cache = ">=1.1.1,<1.2.0"
 invenio-celery = ">=1.2.4,<1.3.0"
 invenio-config = ">=1.0.3,<1.1.0"
 invenio-i18n = ">=2.0.0,<3.0.0"
-invenio-db = {version = ">=1.0.14,<1.1.0", extras = ["postgresql"]}
+invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
 # Invenio base bundle
 invenio-admin = ">=1.4.0,<1.5.0"
 invenio-assets = ">=2.0.0,<3.0.0"
-invenio-formatter = ">=1.2.0,<1.3.0"
-invenio-logging = {version = ">=1.3.2,<1.4.0", extras = ["sentry-sdk"]}
-invenio-mail = ">=1.0.2,<1.1.0"
-invenio-rest = ">=1.2.8,<1.3.0"
+invenio-formatter = ">=2.0.0,<3.0.0"
+invenio-logging = {version = ">=2.0.0,<3.0.0"}
+invenio-mail = ">=2.0.0,<3.0.0"
+invenio-rest = ">=1.3.0,<1.4.0"
 invenio-theme = ">=2.0.0,<3.0.0"
 # Invenio auth bundle
-invenio-access = ">=1.4.4,<1.5.0"
-invenio-accounts = ">=2.1.0,<2.2.0"
-invenio-oauth2server = ">=2.0.0,<2.1.0"
-invenio-oauthclient = ">=2.2.0,<3.0.0"
-invenio-userprofiles = ">=2.2.0,<2.3.0"
+invenio-access = ">=2.0.0,<3.0.0"
+invenio-accounts = ">=3.0.0,<4.0.0"
+invenio-oauth2server = ">=2.0.0,<3.0.0"
+invenio-oauthclient = ">=3.0.0,<4.0.0"
+invenio-userprofiles = ">=2.3.0,<3.0.0"
 # Invenio metadata bundle
-invenio-indexer = ">=2.1.0,<2.2.0"
+invenio-indexer = ">=2.2.0,<3.0.0"
 invenio-jsonschemas = ">=1.1.4,<1.2.0"
 invenio-oaiserver = ">=2.2.0,<2.3.0"
 invenio-pidstore = ">=1.3.0,<1.4.0"
 invenio-records-rest = ">=2.2.0,<2.3.0"
 invenio-records-ui = ">=1.2.0,<1.3.0"
-invenio-records = "2.1.0,<2.2.0"
+invenio-records = ">=2.1.0,<2.2.0"
 #invenio-search-ui = ">=2.4.0,<3.0.0"
 # Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
+
+# Default from Invenio
+lxml = ">=4.9.1"
+marshmallow = ">=3.0.0,<4.0.0"
+uwsgi = ">=2.0"
+uwsgi-tools = ">=1.1.1"
+uwsgitop = ">=0.11"
+# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
 Flask = ">=2.2.0,<2.3.0"
+sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
 dojson = ">=1.4.0"
 # TODO: dojson problem = AttributeError: 'Group' object has no attribute 'resultcallback'
 click = "<8.1.0"
@@ -80,7 +84,8 @@ pydocstyle = ">=6.1.1"
 ## RERO ILS specific python modules
 poethepoet = "*"
 # TODO: solve requires invenio-indexer (<2.0.0)
-rero-invenio-base = ">=0.2.1"
+# rero-invenio-base = ">=0.2.1"
+rero-invenio-base = { git = "https://github.com/rero/rero-invenio-base.git", branch = "master" }
 sqlitedict = "^2.1.0"
 werkzeug = "<2.3.0"
 sqlalchemy-continuum = ">=1.3.12,<1.4.0"

diff --git a/rero_mef/agents/cli.py b/rero_mef/agents/cli.py
@@ -69,26 +69,28 @@ def get_pids_from_json(json_file):
 
     if missing:
         missing_pids, non_existing_pids = AgentMefRecord. \
-            get_all_missing_viaf_pids(
-                verbose=progress or verbose
-            )
+            get_all_missing_viaf_pids(verbose=(progress or verbose))
         progress_bar = progressbar(
             items=missing_pids,
             length=len(missing_pids),
-            verbose=progress
+            verbose=progress,
+            label='VIAF missing'
         )
     elif viaf_file:
         progress_bar = progressbar(
             items=get_pids_from_json(viaf_file),
             length=number_records_in_file(viaf_file.name, 'json'),
-            verbose=progress
+            verbose=progress,
+            label='VIAF file'
         )
     else:
         progress_bar = progressbar(
             items=AgentViafRecord.get_all_pids(),
             length=counts['viaf']['old'],
-            verbose=progress
+            verbose=progress,
+            label='VIAF all'
         )
+    click.echo('Create MEF and agents from VIAF')
     for pid in progress_bar:
         if enqueue:
             task = task_create_mef_and_agents_from_viaf.delay(

diff --git a/rero_mef/agents/mef/api.py b/rero_mef/agents/mef/api.py
@@ -83,7 +83,8 @@ def get_all_missing_viaf_pids(cls, verbose=False):
         progress = progressbar(
             items=AgentViafRecord.get_all_pids(),
             length=AgentViafRecord.count(),
-            verbose=verbose
+            verbose=verbose,
+            label='VIAF all'
         )
         missing_pids = {pid: 1 for pid in progress}
         if verbose:
@@ -92,7 +93,8 @@ def get_all_missing_viaf_pids(cls, verbose=False):
         progress = progressbar(
             items=query.source(['pid', 'viaf_pid']).scan(),
             length=query.count(),
-            verbose=True
+            verbose=verbose,
+            label='VIAF from MEF'
         )
         non_existing_pids = {hit.pid: hit.viaf_pid for hit in progress
                              if not missing_pids.pop(hit.viaf_pid, None)}

diff --git a/rero_mef/agents/viaf/api.py b/rero_mef/agents/viaf/api.py
@@ -32,8 +32,6 @@
 from .. import AgentGndRecord, AgentIdrefRecord, AgentMefRecord, \
     AgentReroRecord
 from ..api import Action, ReroIndexer, ReroMefRecord
-from ..mef.api import AgentMefRecord
-from ..utils import get_entity_class
 from ...filter import exists_filter
 from ...utils import add_md5, get_entity_class, progressbar, \
     requests_retry_session
@@ -209,7 +207,7 @@ def update_online(agent_class, pid, online):
             if agent_class.provider.pid_type in online:
                 data, msg = agent_class.get_online_record(id_=pid)
                 if online_verbose:
-                    click.echo(msg)
+                    click.echo(f'\n{msg}')
                 if data and not data.get('NO TRANSFORMATION'):
                     agent_record, action = agent_class.create_or_update(
                         data=data, dbcommit=dbcommit, reindex=reindex)
@@ -394,7 +392,6 @@ def update_online(self, dbcommit=False, reindex=False):
         :param reindex: Reindex record.
         :returns: record and actions message.
         """
-        from rero_mef.api import Action
         online_data, _ = self.get_online_record(
             viaf_source_code='VIAF',
             pid=self.pid
@@ -491,7 +488,7 @@ def delete(self, force=True, dbcommit=False, delindex=False):
             AgentMefRecord.flush_indexes()
         # recreate MEF records for agents
         for agent_record in old_agent_records.values():
-            mef, test = agent_record.create_or_update_mef(
+            mef, _ = agent_record.create_or_update_mef(
                 dbcommit=True,
                 reindex=True
             )

diff --git a/scripts/test b/scripts/test
@@ -69,16 +69,26 @@ set -e
 #    Vulnerability ID: 45183
 # -> Vulnerability found in flask-security version 3.0.0
 #    Vulnerability ID: 44501
-# -> Vulnerability found in sqlalchemy version 1.4.48
+#    CVE-2021-23385, an open redirect vulnerability: When using the...
+# -> Vulnerability found in sqlalchemy version 1.4.50
 #    Vulnerability ID: 51668
 # -> Vulnerability found in sqlalchemy-utils version 0.38.3
 #    Vulnerability ID: 42194
 # -> Vulnerability found in wtforms version 2.3.3
 #    Vulnerability ID: 42852
+# -> Vulnerability found in werkzeug version 2.2.3
+#    Vulnerability ID: 62019
+# -> Vulnerability found in pip version 23.2.1
+#    Vulnerability ID: 62044
 # -> Vulnerability found in py version 1.11.0
 #    Vulnerability ID: 51457
-info_msg "Test safety:"
-safety check -o bare -i 40459 -i 45183 -i 44501 -i 51668 -i 42194 -i 42852 -i 51457
+info_msg "Check vulnerabilities:"
+safety_exceptions="-i 40459 -i 45183 -i 44501 -i 51668 -i 42194 -i 42852 -i 62019 -i 62044 -i 51457"
+msg=$(safety check -o text ${safety_exceptions}) || {
+  echo "Safety vulnerabilites found for packages:" $(safety check -o bare ${safety_exceptions})
+  echo "Run:" "safety check -o screen ${safety_exceptions} | grep -i vulnerability" "for more details"
+  exit 1
+}
 info_msg "Test pydocstyle:"
 pydocstyle rero_mef tests docs
 info_msg "Test isort:"