Try to fix API and integration workflows

.github/workflows/api-and-integration-tests.yml

@@ -20,7 +20,6 @@ jobs:
             )
        }}
     steps:
-
     - uses: actions/checkout@v4
 
     - name: Set up Python
@@ -70,14 +69,15 @@ jobs:
       run: |
         until $(curl -k --output /dev/null --silent --head --fail https://portal.yoda:8443 ); do printf '.' ; sleep 1; done
         docker exec provider.yoda sh -c 'while ! pgrep irodsServer > /dev/null  ; do echo Waiting for iRODS to start ...  ; sleep 1; done'
+        docker exec provider.yoda sh -c 'set -x ; sudo /usr/sbin/rsyslogd -n -f /etc/rsyslog.conf'
 
     - name: Pull and install latest version of ruleset
       shell: bash
       run: |
         cd yoda/docker/compose
         docker exec provider.yoda sh -c 'set -x ; cd /etc/irods/yoda-ruleset && sudo chown irods:irods -R /etc/irods/yoda-ruleset && sudo -u irods git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && sudo -u irods git pull && sudo -u irods git status'
-        docker exec provider.yoda sh -c "set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git checkout ${{ steps.extract_branch.outputs.branch }} && sudo -u irods python -m pip --no-cache-dir install --user -r /etc/irods/yoda-ruleset/requirements.txt && sudo -u irods make && sudo -u irods make install"
-        docker exec provider.yoda sh -c "set -x ; sudo -u irods /var/lib/irods/irodsctl restart"
+        docker exec provider.yoda sh -c "set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git checkout ${{ steps.extract_branch.outputs.branch }} && sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages -r /etc/irods/yoda-ruleset/requirements.txt && sudo -u irods make && sudo -u irods make install"
+        docker exec provider.yoda sh -c 'set -x ; sudo -u irods /var/lib/irods/irodsctl restart'
 
     - name: Pull and install latest version of portal
       shell: bash
@@ -122,8 +122,7 @@ jobs:
     - name: Output rodsLogs
       if: failure()
       run: |
-        docker exec provider.yoda sh -c 'set -x ; cat /var/lib/irods/log/rodsLog*'
-
+        docker exec provider.yoda sh -c 'set -x ; tail -f /var/log/syslog'
 
     - name: Output web server logs
       if: failure()

.github/workflows/api-documentation.yml

@@ -14,21 +14,14 @@ on:
 
 jobs:
   build:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python-version: [2.7]
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Python
-        # setup-python stopped supporting Python 2.7, use https://github.com/MatteoH2O1999/setup-python
-        uses: MatteoH2O1999/[email protected]
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
-          allow-build: info
-          cache-build: true
+          python-version: 3.12
 
       - name: Install dependencies
         run: |

setup.cfg

@@ -4,7 +4,7 @@ import-order-style=smarkets
 strictness=short
 docstring_style=sphinx
 max-line-length=127
-exclude=__init__.py,tools,tests
+exclude=__init__.py,tools,tests/env/
 application-import-names=avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils
 
 [mypy]

tests/conftest.py

@@ -197,7 +197,7 @@ def login(user, password):
     p = re.compile("tokenValue: '([a-zA-Z0-9._-]*)'")
     found_csrf_tokens = p.findall(content)
     if len(found_csrf_tokens) == 0:
-        print(f"Error: could not find login CSRF token in response from server for login of user {user}. Response was:")
+        print(f"Error: could not find login CSRF token in response from server for login of user {user}. Response was: ")
         print(content)
         sys.exit(1)
     csrf = found_csrf_tokens[0]
@@ -215,7 +215,7 @@ def login(user, password):
     p = re.compile("tokenValue: '([a-zA-Z0-9._-]*)'")
     found_csrf_tokens = p.findall(content)
     if len(found_csrf_tokens) == 0:
-        print(f"Error: could not find authenticated CSRF token in response from server for login of user {user}. Response was:")
+        print(f"Error: could not find authenticated CSRF token in response from server for login of user {user}. Response was: ")
         print(content)
         sys.exit(1)
     csrf = found_csrf_tokens[0]

tools/api/generate-openapi.py

@@ -12,24 +12,21 @@
 This module imports (and therefore executes) ruleset code.
 Do not run it on untrusted codebases.
 """
-from __future__ import print_function
 
 __copyright__ = 'Copyright (c) 2020-2024, Utrecht University'
 __license__   = 'GPLv3, see LICENSE'
 
-__author__    =  ('Chris Smeele')
-__author__    =  ('Lazlo Westerhof')
+__author__    = ('Chris Smeele')
+__author__    = ('Lazlo Westerhof')
 # (in alphabetical order)
 
-import sys
-import re
+import argparse
 import inspect
 import json
-
-from importlib import import_module
+import re
+import sys
 from collections import OrderedDict
-
-import argparse
+from importlib import import_module
 
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument('ruleset', metavar='RULESET', type=str,
@@ -44,6 +41,7 @@
 core = args.core
 module = args.module
 
+
 # Strategy: Import the requested ruleset with an instrumented environment, and
 # apply introspection to extract API function information.
 
@@ -55,9 +53,15 @@ class Sandbag(object):
     Used as a stub for various internal irods modules so that we can import
     rulesets without errors.
     """
-    def __init__(self, *_, **kw): self._data = kw
-    def __call__(self, *_, **__): return Sandbag()
-    def __getattr__(self, k):     return self._data.get(k, Sandbag())
+    def __init__(self, *_, **kw):
+        self._data = kw
+
+    def __call__(self, *_, **__):
+        return Sandbag()
+
+    def __getattr__(self, k):
+        return self._data.get(k, Sandbag())
+
     def __setattr__(self, k, v):
         if k == '_data':
             return super(Sandbag, self).__setattr__(k, v)
@@ -80,6 +84,7 @@ def f(g):
             return g
         return f
 
+
 # Inject iRODS modules.
 sys.modules['irods_types']       = Sandbag()
 sys.modules['genquery']          = Sandbag()

util/user.py

@@ -123,7 +123,7 @@ def number_of_connections(ctx: 'rule.Context') -> int:
     try:
         # We don't use the -a option with the ips command, because this takes
         # significantly more time, which would significantly reduce performance.
-        ips = subprocess.check_output(["ips"])
+        ips = (subprocess.check_output(["ips"])).decode("utf-8")
         username = session_vars.get_map(ctx.rei)['client_user']['user_name']
         connections = ips.count(username)
     except Exception as e: