Merge branch 'develop' into dependabot/pip/server/pytest-8.3.4

.github/workflows/tests.yml

@@ -6,67 +6,10 @@ on:
 jobs:
 
   server:
-    runs-on: ubuntu-latest
-    services:
-      redis:
-          image: redis:alpine
-          ports:
-              - "6379:6379"
-
-      mongo:
-          image: mongo:3.6
-          ports:
-              - "27017:27017"
-
-      elastic:
-          image: docker.elastic.co/elasticsearch/elasticsearch:7.17.23
-          ports:
-              - "9200:9200"
-          env:
-              discovery.type: single-node
-
-    defaults:
-      run:
-        working-directory: server
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: setup python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - run: sudo apt-get update && sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
-
-      - name: pip install
-        run: |
-          python -m pip install --upgrade pip wheel setuptools
-          pip install -r dev-requirements.txt
-
-      - name: pytest
-        run: pytest --disable-pytest-warnings --cov=./
-
-      - name: behave
-        run: behave --format progress2 --logging-level=ERROR
+    uses: superdesk/newsroom-app/.github/workflows/server.yml@develop
 
+  pytest:
+    uses: superdesk/newsroom-app/.github/workflows/pytest.yml@develop
 
   client:
-    runs-on: ubuntu-latest
-
-    defaults:
-      run:
-        working-directory: client
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions/setup-node@v1
-        with:
-          node-version: '14.x'
-
-      - name: install
-        run: npm install
-
-      - name: build
-        run: npm run build
+    uses: superdesk/newsroom-app/.github/workflows/client.yml@develop

.gitignore

@@ -120,3 +120,5 @@ node_modules
 
 # data dumps
 dump
+
+server/saml

server/dev-requirements.txt

@@ -1,11 +1,12 @@
 -r requirements.txt
 
-flake8
+black==24.10.0
+flake8==7.1.1
 pytest==8.3.4
 pytest-cov==5.0.0
 pytest-mock==3.14.0
-responses>=0.10.6,<0.26
-httmock
-wooper
-mypy
-typing_extensions>=3.7.4
+responses==0.25.3
+httmock==1.4.0
+httmock==1.4.0
+mypy==1.13.0
+typing_extensions==4.12.2

server/features/environment.py

@@ -11,32 +11,32 @@ def get_app(*args, **kwargs):
 
 def before_all(context):
     config = {
-        'BEHAVE': True,
-        'CORE_APPS': CORE_APPS,
-        'INSTALLED_APPS': [],
-        'ELASTICSEARCH_FORCE_REFRESH': True,
-        'NEWS_API_ENABLED': True,
-        'NEWS_API_TIME_LIMIT_DAYS': 100,
-        'NEWS_API_BEHAVE_TESTS': True,
-        'CACHE_TYPE': "null",
+        "BEHAVE": True,
+        "CORE_APPS": CORE_APPS,
+        "INSTALLED_APPS": [],
+        "ELASTICSEARCH_FORCE_REFRESH": True,
+        "NEWS_API_ENABLED": True,
+        "NEWS_API_TIME_LIMIT_DAYS": 100,
+        "NEWS_API_BEHAVE_TESTS": True,
+        "CACHE_TYPE": "null",
     }
     setup_before_all(context, config, app_factory=get_app)
 
 
 def before_scenario(context, scenario):
     config = {
-        'BEHAVE': True,
-        'CORE_APPS': CORE_APPS,
-        'INSTALLED_APPS': [],
-        'ELASTICSEARCH_FORCE_REFRESH': True,
-        'NEWS_API_ENABLED': True,
-        'NEWS_API_TIME_LIMIT_DAYS': 100,
-        'NEWS_API_BEHAVE_TESTS': True,
-        'CACHE_TYPE': "null",
+        "BEHAVE": True,
+        "CORE_APPS": CORE_APPS,
+        "INSTALLED_APPS": [],
+        "ELASTICSEARCH_FORCE_REFRESH": True,
+        "NEWS_API_ENABLED": True,
+        "NEWS_API_TIME_LIMIT_DAYS": 100,
+        "NEWS_API_BEHAVE_TESTS": True,
+        "CACHE_TYPE": "null",
     }
 
-    if 'rate_limit' in scenario.tags:
-        config['RATE_LIMIT_PERIOD'] = 300  # 5 minutes
-        config['RATE_LIMIT_REQUESTS'] = 2
+    if "rate_limit" in scenario.tags:
+        config["RATE_LIMIT_PERIOD"] = 300  # 5 minutes
+        config["RATE_LIMIT_REQUESTS"] = 2
 
     setup_before_scenario(context, scenario, config, app_factory=get_app)

server/features/steps/steps.py

@@ -1 +1 @@
-from newsroom.tests.steps import * # noqa
+from newsroom.tests.steps import *  # noqa

server/gunicorn_config.py

@@ -8,5 +8,5 @@
 accesslog = "-"
 access_log_format = "%(m)s %(U)s status=%(s)s time=%(T)ss size=%(B)sb"
 
-reload = 'NEWSROOM_RELOAD' in os.environ
-timeout = int(os.environ.get('WEB_TIMEOUT', 30))
+reload = "NEWSROOM_RELOAD" in os.environ
+timeout = int(os.environ.get("WEB_TIMEOUT", 30))

server/manage.py

@@ -1,4 +1,4 @@
-from newsroom.commands import * # noqa
+from newsroom.commands import *  # noqa
 from newsroom.commands.manager import manager
 
 

server/requirements.txt

@@ -35,9 +35,9 @@ blinker==1.8.2
     #   raven
     #   sentry-sdk
     #   superdesk-core
-boto3==1.35.67
+boto3==1.35.72
     # via superdesk-core
-botocore==1.35.67
+botocore==1.35.72
     # via
     #   boto3
     #   s3transfer
@@ -87,7 +87,7 @@ click-repl==0.3.0
     # via celery
 croniter==3.0.4
     # via superdesk-core
-cryptography==43.0.3
+cryptography==44.0.0
     # via
     #   authlib
     #   jwcrypto
@@ -181,8 +181,6 @@ idna==3.10
     # via
     #   email-validator
     #   requests
-importlib-resources==6.4.5
-    # via limits
 isodate==0.7.2
     # via python3-saml
 itsdangerous==1.1.0
@@ -210,7 +208,7 @@ kombu==5.4.2
     #   superdesk-core
 ldap3==2.9.1
     # via superdesk-core
-limits==3.13.0
+limits==3.14.1
     # via flask-limiter
 lxml==5.2.2
     # via
@@ -352,7 +350,7 @@ rsa==4.9
     #   oauth2client
 s3transfer==0.10.4
     # via boto3
-sentry-sdk[flask]==2.18.0
+sentry-sdk[flask]==2.19.0
     # via newsroom-core
 sgmllib3k==1.0.0
     # via feedparser

server/settings_newsapi.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from superdesk.default_settings import env, strtobool # noqa
+from superdesk.default_settings import env, strtobool  # noqa
 from newsroom.news_api.default_settings import CORE_APPS, INSTALLED_APPS, BLUEPRINTS
 
 

server/setup.cfg

@@ -8,7 +8,7 @@ confcutdir = tests
 addopts = --tb=short
 
 [mypy]
-python_version = 3.8
+python_version = 3.10
 allow_untyped_globals = True
 ignore_missing_imports = True
 

server/setup.py

@@ -2,6 +2,6 @@
 
 
 setup(
-    name='Newsroom App STT',
+    name="Newsroom App STT",
     packages=find_packages(),
 )

server/stt/external_links.py

@@ -1,4 +1,5 @@
-
 def init_app(app):
-    app.sidenav('Kuvakauppa', url='http://kuvakauppa.lehtikuva.fi', icon='photo', group=8)
-    app.sidenav('STT Info', url='https://www.sttinfo.fi', icon='info', group=8)
+    app.sidenav(
+        "Kuvakauppa", url="http://kuvakauppa.lehtikuva.fi", icon="photo", group=8
+    )
+    app.sidenav("STT Info", url="https://www.sttinfo.fi", icon="info", group=8)

server/stt/filters.py

@@ -13,57 +13,63 @@
 def get_previous_version(app, guid, version):
     for i in range(int(version) - 1, 1, -1):
         id = "{}:{}".format(guid, i)
-        original = app.data.find_one('items', req=None, _id=id)
+        original = app.data.find_one("items", req=None, _id=id)
         if original:
             return original
 
-    return app.data.find_one('items', req=None, _id=guid)
+    return app.data.find_one("items", req=None, _id=guid)
 
 
 def on_publish_item(app, item, is_new, **kwargs):
     """Populate stt department and version fields."""
-    if item.get('subject'):
-        for subject in item['subject']:
-            if subject.get('scheme', '') in STT_FIELDS:
-                item[subject['scheme']] = subject.get('name', subject.get('code'))
-        item['subject'] = [subject for subject in item['subject'] if subject.get('scheme') != 'sttdone1']
+    if item.get("subject"):
+        for subject in item["subject"]:
+            if subject.get("scheme", "") in STT_FIELDS:
+                item[subject["scheme"]] = subject.get("name", subject.get("code"))
+        item["subject"] = [
+            subject
+            for subject in item["subject"]
+            if subject.get("scheme") != "sttdone1"
+        ]
 
     # add private note to ednote
-    if item.get('extra', {}).get('sttnote_private'):
-        if item.get('ednote'):
-            item['ednote'] = '{}\n{}'.format(item['ednote'], item['extra']['sttnote_private'])
+    if item.get("extra", {}).get("sttnote_private"):
+        if item.get("ednote"):
+            item["ednote"] = "{}\n{}".format(
+                item["ednote"], item["extra"]["sttnote_private"]
+            )
         else:
-            item['ednote'] = item['extra']['sttnote_private']
+            item["ednote"] = item["extra"]["sttnote_private"]
 
     # set versioncreated for archive items
-    if item.get('firstpublished') and is_new:
-        if isinstance(item.get('firstpublished'), str):
-            firstpublished = parse_date(item['firstpublished'])
+    if item.get("firstpublished") and is_new:
+        if isinstance(item.get("firstpublished"), str):
+            firstpublished = parse_date(item["firstpublished"])
         else:
-            firstpublished = item['firstpublished']
+            firstpublished = item["firstpublished"]
 
-        if firstpublished < item['versioncreated']:
-            item['versioncreated'] = firstpublished
+        if firstpublished < item["versioncreated"]:
+            item["versioncreated"] = firstpublished
 
     # link the previous versions and update the id of the story
-    if not is_new and 'evolvedfrom' not in item:
-        original = get_previous_version(app, item['guid'], item['version'])
+    if not is_new and "evolvedfrom" not in item:
+        original = get_previous_version(app, item["guid"], item["version"])
 
         if original:
-            if original.get('version') == item['version']:
+            if original.get("version") == item["version"]:
                 # the same version of the story been sent again so no need to create new version
                 return
 
-            service = superdesk.get_resource_service('content_api')
-            new_id = '{}:{}'.format(item['guid'], item['version'])
-            service.system_update(original['_id'], {'nextversion': new_id}, original)
-            item['guid'] = new_id
-            item['ancestors'] = copy(original.get('ancestors', []))
-            item['ancestors'].append(original['_id'])
-            item['bookmarks'] = original.get('bookmarks', [])
+            service = superdesk.get_resource_service("content_api")
+            new_id = "{}:{}".format(item["guid"], item["version"])
+            service.system_update(original["_id"], {"nextversion": new_id}, original)
+            item["guid"] = new_id
+            item["ancestors"] = copy(original.get("ancestors", []))
+            item["ancestors"].append(original["_id"])
+            item["bookmarks"] = original.get("bookmarks", [])
 
     # dump abstract
-    for field in ('description_html', 'description_text'):
+    for field in ("description_html", "description_text"):
         item.pop(field, None)
 
 
@@ -72,11 +78,15 @@ def init_app(app):
 
     # add extra fields to elastic mapping
     for field in STT_ROOT_FIELDS:
-        for resource in ('items', 'content_api'):
-            app.config['DOMAIN'][resource]['schema'].update({
-                field: {'type': 'string', 'mapping': not_analyzed},
-            })
-
-            app.config['SOURCES'][resource]['projection'].update({
-                field: 1,
-            })
+        for resource in ("items", "content_api"):
+            app.config["DOMAIN"][resource]["schema"].update(
+                {
+                    field: {"type": "string", "mapping": not_analyzed},
+                }
+            )
+
+            app.config["SOURCES"][resource]["projection"].update(
+                {
+                    field: 1,
+                }
+            )