diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 9e1e41d..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Lint" - -on: [ push, pull_request ] - -jobs: - - flake8: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install flake8 - - run: flake8 - - mypy: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install mypy - - run: mypy . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25da05e..f29d458 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,67 +6,10 @@ on: jobs: server: - runs-on: ubuntu-latest - services: - redis: - image: redis:alpine - ports: - - "6379:6379" - - mongo: - image: mongo:3.6 - ports: - - "27017:27017" - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.23 - ports: - - "9200:9200" - env: - discovery.type: single-node - - defaults: - run: - working-directory: server - - steps: - - uses: actions/checkout@v2 - - - name: setup python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - run: sudo apt-get update && sudo apt-get -y install libxml2-dev libxmlsec1-dev libxmlsec1-openssl - - - name: pip install - run: | - python -m pip install --upgrade pip wheel setuptools - pip install -r dev-requirements.txt - - - name: pytest - run: pytest --disable-pytest-warnings --cov=./ - - - name: behave - run: behave --format progress2 --logging-level=ERROR + uses: superdesk/newsroom-app/.github/workflows/server.yml@develop + pytest: + uses: superdesk/newsroom-app/.github/workflows/pytest.yml@develop client: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: client - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-node@v1 - with: - node-version: '14.x' - - - name: install - run: npm install - - - name: build - run: npm run build + uses: superdesk/newsroom-app/.github/workflows/client.yml@develop diff --git a/.gitignore b/.gitignore index e9ccc95..be364a8 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,5 @@ node_modules # data dumps dump + +server/saml diff --git a/server/dev-requirements.txt b/server/dev-requirements.txt index e7025d1..2c76336 100644 --- a/server/dev-requirements.txt +++ b/server/dev-requirements.txt @@ -1,11 +1,12 @@ -r requirements.txt -flake8 +black==24.10.0 +flake8==7.1.1 pytest==8.3.4 pytest-cov==5.0.0 pytest-mock==3.14.0 -responses>=0.10.6,<0.26 -httmock -wooper -mypy -typing_extensions>=3.7.4 +responses==0.25.3 +httmock==1.4.0 +httmock==1.4.0 +mypy==1.13.0 +typing_extensions==4.12.2 diff --git a/server/features/environment.py b/server/features/environment.py index d1bb862..30652f9 100644 --- a/server/features/environment.py +++ b/server/features/environment.py @@ -11,32 +11,32 @@ def get_app(*args, **kwargs): def before_all(context): config = { - 'BEHAVE': True, - 'CORE_APPS': CORE_APPS, - 'INSTALLED_APPS': [], - 'ELASTICSEARCH_FORCE_REFRESH': True, - 'NEWS_API_ENABLED': True, - 'NEWS_API_TIME_LIMIT_DAYS': 100, - 'NEWS_API_BEHAVE_TESTS': True, - 'CACHE_TYPE': "null", + "BEHAVE": True, + "CORE_APPS": CORE_APPS, + "INSTALLED_APPS": [], + "ELASTICSEARCH_FORCE_REFRESH": True, + "NEWS_API_ENABLED": True, + "NEWS_API_TIME_LIMIT_DAYS": 100, + "NEWS_API_BEHAVE_TESTS": True, + "CACHE_TYPE": "null", } setup_before_all(context, config, app_factory=get_app) def before_scenario(context, scenario): config = { - 'BEHAVE': True, - 'CORE_APPS': CORE_APPS, - 'INSTALLED_APPS': [], - 'ELASTICSEARCH_FORCE_REFRESH': True, - 'NEWS_API_ENABLED': True, - 'NEWS_API_TIME_LIMIT_DAYS': 100, - 'NEWS_API_BEHAVE_TESTS': True, - 'CACHE_TYPE': "null", + "BEHAVE": True, + "CORE_APPS": CORE_APPS, + "INSTALLED_APPS": [], + "ELASTICSEARCH_FORCE_REFRESH": True, + "NEWS_API_ENABLED": True, + "NEWS_API_TIME_LIMIT_DAYS": 100, + "NEWS_API_BEHAVE_TESTS": True, + "CACHE_TYPE": "null", } - if 'rate_limit' in scenario.tags: - config['RATE_LIMIT_PERIOD'] = 300 # 5 minutes - config['RATE_LIMIT_REQUESTS'] = 2 + if "rate_limit" in scenario.tags: + config["RATE_LIMIT_PERIOD"] = 300 # 5 minutes + config["RATE_LIMIT_REQUESTS"] = 2 setup_before_scenario(context, scenario, config, app_factory=get_app) diff --git a/server/features/steps/steps.py b/server/features/steps/steps.py index 8cae8db..be4729d 100644 --- a/server/features/steps/steps.py +++ b/server/features/steps/steps.py @@ -1 +1 @@ -from newsroom.tests.steps import * # noqa +from newsroom.tests.steps import * # noqa diff --git a/server/gunicorn_config.py b/server/gunicorn_config.py index 21f2d15..f114b14 100644 --- a/server/gunicorn_config.py +++ b/server/gunicorn_config.py @@ -8,5 +8,5 @@ accesslog = "-" access_log_format = "%(m)s %(U)s status=%(s)s time=%(T)ss size=%(B)sb" -reload = 'NEWSROOM_RELOAD' in os.environ -timeout = int(os.environ.get('WEB_TIMEOUT', 30)) +reload = "NEWSROOM_RELOAD" in os.environ +timeout = int(os.environ.get("WEB_TIMEOUT", 30)) diff --git a/server/manage.py b/server/manage.py index a6ccc73..429e0aa 100644 --- a/server/manage.py +++ b/server/manage.py @@ -1,4 +1,4 @@ -from newsroom.commands import * # noqa +from newsroom.commands import * # noqa from newsroom.commands.manager import manager diff --git a/server/requirements.txt b/server/requirements.txt index 62e67a0..ac6ed32 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -35,9 +35,9 @@ blinker==1.8.2 # raven # sentry-sdk # superdesk-core -boto3==1.35.67 +boto3==1.35.72 # via superdesk-core -botocore==1.35.67 +botocore==1.35.72 # via # boto3 # s3transfer @@ -87,7 +87,7 @@ click-repl==0.3.0 # via celery croniter==3.0.4 # via superdesk-core -cryptography==43.0.3 +cryptography==44.0.0 # via # authlib # jwcrypto @@ -181,8 +181,6 @@ idna==3.10 # via # email-validator # requests -importlib-resources==6.4.5 - # via limits isodate==0.7.2 # via python3-saml itsdangerous==1.1.0 @@ -210,7 +208,7 @@ kombu==5.4.2 # superdesk-core ldap3==2.9.1 # via superdesk-core -limits==3.13.0 +limits==3.14.1 # via flask-limiter lxml==5.2.2 # via @@ -352,7 +350,7 @@ rsa==4.9 # oauth2client s3transfer==0.10.4 # via boto3 -sentry-sdk[flask]==2.18.0 +sentry-sdk[flask]==2.19.0 # via newsroom-core sgmllib3k==1.0.0 # via feedparser diff --git a/server/settings_newsapi.py b/server/settings_newsapi.py index b7bee90..5045ae2 100644 --- a/server/settings_newsapi.py +++ b/server/settings_newsapi.py @@ -1,6 +1,6 @@ from pathlib import Path -from superdesk.default_settings import env, strtobool # noqa +from superdesk.default_settings import env, strtobool # noqa from newsroom.news_api.default_settings import CORE_APPS, INSTALLED_APPS, BLUEPRINTS diff --git a/server/setup.cfg b/server/setup.cfg index 4a76eea..2e43291 100644 --- a/server/setup.cfg +++ b/server/setup.cfg @@ -8,7 +8,7 @@ confcutdir = tests addopts = --tb=short [mypy] -python_version = 3.8 +python_version = 3.10 allow_untyped_globals = True ignore_missing_imports = True diff --git a/server/setup.py b/server/setup.py index 72856f0..a05734d 100644 --- a/server/setup.py +++ b/server/setup.py @@ -2,6 +2,6 @@ setup( - name='Newsroom App STT', + name="Newsroom App STT", packages=find_packages(), ) diff --git a/server/stt/external_links.py b/server/stt/external_links.py index d5dff82..5c26f97 100644 --- a/server/stt/external_links.py +++ b/server/stt/external_links.py @@ -1,4 +1,5 @@ - def init_app(app): - app.sidenav('Kuvakauppa', url='http://kuvakauppa.lehtikuva.fi', icon='photo', group=8) - app.sidenav('STT Info', url='https://www.sttinfo.fi', icon='info', group=8) + app.sidenav( + "Kuvakauppa", url="http://kuvakauppa.lehtikuva.fi", icon="photo", group=8 + ) + app.sidenav("STT Info", url="https://www.sttinfo.fi", icon="info", group=8) diff --git a/server/stt/filters.py b/server/stt/filters.py index c77efe5..ff6c5dd 100644 --- a/server/stt/filters.py +++ b/server/stt/filters.py @@ -13,57 +13,63 @@ def get_previous_version(app, guid, version): for i in range(int(version) - 1, 1, -1): id = "{}:{}".format(guid, i) - original = app.data.find_one('items', req=None, _id=id) + original = app.data.find_one("items", req=None, _id=id) if original: return original - return app.data.find_one('items', req=None, _id=guid) + return app.data.find_one("items", req=None, _id=guid) def on_publish_item(app, item, is_new, **kwargs): """Populate stt department and version fields.""" - if item.get('subject'): - for subject in item['subject']: - if subject.get('scheme', '') in STT_FIELDS: - item[subject['scheme']] = subject.get('name', subject.get('code')) - item['subject'] = [subject for subject in item['subject'] if subject.get('scheme') != 'sttdone1'] + if item.get("subject"): + for subject in item["subject"]: + if subject.get("scheme", "") in STT_FIELDS: + item[subject["scheme"]] = subject.get("name", subject.get("code")) + item["subject"] = [ + subject + for subject in item["subject"] + if subject.get("scheme") != "sttdone1" + ] # add private note to ednote - if item.get('extra', {}).get('sttnote_private'): - if item.get('ednote'): - item['ednote'] = '{}\n{}'.format(item['ednote'], item['extra']['sttnote_private']) + if item.get("extra", {}).get("sttnote_private"): + if item.get("ednote"): + item["ednote"] = "{}\n{}".format( + item["ednote"], item["extra"]["sttnote_private"] + ) else: - item['ednote'] = item['extra']['sttnote_private'] + item["ednote"] = item["extra"]["sttnote_private"] # set versioncreated for archive items - if item.get('firstpublished') and is_new: - if isinstance(item.get('firstpublished'), str): - firstpublished = parse_date(item['firstpublished']) + if item.get("firstpublished") and is_new: + if isinstance(item.get("firstpublished"), str): + firstpublished = parse_date(item["firstpublished"]) else: - firstpublished = item['firstpublished'] + firstpublished = item["firstpublished"] - if firstpublished < item['versioncreated']: - item['versioncreated'] = firstpublished + if firstpublished < item["versioncreated"]: + item["versioncreated"] = firstpublished # link the previous versions and update the id of the story - if not is_new and 'evolvedfrom' not in item: - original = get_previous_version(app, item['guid'], item['version']) + if not is_new and "evolvedfrom" not in item: + original = get_previous_version(app, item["guid"], item["version"]) if original: - if original.get('version') == item['version']: + if original.get("version") == item["version"]: # the same version of the story been sent again so no need to create new version return - service = superdesk.get_resource_service('content_api') - new_id = '{}:{}'.format(item['guid'], item['version']) - service.system_update(original['_id'], {'nextversion': new_id}, original) - item['guid'] = new_id - item['ancestors'] = copy(original.get('ancestors', [])) - item['ancestors'].append(original['_id']) - item['bookmarks'] = original.get('bookmarks', []) + service = superdesk.get_resource_service("content_api") + new_id = "{}:{}".format(item["guid"], item["version"]) + service.system_update(original["_id"], {"nextversion": new_id}, original) + item["guid"] = new_id + item["ancestors"] = copy(original.get("ancestors", [])) + item["ancestors"].append(original["_id"]) + item["bookmarks"] = original.get("bookmarks", []) # dump abstract - for field in ('description_html', 'description_text'): + for field in ("description_html", "description_text"): item.pop(field, None) @@ -72,11 +78,15 @@ def init_app(app): # add extra fields to elastic mapping for field in STT_ROOT_FIELDS: - for resource in ('items', 'content_api'): - app.config['DOMAIN'][resource]['schema'].update({ - field: {'type': 'string', 'mapping': not_analyzed}, - }) - - app.config['SOURCES'][resource]['projection'].update({ - field: 1, - }) + for resource in ("items", "content_api"): + app.config["DOMAIN"][resource]["schema"].update( + { + field: {"type": "string", "mapping": not_analyzed}, + } + ) + + app.config["SOURCES"][resource]["projection"].update( + { + field: 1, + } + ) diff --git a/server/stt/signals.py b/server/stt/signals.py index 1e281fc..932bc40 100644 --- a/server/stt/signals.py +++ b/server/stt/signals.py @@ -1,4 +1,3 @@ - from newsroom.signals import publish_planning diff --git a/server/tests/fixtures.py b/server/tests/fixtures.py index 459946f..3dff8a5 100644 --- a/server/tests/fixtures.py +++ b/server/tests/fixtures.py @@ -10,5 +10,5 @@ init_agenda_items, init_auth, setup_user_company, - init_company + init_company, ) diff --git a/server/tests/test_signals.py b/server/tests/test_signals.py index f80007e..0a47a9d 100644 --- a/server/tests/test_signals.py +++ b/server/tests/test_signals.py @@ -1,4 +1,3 @@ - from stt.signals import publish_planning, set_planning_all_day diff --git a/server/tests/test_stt_filters.py b/server/tests/test_stt_filters.py index cced902..ab1f95c 100644 --- a/server/tests/test_stt_filters.py +++ b/server/tests/test_stt_filters.py @@ -1,4 +1,3 @@ - import bson import hmac from flask import json @@ -6,113 +5,116 @@ def get_signature_headers(data, key): - mac = hmac.new(key, data.encode(), 'sha1') - return {'x-superdesk-signature': 'sha1=%s' % mac.hexdigest()} + mac = hmac.new(key, data.encode(), "sha1") + return {"x-superdesk-signature": "sha1=%s" % mac.hexdigest()} item = { - 'guid': 'foo', - 'type': 'text', - 'headline': 'Foo', - 'firstcreated': '2017-11-27T08:00:57+0000', - 'body_html': '
foo bar
', - 'renditions': { - 'thumbnail': { - 'href': 'http://example.com/foo', - 'media': 'foo', + "guid": "foo", + "type": "text", + "headline": "Foo", + "firstcreated": "2017-11-27T08:00:57+0000", + "body_html": "foo bar
", + "renditions": { + "thumbnail": { + "href": "http://example.com/foo", + "media": "foo", } }, - 'genre': [{'name': 'News', 'code': 'news'}], - 'associations': { - 'featured': { - 'type': 'picture', - 'renditions': { - 'thumbnail': { - 'href': 'http://example.com/bar', - 'media': 'bar', + "genre": [{"name": "News", "code": "news"}], + "associations": { + "featured": { + "type": "picture", + "renditions": { + "thumbnail": { + "href": "http://example.com/bar", + "media": "bar", } - } + }, } }, - 'event_id': 'urn:event/1', - 'coverage_id': 'urn:coverage/1', + "event_id": "urn:event/1", + "coverage_id": "urn:coverage/1", } def test_push_updates_ednote(client, app): from stt.filters import init_app + init_app(app) payload = item.copy() - payload['ednote'] = 'foo' - client.post('/push', data=json.dumps(payload), content_type='application/json') - parsed = get_entity_or_404(item['guid'], 'items') - assert parsed['ednote'] == 'foo' + payload["ednote"] = "foo" + client.post("/push", data=json.dumps(payload), content_type="application/json") + parsed = get_entity_or_404(item["guid"], "items") + assert parsed["ednote"] == "foo" - payload['guid'] = 'bar' - payload['extra'] = {'sttnote_private': 'private message'} - client.post('/push', data=json.dumps(payload), content_type='application/json') - parsed = get_entity_or_404(payload['guid'], 'items') - assert parsed['ednote'] == 'foo\nprivate message' + payload["guid"] = "bar" + payload["extra"] = {"sttnote_private": "private message"} + client.post("/push", data=json.dumps(payload), content_type="application/json") + parsed = get_entity_or_404(payload["guid"], "items") + assert parsed["ednote"] == "foo\nprivate message" - payload['guid'] = 'baz' - payload.pop('ednote') - payload['extra'] = {'sttnote_private': 'private message'} - client.post('/push', data=json.dumps(payload), content_type='application/json') - parsed = get_entity_or_404(payload['guid'], 'items') - assert parsed['ednote'] == 'private message' + payload["guid"] = "baz" + payload.pop("ednote") + payload["extra"] = {"sttnote_private": "private message"} + client.post("/push", data=json.dumps(payload), content_type="application/json") + parsed = get_entity_or_404(payload["guid"], "items") + assert parsed["ednote"] == "private message" def test_push_firstcreated_is_older_copies_to_versioncreated(client, app): from stt.filters import init_app + init_app(app) payload = item.copy() - payload['firstpublished'] = '2017-11-26T08:00:57+0000' - payload['versioncreated'] = '2017-11-27T08:00:57+0000' - payload['version'] = '1' - client.post('/push', data=json.dumps(payload), content_type='application/json') - parsed = get_entity_or_404(item['guid'], 'items') - assert parsed['firstpublished'] == parsed['versioncreated'] + payload["firstpublished"] = "2017-11-26T08:00:57+0000" + payload["versioncreated"] = "2017-11-27T08:00:57+0000" + payload["version"] = "1" + client.post("/push", data=json.dumps(payload), content_type="application/json") + parsed = get_entity_or_404(item["guid"], "items") + assert parsed["firstpublished"] == parsed["versioncreated"] # post the same story again as a correction, versioncreated is preserved - payload['versioncreated'] = '2017-11-28T08:00:57+0000' - client.post('/push', data=json.dumps(payload), content_type='application/json') - parsed = get_entity_or_404(item['guid'], 'items') - assert parsed['firstpublished'].strftime('%Y%m%d%H%M') == '201711260800' - assert parsed['versioncreated'].strftime('%Y%m%d%H%M') == '201711280800' + payload["versioncreated"] = "2017-11-28T08:00:57+0000" + client.post("/push", data=json.dumps(payload), content_type="application/json") + parsed = get_entity_or_404(item["guid"], "items") + assert parsed["firstpublished"].strftime("%Y%m%d%H%M") == "201711260800" + assert parsed["versioncreated"].strftime("%Y%m%d%H%M") == "201711280800" def test_push_new_versions_will_update_ancestors(client, app): from stt.filters import init_app + init_app(app) payload = item.copy() - payload['version'] = '1' - client.post('/push', data=json.dumps(payload), content_type='application/json') - parsed = get_entity_or_404(item['guid'], 'items') - assert parsed['version'] == '1' + payload["version"] = "1" + client.post("/push", data=json.dumps(payload), content_type="application/json") + parsed = get_entity_or_404(item["guid"], "items") + assert parsed["version"] == "1" bookmarks = [str(bson.ObjectId())] - saved = get_entity_or_404(item["guid"], 'items') - app.data.update("items", saved['_id'], {"bookmarks": bookmarks}, saved) + saved = get_entity_or_404(item["guid"], "items") + app.data.update("items", saved["_id"], {"bookmarks": bookmarks}, saved) # post the new version of the story, it will update the ancestors - payload['version'] = '2' - payload['headline'] = 'bar' - client.post('/push', data=json.dumps(payload), content_type='application/json') - new_story = get_entity_or_404('foo:2', 'items') - original_story = get_entity_or_404(item['guid'], 'items') - assert new_story['version'] == '2' - assert new_story['ancestors'] == ['foo'] - assert new_story['headline'] == 'bar' - assert new_story['bookmarks'] == bookmarks - assert original_story['nextversion'] == 'foo:2' + payload["version"] = "2" + payload["headline"] = "bar" + client.post("/push", data=json.dumps(payload), content_type="application/json") + new_story = get_entity_or_404("foo:2", "items") + original_story = get_entity_or_404(item["guid"], "items") + assert new_story["version"] == "2" + assert new_story["ancestors"] == ["foo"] + assert new_story["headline"] == "bar" + assert new_story["bookmarks"] == bookmarks + assert original_story["nextversion"] == "foo:2" # post the same version of the story, it will update keep ancestors but update the current story - payload['headline'] = 'baz' - client.post('/push', data=json.dumps(payload), content_type='application/json') - new_story = get_entity_or_404('foo:2', 'items') - original_story = get_entity_or_404(item['guid'], 'items') - assert new_story['version'] == '2' - assert new_story['ancestors'] == ['foo'] - assert new_story['headline'] == 'baz' - assert new_story['bookmarks'] == bookmarks - assert original_story['nextversion'] == 'foo:2' + payload["headline"] = "baz" + client.post("/push", data=json.dumps(payload), content_type="application/json") + new_story = get_entity_or_404("foo:2", "items") + original_story = get_entity_or_404(item["guid"], "items") + assert new_story["version"] == "2" + assert new_story["ancestors"] == ["foo"] + assert new_story["headline"] == "baz" + assert new_story["bookmarks"] == bookmarks + assert original_story["nextversion"] == "foo:2"