From 9e38713fdced6e20f31a3f92cabc1e4883fd4b91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Ja=C5=A1ek?= Date: Wed, 13 Nov 2024 13:13:06 +0100 Subject: [PATCH] inherit lint github action from newsroom-app (#200) --- .github/workflows/lint.yml | 31 +-------- server/cp/commands/__init__.py | 2 +- server/cp/commands/fix_language.py | 6 +- server/cp/common_settings.py | 23 ++++--- server/cp/mgmt_api/app.py | 6 +- server/cp/mgmt_api/default_settings.py | 36 ++++++----- server/cp/mgmt_api/factory.py | 87 +++++++++++++++----------- server/cp/mgmt_api/navigations.py | 4 +- server/cp/mgmt_api/products.py | 2 +- server/cp/mgmt_api/topics.py | 30 +++++---- server/cp/mgmt_api/users.py | 26 ++++---- server/cp/password_reset_form.py | 4 +- server/cp/sidenav.py | 1 - server/dev-requirements.txt | 4 +- server/features/environment.py | 36 ++++++----- server/features/steps/steps.py | 7 +-- server/gunicorn_config.py | 4 +- server/manage.py | 2 +- server/mypy-requirements.in | 3 - server/mypy-requirements.txt | 29 +-------- server/settings_mgmtapi.py | 2 +- server/settings_newsapi.py | 2 +- server/setup.py | 2 +- 23 files changed, 165 insertions(+), 184 deletions(-) delete mode 100644 server/mypy-requirements.in diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e47e35f..c0ef70f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,31 +3,6 @@ name: "Lint" on: [ push, pull_request ] jobs: - - flake8: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - run: pip install flake8 - - run: flake8 - - mypy: - runs-on: ubuntu-latest - - defaults: - run: - working-directory: server - - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - run: pip install -r mypy-requirements.txt - - run: mypy cp + lint: + name: "Lint" + uses: superdesk/newsroom-app/.github/workflows/lint.yml@develop diff --git a/server/cp/commands/__init__.py b/server/cp/commands/__init__.py index 574c052..f439a0a 100644 --- a/server/cp/commands/__init__.py +++ b/server/cp/commands/__init__.py @@ -1,2 +1,2 @@ from . import fix_language # noqa -from . import fix_mediaformat # noqa \ No newline at end of file +from . import fix_mediaformat # noqa diff --git a/server/cp/commands/fix_language.py b/server/cp/commands/fix_language.py index 25c01be..d83b106 100644 --- a/server/cp/commands/fix_language.py +++ b/server/cp/commands/fix_language.py @@ -1,4 +1,3 @@ - import time from superdesk import get_resource_service @@ -9,7 +8,10 @@ def fix_language(resource="items", limit=50, sleep_secs=2): service = get_resource_service(resource) - source = {"query": {"terms": {"language": ["en-CA", "en-US", "fr-CA"]}}, "size": 100} + source = { + "query": {"terms": {"language": ["en-CA", "en-US", "fr-CA"]}}, + "size": 100, + } for i in range(int(limit)): items = service.search(source) diff --git a/server/cp/common_settings.py b/server/cp/common_settings.py index d3395b6..1b49862 100644 --- a/server/cp/common_settings.py +++ b/server/cp/common_settings.py @@ -3,12 +3,17 @@ from newsroom.web.default_settings import AUTH_PROVIDERS from newsroom.types import AuthProviderType -AUTH_PROVIDERS.extend([{ - "_id": "gip", - "name": lazy_gettext("Firebase"), - "auth_type": AuthProviderType.FIREBASE, -}, { - "_id": "azure", - "name": lazy_gettext("Azure"), - "auth_type": AuthProviderType.SAML, -}]) +AUTH_PROVIDERS.extend( + [ + { + "_id": "gip", + "name": lazy_gettext("Firebase"), + "auth_type": AuthProviderType.FIREBASE, + }, + { + "_id": "azure", + "name": lazy_gettext("Azure"), + "auth_type": AuthProviderType.SAML, + }, + ] +) diff --git a/server/cp/mgmt_api/app.py b/server/cp/mgmt_api/app.py index 959df99..afd1d4f 100644 --- a/server/cp/mgmt_api/app.py +++ b/server/cp/mgmt_api/app.py @@ -16,7 +16,7 @@ app = get_app() -if __name__ == '__main__': - host = '0.0.0.0' - port = int(os.environ.get('MGMTAPI_PORT', '5500')) +if __name__ == "__main__": + host = "0.0.0.0" + port = int(os.environ.get("MGMTAPI_PORT", "5500")) app.run(host=host, port=port, debug=True, use_reloader=True) diff --git a/server/cp/mgmt_api/default_settings.py b/server/cp/mgmt_api/default_settings.py index 29944f2..22b6faf 100644 --- a/server/cp/mgmt_api/default_settings.py +++ b/server/cp/mgmt_api/default_settings.py @@ -10,7 +10,7 @@ # at https://www.sourcefabric.org/superdesk/license from superdesk.default_settings import urlparse -from newsroom.web.default_settings import ( # noqa +from newsroom.web.default_settings import ( # noqa env, ELASTICSEARCH_URL, ELASTICSEARCH_SETTINGS, @@ -21,42 +21,44 @@ ) import os -MGMTAPI_URL = env('MGMTAPI_URL', 'http://localhost:5500/api') +MGMTAPI_URL = env("MGMTAPI_URL", "http://localhost:5500/api") server_url = urlparse(MGMTAPI_URL) URL_PREFIX = env("MGMTAPI_URL_PREFIX", server_url.path.strip("/")) or "" BLUEPRINTS = [] CORE_APPS = [ - 'cp.mgmt_api.companies', - 'cp.mgmt_api.users', - 'cp.mgmt_api.products', - 'cp.mgmt_api.navigations', - 'cp.mgmt_api.topics', + "cp.mgmt_api.companies", + "cp.mgmt_api.users", + "cp.mgmt_api.products", + "cp.mgmt_api.navigations", + "cp.mgmt_api.topics", ] INSTALLED_APPS = [] -LANGUAGES = ['en', 'fi', 'cs', 'fr_CA'] -DEFAULT_LANGUAGE = 'en' +LANGUAGES = ["en", "fi", "cs", "fr_CA"] +DEFAULT_LANGUAGE = "en" # newsroom default db and index names -MONGO_DBNAME = env('MONGO_DBNAME', 'newsroom') +MONGO_DBNAME = env("MONGO_DBNAME", "newsroom") # mongo -MONGO_URI = env('MONGO_URI', f'mongodb://localhost/{MONGO_DBNAME}') -CONTENTAPI_MONGO_URI = env('CONTENTAPI_MONGO_URI', f'mongodb://localhost/{MONGO_DBNAME}') +MONGO_URI = env("MONGO_URI", f"mongodb://localhost/{MONGO_DBNAME}") +CONTENTAPI_MONGO_URI = env( + "CONTENTAPI_MONGO_URI", f"mongodb://localhost/{MONGO_DBNAME}" +) # elastic -ELASTICSEARCH_INDEX = env('ELASTICSEARCH_INDEX', MONGO_DBNAME) -CONTENTAPI_ELASTICSEARCH_INDEX = env('CONTENTAPI_ELASTICSEARCH_INDEX', MONGO_DBNAME) +ELASTICSEARCH_INDEX = env("ELASTICSEARCH_INDEX", MONGO_DBNAME) +CONTENTAPI_ELASTICSEARCH_INDEX = env("CONTENTAPI_ELASTICSEARCH_INDEX", MONGO_DBNAME) FILTER_AGGREGATIONS = False -ELASTIC_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +ELASTIC_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S" -CACHE_TYPE = os.environ.get('CACHE_TYPE', 'simple') # in-memory cache +CACHE_TYPE = os.environ.get("CACHE_TYPE", "simple") # in-memory cache # The default timeout that is used if no timeout is specified in sec CACHE_DEFAULT_TIMEOUT = 3600 # Redis host (used only if CACHE_TYPE is redis) -CACHE_REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379') +CACHE_REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") # fix superdesk cache config CACHE_URL = CACHE_REDIS_URL diff --git a/server/cp/mgmt_api/factory.py b/server/cp/mgmt_api/factory.py index 9677b6a..1c072c5 100644 --- a/server/cp/mgmt_api/factory.py +++ b/server/cp/mgmt_api/factory.py @@ -27,19 +27,21 @@ class NewsroomMGMTAPI(BaseNewsroomApp): - INSTANCE_CONFIG = 'settings_mgmtapi.py' + INSTANCE_CONFIG = "settings_mgmtapi.py" AUTH_SERVICE = JWTAuth def __init__(self, import_name=__package__, config=None, **kwargs): - if not hasattr(self, 'settings'): - self.settings = flask.Config('.') + if not hasattr(self, "settings"): + self.settings = flask.Config(".") - if config and config.get('BEHAVE'): + if config and config.get("BEHAVE"): # ``superdesk.tests.update_config`` adds ``planning`` to ``INSTALLED_APPS`` # So if we're running behave tests, reset this config here - config['INSTALLED_APPS'] = [] + config["INSTALLED_APPS"] = [] - super(NewsroomMGMTAPI, self).__init__(import_name=import_name, config=config, **kwargs) + super(NewsroomMGMTAPI, self).__init__( + import_name=import_name, config=config, **kwargs + ) def load_app_default_config(self): """ @@ -48,7 +50,7 @@ def load_app_default_config(self): # default config from `content_api.app.settings` super().load_app_default_config() # default config from `cp.mgmt_api.default_settings` - self.config.from_object('cp.mgmt_api.default_settings') + self.config.from_object("cp.mgmt_api.default_settings") def load_app_instance_config(self): """ @@ -57,52 +59,61 @@ def load_app_instance_config(self): # config from newsroom-app settings_newsapi.py file super().load_app_instance_config() # config from env var - self.config.from_envvar('MGMT_API_SETTINGS', silent=True) + self.config.from_envvar("MGMT_API_SETTINGS", silent=True) def run(self, host=None, port=None, debug=None, **options): - if not self.config.get('MGMT_API_ENABLED', False): - raise RuntimeError('Management API is not enabled') + if not self.config.get("MGMT_API_ENABLED", False): + raise RuntimeError("Management API is not enabled") super(NewsroomMGMTAPI, self).run(host, port, debug, **options) def setup_error_handlers(self): def json_error(err): - return flask.jsonify(err), err['code'] + return flask.jsonify(err), err["code"] def handle_werkzeug_errors(err): - return json_error({ - 'error': str(err), - 'message': getattr(err, 'description') or None, - 'code': getattr(err, 'code') or 500 - }) + return json_error( + { + "error": str(err), + "message": getattr(err, "description") or None, + "code": getattr(err, "code") or 500, + } + ) def superdesk_api_error(err): - return json_error({ - 'error': err.message or '', - 'message': err.payload, - 'code': err.status_code or 500, - }) + return json_error( + { + "error": err.message or "", + "message": err.payload, + "code": err.status_code or 500, + } + ) def assertion_error(err): - return json_error({ - 'error': err.args[0] if err.args else 1, - 'message': str(err), - 'code': 400 - }) + return json_error( + { + "error": err.args[0] if err.args else 1, + "message": str(err), + "code": 400, + } + ) def base_exception_error(err): - if type(err) is ElasticRequestError and err.error == 'search_phase_execution_exception': - return json_error({ - 'error': 1, - 'message': 'Invalid search query', - 'code': 400 - }) - - return json_error({ - 'error': err.args[0] if err.args else 1, - 'message': str(err), - 'code': 500 - }) + if ( + type(err) is ElasticRequestError + and err.error == "search_phase_execution_exception" + ): + return json_error( + {"error": 1, "message": "Invalid search query", "code": 400} + ) + + return json_error( + { + "error": err.args[0] if err.args else 1, + "message": str(err), + "code": 500, + } + ) for cls in HTTPException.__subclasses__(): self.register_error_handler(cls, handle_werkzeug_errors) diff --git a/server/cp/mgmt_api/navigations.py b/server/cp/mgmt_api/navigations.py index e5a2ede..c621a58 100644 --- a/server/cp/mgmt_api/navigations.py +++ b/server/cp/mgmt_api/navigations.py @@ -4,4 +4,6 @@ def init_app(app): NavigationsResource.internal_resource = False - superdesk.register_resource('navigations', NavigationsResource, NavigationsService, _app=app) + superdesk.register_resource( + "navigations", NavigationsResource, NavigationsService, _app=app + ) diff --git a/server/cp/mgmt_api/products.py b/server/cp/mgmt_api/products.py index 9864756..484e1c4 100644 --- a/server/cp/mgmt_api/products.py +++ b/server/cp/mgmt_api/products.py @@ -4,4 +4,4 @@ def init_app(app): ProductsResource.internal_resource = False - superdesk.register_resource('products', ProductsResource, ProductsService, _app=app) + superdesk.register_resource("products", ProductsResource, ProductsService, _app=app) diff --git a/server/cp/mgmt_api/topics.py b/server/cp/mgmt_api/topics.py index 2ae4af7..2e289c1 100644 --- a/server/cp/mgmt_api/topics.py +++ b/server/cp/mgmt_api/topics.py @@ -9,32 +9,38 @@ def init_app(app): TopicsResource.internal_resource = False - superdesk.register_resource('topics', GlobalTopicsResource, GlobalTopicsService, _app=app) - superdesk.register_resource('topic_folders', FoldersResource, FoldersService, _app=app) + superdesk.register_resource( + "topics", GlobalTopicsResource, GlobalTopicsService, _app=app + ) + superdesk.register_resource( + "topic_folders", FoldersResource, FoldersService, _app=app + ) class GlobalTopicsResource(TopicsResource): - url = 'topics' - resource_methods = ['GET', 'POST', 'DELETE'] + url = "topics" + resource_methods = ["GET", "POST", "DELETE"] class GlobalTopicsService(TopicsService): def on_create(self, docs): super().on_create(docs) for doc in docs: - user = doc.get('user') + user = doc.get("user") if user: - doc['original_creator'] = user - doc['version_creator'] = user - elif not doc.get('is_global'): - message = ("Please set is_global True, or provide user in the body.") - raise SuperdeskApiError.badRequestError(message=message, payload=message) + doc["original_creator"] = user + doc["version_creator"] = user + elif not doc.get("is_global"): + message = "Please set is_global True, or provide user in the body." + raise SuperdeskApiError.badRequestError( + message=message, payload=message + ) def on_created(self, docs): super().on_created(docs) for doc in docs: - app.cache.set(str(doc['_id']), doc) + app.cache.set(str(doc["_id"]), doc) def on_update(self, updates, original): super().on_update(updates, original) - app.cache.delete(str(original['_id'])) + app.cache.delete(str(original["_id"])) diff --git a/server/cp/mgmt_api/users.py b/server/cp/mgmt_api/users.py index c12f6d0..32c313c 100644 --- a/server/cp/mgmt_api/users.py +++ b/server/cp/mgmt_api/users.py @@ -21,7 +21,7 @@ class CPUsersResource(newsroom.Resource): def init_app(app): - superdesk.register_resource('users', CPUsersResource, CPUsersService, _app=app) + superdesk.register_resource("users", CPUsersResource, CPUsersService, _app=app) class CPUsersService(UsersService): @@ -34,15 +34,19 @@ def check_permissions(self, doc, updates=None): def on_create(self, docs): super().on_create(docs) for doc in docs: - if doc.get('user_type') != 'administrator' and not doc.get('company'): - message = ("Company is required if user type is not administrator.") - raise SuperdeskApiError.badRequestError(message=message, payload=message) - locale = doc.get('locale') - if locale and locale not in app.config['LANGUAGES']: - message = ("Locale is not in configured list of locales.") - raise SuperdeskApiError.badRequestError(message=message, payload=message) - if doc.get('company'): - doc['company'] = ObjectId(doc.get('company')) + if doc.get("user_type") != "administrator" and not doc.get("company"): + message = "Company is required if user type is not administrator." + raise SuperdeskApiError.badRequestError( + message=message, payload=message + ) + locale = doc.get("locale") + if locale and locale not in app.config["LANGUAGES"]: + message = "Locale is not in configured list of locales." + raise SuperdeskApiError.badRequestError( + message=message, payload=message + ) + if doc.get("company"): + doc["company"] = ObjectId(doc.get("company")) if doc.get("products"): validate_product_refs(doc["products"]) @@ -55,5 +59,5 @@ def on_update(self, updates, original): def get(self, req, lookup): """""" cursor = super().get(req, lookup) - cursor.collation(Collation(locale='en', strength=1)) + cursor.collation(Collation(locale="en", strength=1)) return cursor diff --git a/server/cp/password_reset_form.py b/server/cp/password_reset_form.py index 10ac6e5..7868282 100644 --- a/server/cp/password_reset_form.py +++ b/server/cp/password_reset_form.py @@ -5,5 +5,7 @@ class PasswordResetForm(FlaskForm): - email = StringField(lazy_gettext("Email"), validators=[DataRequired(), Length(1, 64), Email()]) + email = StringField( + lazy_gettext("Email"), validators=[DataRequired(), Length(1, 64), Email()] + ) email_sent = BooleanField("email_sent", validators=[]) diff --git a/server/cp/sidenav.py b/server/cp/sidenav.py index 6588f91..e3ab043 100644 --- a/server/cp/sidenav.py +++ b/server/cp/sidenav.py @@ -1,4 +1,3 @@ - def init_app(app): app.sidenav( name="CP Images", diff --git a/server/dev-requirements.txt b/server/dev-requirements.txt index 2d366cc..f693d7f 100644 --- a/server/dev-requirements.txt +++ b/server/dev-requirements.txt @@ -1,6 +1,6 @@ -r requirements.txt +-r mypy-requirements.txt -black>=23,<24 flake8 sphinx sphinx-autobuild @@ -10,5 +10,3 @@ pytest-mock responses httmock wooper -mypy -typing_extensions>=3.7.4 diff --git a/server/features/environment.py b/server/features/environment.py index f8e2ea1..4e63204 100644 --- a/server/features/environment.py +++ b/server/features/environment.py @@ -10,36 +10,38 @@ def get_app(*args, **kwargs): return _get_app(*args, testing=True, **kwargs) -class TestClient(): - client_id = 'test' +class TestClient: + client_id = "test" def before_all(context): config = { - 'BEHAVE': True, - 'CORE_APPS': CORE_APPS, - 'INSTALLED_APPS': [], - 'ELASTICSEARCH_FORCE_REFRESH': True, - 'MGMT_API_ENABLED': True, - 'CACHE_TYPE': "null", + "BEHAVE": True, + "CORE_APPS": CORE_APPS, + "INSTALLED_APPS": [], + "ELASTICSEARCH_FORCE_REFRESH": True, + "MGMT_API_ENABLED": True, + "CACHE_TYPE": "null", } setup_before_all(context, config, app_factory=get_app) def before_scenario(context, scenario): config = { - 'BEHAVE': True, - 'CORE_APPS': CORE_APPS, - 'INSTALLED_APPS': [], - 'ELASTICSEARCH_FORCE_REFRESH': True, - 'MGMT_API_ENABLED': True, - 'AUTH_SERVER_SHARED_SECRET': 'test-secret', - 'CACHE_TYPE': "null", + "BEHAVE": True, + "CORE_APPS": CORE_APPS, + "INSTALLED_APPS": [], + "ELASTICSEARCH_FORCE_REFRESH": True, + "MGMT_API_ENABLED": True, + "AUTH_SERVER_SHARED_SECRET": "test-secret", + "CACHE_TYPE": "null", } setup_before_scenario(context, scenario, config, app_factory=get_app) with context.app.app_context(): config_oauth(context.app) - token = generate_jwt_token(TestClient(), 'client_credentials', 'test', '').decode() - context.headers.append(('Authorization', f'Bearer {token}')) + token = generate_jwt_token( + TestClient(), "client_credentials", "test", "" + ).decode() + context.headers.append(("Authorization", f"Bearer {token}")) diff --git a/server/features/steps/steps.py b/server/features/steps/steps.py index 0ea7e1b..bc96fa0 100644 --- a/server/features/steps/steps.py +++ b/server/features/steps/steps.py @@ -1,10 +1,9 @@ -from newsroom.tests.steps import * # noqa +from newsroom.tests.steps import * # noqa from behave import given -@given('empty auth token') +@given("empty auth token") def given_empty_auth_token(context): context.headers = [ - header for header in context.headers - if header[0] != 'Authorization' + header for header in context.headers if header[0] != "Authorization" ] diff --git a/server/gunicorn_config.py b/server/gunicorn_config.py index a67084b..4d5e2ac 100644 --- a/server/gunicorn_config.py +++ b/server/gunicorn_config.py @@ -8,5 +8,5 @@ accesslog = "-" access_log_format = "%(m)s %(U)s status=%(s)s time=%(T)ss size=%(B)sb" -reload = 'SUPERDESK_RELOAD' in os.environ -timeout = int(os.environ.get('WEB_TIMEOUT', 30)) +reload = "SUPERDESK_RELOAD" in os.environ +timeout = int(os.environ.get("WEB_TIMEOUT", 30)) diff --git a/server/manage.py b/server/manage.py index dd47659..d3b2016 100644 --- a/server/manage.py +++ b/server/manage.py @@ -1,4 +1,4 @@ -from newsroom.commands import * # noqa +from newsroom.commands import * # noqa from newsroom.commands.manager import manager import cp.commands # noqa diff --git a/server/mypy-requirements.in b/server/mypy-requirements.in deleted file mode 100644 index 0c35e05..0000000 --- a/server/mypy-requirements.in +++ /dev/null @@ -1,3 +0,0 @@ -mypy -types-flask -types-requests diff --git a/server/mypy-requirements.txt b/server/mypy-requirements.txt index 9b180b8..df722a3 100644 --- a/server/mypy-requirements.txt +++ b/server/mypy-requirements.txt @@ -1,28 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile mypy-requirements.in -# mypy==1.13.0 - # via -r mypy-requirements.in -mypy-extensions==1.0.0 - # via mypy -tomli==2.0.1 - # via mypy -types-click==7.1.8 - # via types-flask + types-flask==1.1.6 - # via -r mypy-requirements.in -types-jinja2==2.11.9 - # via types-flask -types-markupsafe==1.1.10 - # via types-jinja2 -types-requests==2.32.0.20241016 - # via -r mypy-requirements.in -types-werkzeug==1.0.9 - # via types-flask -typing-extensions==4.12.2 - # via mypy -urllib3==2.2.3 - # via types-requests +types-requests==2.28.11.8 +types-docutils==0.21.0.20241005 \ No newline at end of file diff --git a/server/settings_mgmtapi.py b/server/settings_mgmtapi.py index a75b168..d6c9acf 100644 --- a/server/settings_mgmtapi.py +++ b/server/settings_mgmtapi.py @@ -1,6 +1,6 @@ from pathlib import Path -from superdesk.default_settings import env, strtobool # noqa +from superdesk.default_settings import env, strtobool # noqa from cp.common_settings import AUTH_PROVIDERS # noqa diff --git a/server/settings_newsapi.py b/server/settings_newsapi.py index 82d0b66..7fb56d1 100644 --- a/server/settings_newsapi.py +++ b/server/settings_newsapi.py @@ -1,6 +1,6 @@ from pathlib import Path -from superdesk.default_settings import env, strtobool # noqa +from superdesk.default_settings import env, strtobool # noqa from newsroom.news_api.default_settings import CORE_APPS, INSTALLED_APPS, BLUEPRINTS diff --git a/server/setup.py b/server/setup.py index c06b1fb..4746a8c 100644 --- a/server/setup.py +++ b/server/setup.py @@ -2,6 +2,6 @@ setup( - name='Newsroom App CP', + name="Newsroom App CP", packages=find_packages(), )