From 5d467db5a3ea172ddf624a7b62fe7b6c9af200f7 Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Fri, 7 Jun 2024 17:36:09 -0400 Subject: [PATCH 1/8] Settings class that selectively allows environment variable override. --- src/palace/manager/util/settings.py | 140 ++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 src/palace/manager/util/settings.py diff --git a/src/palace/manager/util/settings.py b/src/palace/manager/util/settings.py new file mode 100644 index 0000000000..29b1be9a68 --- /dev/null +++ b/src/palace/manager/util/settings.py @@ -0,0 +1,140 @@ +import functools +from collections.abc import Callable +from typing import Any + +from pydantic.env_settings import BaseSettings, SettingsSourceCallable +from pydantic.fields import ModelField + +from palace.manager.core.config import CannotLoadConfiguration +from palace.manager.util.log import LoggerMixin + + +def _env_var_for(field: ModelField) -> str | None: + env_prefix = field.model_config.env_prefix # type: ignore[attr-defined] + return (env_prefix + field.name).upper() + + +def _restrict_environment( + env_settings: Callable[[BaseSettings], dict[str, Any]], settings: BaseSettings +) -> dict[str, Any]: + """Limit environment variables to those not restricted by the `environment_override_*` settings. + + :param env_settings: The environment settings source function, usually indirectly from `pydantic`.. + :param settings: A pydantic model instance. + :return: A dictionary by field alias of values from the environment. + + :raises CannotLoadConfiguration: Under the following conditions: + - A non-existent field is specified in one of the `environment_override_*` settings. + - A field is specified in more than one `environment_override_*` setting. + - A field specified in `environment_override_error_fields` is overridden in the environment + + If a field is (1) specified in `environment_override_warning_fields` and (2) overridden in the + environment, then a warning is logged and the field is NOT overridden. + """ + env_settings_by_alias = env_settings(settings) + if not env_settings_by_alias: + return env_settings_by_alias + + config = settings.__config__ + logger = settings.log # type: ignore[attr-defined] + + fields_by_name = settings.__fields__ + fields_by_alias = {field.alias: field for name, field in fields_by_name.items()} + env_settings_by_name = { + fields_by_alias[alias].name: value + for alias, value in env_settings_by_alias.items() + if alias in fields_by_alias + } + + warning_fields: set[str] = config.environment_override_warning_fields or set() # type: ignore[attr-defined] + error_fields: set[str] = config.environment_override_error_fields or set() # type: ignore[attr-defined] + + if nonexistent_fields := (warning_fields | error_fields) - set(fields_by_name): + raise CannotLoadConfiguration( + "Only existing fields may be specified in either the `environment_override_warning_fields` " + "or `environment_override_error_fields` settings. The following fields do not exist: " + f"{nonexistent_fields}." + ) + if overlapping_fields := warning_fields & error_fields: + raise CannotLoadConfiguration( + "A field may not be specified in both the `environment_override_warning_fields` and " + "`environment_override_error_fields` settings. The following are specified in both: " + f"{overlapping_fields}." + ) + if warnings := set(env_settings_by_name) & warning_fields: + _msg = ( + "Some `environment_override_warning_fields` are overridden in the environment. " + "The value from the environment will be ignored." + ) + for field in (fields_by_name[name] for name in warnings): + _msg += f"\n {field.name}: alias={field.alias}, env={_env_var_for(field)}" + logger.warning(_msg) + + if errors := set(env_settings_by_name) & error_fields: + _msg = ( + "Some `environment_override_error_fields` are overridden in the environment. " + "Please remove them from the environment or from the configuration settings." + ) + for field in (fields_by_name[name] for name in errors): + _msg += f"\n {field.name}: alias={field.alias}, env={_env_var_for(field)}" + raise CannotLoadConfiguration(_msg) + + overridable_names = set(fields_by_name) - warnings - errors + overridable_aliases = { + field.alias + for name, field in fields_by_name.items() + if name in overridable_names + } + + return { + alias: value + for alias, value in env_settings_by_alias.items() + if alias in overridable_aliases + } + + +class BaseSettingsRestrictEnvOverride(BaseSettings, LoggerMixin): + # Fields that can be overridden by environment variables should be specified as normal. + + # For non-overridable fields: + # - Set `const=True` on the field, if nothing should override the default.. + # - Add the field name to one of the `environment_override_*` Config settings. + + class Config: + # Handle environment variable overrides, depending on presence of field name in: + # environment_override_error_fields: report field and raise exception; or + # environment_override_warning_fields: report field and log warning. + # If a field is not specified in one of these lists, an override is permitted. + # If a field is specified in both, it is an error and an exception is raised. + # If a field is NOT specified in one of these lists, then an override is allowed. + # The exception, when raised, will be a `CannotLoadConfiguration`. + environment_override_error_fields: set[str] | None = None + environment_override_warning_fields: set[str] | None = None + + # See the pydantic docs for information on these settings + # https://docs.pydantic.dev/usage/model_config/ + + # Strip whitespace from all strings + anystr_strip_whitespace = True + + # Forbid mutation, so that its clear that settings changes will + # not automatically be saved to the database. + allow_mutation = False + + # See `pydantic` documentation on adding sources. + # https://docs.pydantic.dev/1.10/usage/settings/#adding-sources + @classmethod + def customise_sources( + cls, + init_settings, + env_settings, + file_secret_settings, + ) -> tuple[SettingsSourceCallable, ...]: + # We have to wrap the environment settings source in our own function + # so that we can report on/strip out fields that are not overridable + # before `pydantic` sees them. + return ( + init_settings, + functools.partial(_restrict_environment, env_settings), + file_secret_settings, + ) From 607162fe0bd61830ebe152f703694abb9c7dd118 Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Fri, 7 Jun 2024 17:38:02 -0400 Subject: [PATCH 2/8] Add an Admin UI feature flags class. --- src/palace/manager/api/admin/config.py | 45 +++++++++++++++++++ .../manager/api/admin/controller/view.py | 7 +-- src/palace/manager/api/admin/templates.py | 4 +- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/palace/manager/api/admin/config.py b/src/palace/manager/api/admin/config.py index 1bc5ad36f9..4512607740 100644 --- a/src/palace/manager/api/admin/config.py +++ b/src/palace/manager/api/admin/config.py @@ -2,10 +2,52 @@ from enum import Enum from urllib.parse import urljoin +from pydantic import Field from requests import RequestException +from palace.manager.util.flask_util import _snake_to_camel_case from palace.manager.util.http import HTTP, RequestNetworkException from palace.manager.util.log import LoggerMixin +from palace.manager.util.settings import BaseSettingsRestrictEnvOverride + + +class AdminClientFeatureFlags(BaseSettingsRestrictEnvOverride): + # The following CAN be overridden by environment variables. + reports_only_for_sysadmins: bool = Field( + True, + description="Show inventory reports only for sysadmins.", + ) + + # The following fields CANNOT be overridden by environment variables. + # Setting `const=True` ensures that the default value is not overridden. + # Add them to the one of the `environment_override_*` Config settings + # below to prevent them from being overridden. + # NB: Overriding the `env_prefix` with `env=...` here may lead to + # incorrect values in warnings and exceptions, since `env_prefix` + # is used to generate the full environment variable name. + enable_auto_list: bool = Field( + True, + const=True, + description="Enable auto-list of items.", + ) + show_circ_events_download: bool = Field( + True, + const=True, + description="Show download button for Circulation Events.", + ) + + class Config: + # We use lower camel case aliases, since we're sending to JavaScript. + alias_generator = _snake_to_camel_case + env_prefix = "PALACE_ADMINUI_FEATURES_" + env_file = ".env" + + # Restrict environment variable overrides. + # + environment_override_warning_fields: set[str] | None = { + "enable_auto_list", + "show_circ_events_download", + } class OperationalMode(str, Enum): @@ -53,6 +95,9 @@ class Configuration(LoggerMixin): # Cache the package version after first lookup. _version: str | None = None + # Admin client feature flags + admin_feature_flags = AdminClientFeatureFlags() + @classmethod def operational_mode(cls) -> OperationalMode: return ( diff --git a/src/palace/manager/api/admin/controller/view.py b/src/palace/manager/api/admin/controller/view.py index 606f114e0f..d1eba1825a 100644 --- a/src/palace/manager/api/admin/controller/view.py +++ b/src/palace/manager/api/admin/controller/view.py @@ -67,8 +67,8 @@ def __call__(self, collection, book, path=None): admin_js = AdminClientConfig.lookup_asset_url(key="admin_js") admin_css = AdminClientConfig.lookup_asset_url(key="admin_css") - # We always have local_analytics - show_circ_events_download = True + # # We always have local_analytics + # show_circ_events_download = True response = Response( flask.render_template_string( @@ -77,12 +77,13 @@ def __call__(self, collection, book, path=None): csrf_token=csrf_token, sitewide_tos_href=Configuration.DEFAULT_TOS_HREF, sitewide_tos_text=Configuration.DEFAULT_TOS_TEXT, - show_circ_events_download=show_circ_events_download, + show_circ_events_download=AdminClientConfig.admin_feature_flags.show_circ_events_download, setting_up=setting_up, email=email, roles=roles, admin_js=admin_js, admin_css=admin_css, + feature_flags=AdminClientConfig.admin_feature_flags.json(by_alias=True), ) ) diff --git a/src/palace/manager/api/admin/templates.py b/src/palace/manager/api/admin/templates.py index 712f2e61c7..51a2e3b686 100644 --- a/src/palace/manager/api/admin/templates.py +++ b/src/palace/manager/api/admin/templates.py @@ -17,9 +17,7 @@ settingUp: {{ "true" if setting_up else "false" }}, email: "{{ email }}", roles: [{% for role in roles %}{"role": "{{role.role}}"{% if role.library %}, "library": "{{role.library.short_name}}"{% endif %} },{% endfor %}], - featureFlags: { - enableAutoList: true, - }, + featureFlags: {{ feature_flags| safe }}, }); From 43b802f80a156b05c6545b750966d093d44f51b9 Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Sat, 8 Jun 2024 13:20:17 -0400 Subject: [PATCH 3/8] Build on ServiceConfiguration class. --- src/palace/manager/api/admin/config.py | 27 ++-- .../manager/api/admin/controller/view.py | 6 +- src/palace/manager/util/settings.py | 128 +++++++++--------- tests/manager/service/test_configuration.py | 66 +++++++++ 4 files changed, 149 insertions(+), 78 deletions(-) diff --git a/src/palace/manager/api/admin/config.py b/src/palace/manager/api/admin/config.py index 4512607740..570a4ea3fe 100644 --- a/src/palace/manager/api/admin/config.py +++ b/src/palace/manager/api/admin/config.py @@ -8,10 +8,10 @@ from palace.manager.util.flask_util import _snake_to_camel_case from palace.manager.util.http import HTTP, RequestNetworkException from palace.manager.util.log import LoggerMixin -from palace.manager.util.settings import BaseSettingsRestrictEnvOverride +from palace.manager.util.settings import ServiceConfigurationWithLimitedEnvOverride -class AdminClientFeatureFlags(BaseSettingsRestrictEnvOverride): +class AdminClientFeatureFlags(ServiceConfigurationWithLimitedEnvOverride): # The following CAN be overridden by environment variables. reports_only_for_sysadmins: bool = Field( True, @@ -37,17 +37,19 @@ class AdminClientFeatureFlags(BaseSettingsRestrictEnvOverride): ) class Config: - # We use lower camel case aliases, since we're sending to JavaScript. + env_prefix = "PALACE_ADMINUI_FEATURE_" + + # We use lower camel case aliases, since we're sending to the web. alias_generator = _snake_to_camel_case - env_prefix = "PALACE_ADMINUI_FEATURES_" - env_file = ".env" - # Restrict environment variable overrides. - # - environment_override_warning_fields: set[str] | None = { + # Add any fields that should not be overridden by environment variables here. + # - environment_override_warning_fields: warnings and ignore environment + # - environment_override_error_fields: raise exception + environment_override_warning_fields: set[str] = { "enable_auto_list", "show_circ_events_download", } + environment_override_error_fields: set[str] = set() class OperationalMode(str, Enum): @@ -95,8 +97,15 @@ class Configuration(LoggerMixin): # Cache the package version after first lookup. _version: str | None = None + # Cache the feature flags after the first lookup. + _admin_ui_feature_flags: AdminClientFeatureFlags | None = None + # Admin client feature flags - admin_feature_flags = AdminClientFeatureFlags() + @classmethod + def admin_feature_flags(cls) -> AdminClientFeatureFlags: + if not cls._admin_ui_feature_flags: + cls._admin_ui_feature_flags = AdminClientFeatureFlags() + return cls._admin_ui_feature_flags @classmethod def operational_mode(cls) -> OperationalMode: diff --git a/src/palace/manager/api/admin/controller/view.py b/src/palace/manager/api/admin/controller/view.py index d1eba1825a..ab6a819b59 100644 --- a/src/palace/manager/api/admin/controller/view.py +++ b/src/palace/manager/api/admin/controller/view.py @@ -77,13 +77,15 @@ def __call__(self, collection, book, path=None): csrf_token=csrf_token, sitewide_tos_href=Configuration.DEFAULT_TOS_HREF, sitewide_tos_text=Configuration.DEFAULT_TOS_TEXT, - show_circ_events_download=AdminClientConfig.admin_feature_flags.show_circ_events_download, + show_circ_events_download=AdminClientConfig.admin_feature_flags().show_circ_events_download, setting_up=setting_up, email=email, roles=roles, admin_js=admin_js, admin_css=admin_css, - feature_flags=AdminClientConfig.admin_feature_flags.json(by_alias=True), + feature_flags=AdminClientConfig.admin_feature_flags().json( + by_alias=True + ), ) ) diff --git a/src/palace/manager/util/settings.py b/src/palace/manager/util/settings.py index 29b1be9a68..31a9eac63c 100644 --- a/src/palace/manager/util/settings.py +++ b/src/palace/manager/util/settings.py @@ -6,9 +6,47 @@ from pydantic.fields import ModelField from palace.manager.core.config import CannotLoadConfiguration +from palace.manager.service.configuration import ServiceConfiguration from palace.manager.util.log import LoggerMixin +class ServiceConfigurationWithLimitedEnvOverride(ServiceConfiguration, LoggerMixin): + # Fields that can be overridden by environment variables should be specified as normal. + + # For non-overridable fields: + # - Set `const=True` on the field, if nothing should override the default.. + # - Add the field name to one of the `environment_override_*` Config settings. + + class Config: + # Handle environment variable overrides, depending on presence of field name in: + # environment_override_error_fields: report field and raise exception; or + # environment_override_warning_fields: report field and log warning. + # If a field is not specified in one of these lists, an override is permitted. + # If a field is specified in both, it is an error and an exception is raised. + # If a field is NOT specified in one of these lists, then an override is allowed. + # The exception, when raised, will be a `CannotLoadConfiguration`. + environment_override_error_fields: set[str] | None = None + environment_override_warning_fields: set[str] | None = None + + # See `pydantic` documentation on customizing sources. + # https://docs.pydantic.dev/1.10/usage/settings/#adding-sources + @classmethod + def customise_sources( + cls, + init_settings, + env_settings, + file_secret_settings, + ) -> tuple[SettingsSourceCallable, ...]: + # We have to wrap the environment settings source in our own function + # so that we can report on/strip out fields that are not overridable + # before `pydantic` sees them. + return ( + init_settings, + functools.partial(_restrict_environment, env_settings), + file_secret_settings, + ) + + def _env_var_for(field: ModelField) -> str | None: env_prefix = field.model_config.env_prefix # type: ignore[attr-defined] return (env_prefix + field.name).upper() @@ -31,40 +69,43 @@ def _restrict_environment( If a field is (1) specified in `environment_override_warning_fields` and (2) overridden in the environment, then a warning is logged and the field is NOT overridden. """ - env_settings_by_alias = env_settings(settings) - if not env_settings_by_alias: - return env_settings_by_alias - config = settings.__config__ logger = settings.log # type: ignore[attr-defined] - fields_by_name = settings.__fields__ - fields_by_alias = {field.alias: field for name, field in fields_by_name.items()} - env_settings_by_name = { - fields_by_alias[alias].name: value - for alias, value in env_settings_by_alias.items() - if alias in fields_by_alias - } - warning_fields: set[str] = config.environment_override_warning_fields or set() # type: ignore[attr-defined] error_fields: set[str] = config.environment_override_error_fields or set() # type: ignore[attr-defined] + fields_by_name = settings.__fields__ + fields_by_alias = {field.alias: field for name, field in fields_by_name.items()} + if nonexistent_fields := (warning_fields | error_fields) - set(fields_by_name): raise CannotLoadConfiguration( - "Only existing fields may be specified in either the `environment_override_warning_fields` " - "or `environment_override_error_fields` settings. The following fields do not exist: " + "Only existing fields may be specified in any of the `environment_override_*` " + "settings. The following are not the name of an existing field: " f"{nonexistent_fields}." ) if overlapping_fields := warning_fields & error_fields: raise CannotLoadConfiguration( - "A field may not be specified in both the `environment_override_warning_fields` and " - "`environment_override_error_fields` settings. The following are specified in both: " + "A field may not be specified in more than one `environment_override_*` setting. " + "The following field names are specified in multiple settings: " f"{overlapping_fields}." ) + + env_settings_by_alias = env_settings(settings) + if not env_settings_by_alias: + return env_settings_by_alias + + env_settings_by_name = { + fields_by_alias[alias].name: value + for alias, value in env_settings_by_alias.items() + if alias in fields_by_alias + } + if warnings := set(env_settings_by_name) & warning_fields: _msg = ( - "Some `environment_override_warning_fields` are overridden in the environment. " - "The value from the environment will be ignored." + "Some `environment_override_warning_fields` are overridden in the environment. Please " + "remove from either the environment or the `environment_override_warning_fields` setting." + "The value(s) from the environment will be ignored." ) for field in (fields_by_name[name] for name in warnings): _msg += f"\n {field.name}: alias={field.alias}, env={_env_var_for(field)}" @@ -72,8 +113,8 @@ def _restrict_environment( if errors := set(env_settings_by_name) & error_fields: _msg = ( - "Some `environment_override_error_fields` are overridden in the environment. " - "Please remove them from the environment or from the configuration settings." + "Some `environment_override_error_fields` are overridden in the environment. Please " + "remove from either the environment or the `environment_override_error_fields` setting." ) for field in (fields_by_name[name] for name in errors): _msg += f"\n {field.name}: alias={field.alias}, env={_env_var_for(field)}" @@ -91,50 +132,3 @@ def _restrict_environment( for alias, value in env_settings_by_alias.items() if alias in overridable_aliases } - - -class BaseSettingsRestrictEnvOverride(BaseSettings, LoggerMixin): - # Fields that can be overridden by environment variables should be specified as normal. - - # For non-overridable fields: - # - Set `const=True` on the field, if nothing should override the default.. - # - Add the field name to one of the `environment_override_*` Config settings. - - class Config: - # Handle environment variable overrides, depending on presence of field name in: - # environment_override_error_fields: report field and raise exception; or - # environment_override_warning_fields: report field and log warning. - # If a field is not specified in one of these lists, an override is permitted. - # If a field is specified in both, it is an error and an exception is raised. - # If a field is NOT specified in one of these lists, then an override is allowed. - # The exception, when raised, will be a `CannotLoadConfiguration`. - environment_override_error_fields: set[str] | None = None - environment_override_warning_fields: set[str] | None = None - - # See the pydantic docs for information on these settings - # https://docs.pydantic.dev/usage/model_config/ - - # Strip whitespace from all strings - anystr_strip_whitespace = True - - # Forbid mutation, so that its clear that settings changes will - # not automatically be saved to the database. - allow_mutation = False - - # See `pydantic` documentation on adding sources. - # https://docs.pydantic.dev/1.10/usage/settings/#adding-sources - @classmethod - def customise_sources( - cls, - init_settings, - env_settings, - file_secret_settings, - ) -> tuple[SettingsSourceCallable, ...]: - # We have to wrap the environment settings source in our own function - # so that we can report on/strip out fields that are not overridable - # before `pydantic` sees them. - return ( - init_settings, - functools.partial(_restrict_environment, env_settings), - file_secret_settings, - ) diff --git a/tests/manager/service/test_configuration.py b/tests/manager/service/test_configuration.py index b4ffdb79ca..068b610672 100644 --- a/tests/manager/service/test_configuration.py +++ b/tests/manager/service/test_configuration.py @@ -7,6 +7,7 @@ from palace.manager.core.config import CannotLoadConfiguration from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.util.settings import ServiceConfigurationWithLimitedEnvOverride if TYPE_CHECKING: from pytest import MonkeyPatch @@ -119,3 +120,68 @@ def test_exception_mutation( # Ignore the type error, since it tells us this is immutable, # and we are testing that behavior at runtime. config.string_with_default = "new value" # type: ignore[misc] + + +class TestServiceConfigurationWithLimitedEnvOverride: + def test_unknown_field(self): + class MockConfiguration1(ServiceConfigurationWithLimitedEnvOverride): + existing_field: bool = True + + class Config: + env_prefix = "MOCK_" + environment_override_error_fields = {"non_existing_field"} + + with pytest.raises(CannotLoadConfiguration) as exc_info: + MockConfiguration1() + assert "The following are not the name of an existing field" in str( + exc_info.value + ) + assert "non_existing_field" in str(exc_info.value) + + class MockConfiguration2(ServiceConfigurationWithLimitedEnvOverride): + existing_field: bool = True + + class Config: + env_prefix = "MOCK_" + environment_override_warning_fields = {"non_existing_field"} + + with pytest.raises(CannotLoadConfiguration) as exc_info: + MockConfiguration2() + assert "The following are not the name of an existing field" in str( + exc_info.value + ) + assert "non_existing_field" in str(exc_info.value) + + def test_overlapping_fields(self): + class MockConfiguration(ServiceConfigurationWithLimitedEnvOverride): + field: bool = True + overlapping_field: bool = True + + class Config: + env_prefix = "MOCK_" + environment_override_error_fields = {"field", "overlapping_field"} + environment_override_warning_fields = {"overlapping_field"} + + with pytest.raises(CannotLoadConfiguration) as exc_info: + MockConfiguration() + assert "The following field names are specified in multiple settings" in str( + exc_info.value + ) + assert "overlapping_field" in str(exc_info.value) + + def test_environment_override_warning_fields( + self, + caplog: pytest.LogCaptureFixture, + service_configuration_fixture: ServiceConfigurationFixture, + ): + class MockConfiguration(ServiceConfigurationWithLimitedEnvOverride): + field: bool = True + warning_field1: bool = True + warning_field2: bool = True + + class Config: + env_prefix = "MOCK_" + environment_override_warning_fields = { + "warning_field1", + "warning_field2", + } From 07422ab9defaa7d191205b14a3c37f5f784c8514 Mon Sep 17 00:00:00 2001 From: GitHub -- tdilauro Date: Wed, 12 Jun 2024 09:14:20 -0400 Subject: [PATCH 4/8] Add tests. --- .../manager/api/admin/controller/test_view.py | 72 +++++++++++++++++ tests/manager/service/test_configuration.py | 78 +++++++++++++++++-- 2 files changed, 144 insertions(+), 6 deletions(-) diff --git a/tests/manager/api/admin/controller/test_view.py b/tests/manager/api/admin/controller/test_view.py index afdecbfc16..aba91bac75 100644 --- a/tests/manager/api/admin/controller/test_view.py +++ b/tests/manager/api/admin/controller/test_view.py @@ -1,11 +1,17 @@ import re +from unittest.mock import patch import flask +import pytest +from _pytest.logging import LogCaptureFixture +from _pytest.monkeypatch import MonkeyPatch from werkzeug.http import dump_cookie +from palace.manager.api.admin.config import AdminClientFeatureFlags from palace.manager.api.admin.password_admin_authentication_provider import ( PasswordAdminAuthenticationProvider, ) +from palace.manager.service.logging.configuration import LogLevel from palace.manager.sqlalchemy.model.admin import AdminRole from palace.manager.sqlalchemy.model.library import Library from tests.fixtures.api_admin import AdminControllerFixture @@ -163,3 +169,69 @@ def test_roles(self, admin_ctrl_fixture: AdminControllerFixture): % admin_ctrl_fixture.ctrl.db.default_library().short_name in html ) + + def test_feature_flags_defaults( + self, + admin_ctrl_fixture: AdminControllerFixture, + monkeypatch: pytest.MonkeyPatch, + ): + admin_ctrl_fixture.admin.password_hashed = None + html_feature_flags_re = re.compile( + r"featureFlags: {(.*)?}", re.MULTILINE | re.DOTALL + ) + + with admin_ctrl_fixture.ctrl.app.test_request_context("/admin"): + response = admin_ctrl_fixture.manager.admin_view_controller(None, None) + assert 200 == response.status_code + html = response.get_data(as_text=True) + + match = html_feature_flags_re.search(html) + assert match is not None + feature_flags = match.groups(0)[0] + assert '"enableAutoList": true' in feature_flags + assert '"showCircEventsDownload": true' in feature_flags + assert '"reportsOnlyForSysadmins": true' in feature_flags + + def test_feature_flags_overridden( + self, + admin_ctrl_fixture: AdminControllerFixture, + monkeypatch: MonkeyPatch, + caplog: LogCaptureFixture, + ): + caplog.set_level(LogLevel.warning) + admin_ctrl_fixture.admin.password_hashed = None + html_feature_flags_re = re.compile( + r"featureFlags: {(.*)?}", re.MULTILINE | re.DOTALL + ) + + monkeypatch.setenv("PALACE_ADMINUI_FEATURE_ENABLE_AUTO_LIST", "false") + monkeypatch.setenv("PALACE_ADMINUI_FEATURE_SHOW_CIRC_EVENTS_DOWNLOAD", "false") + monkeypatch.setenv("PALACE_ADMINUI_FEATURE_REPORTS_ONLY_FOR_SYSADMINS", "false") + + with ( + patch( + "palace.manager.api.admin.config.Configuration.admin_feature_flags" + ) as admin_feature_flags, + admin_ctrl_fixture.ctrl.app.test_request_context("/admin"), + ): + # Use fresh feature flags, instead of using a cached value. + admin_feature_flags.return_value = AdminClientFeatureFlags() + assert ( + "Some `environment_override_warning_fields` are overridden in the environment. Please " + in caplog.text + ) + assert "enable_auto_list" in caplog.text + assert "show_circ_events_download" in caplog.text + assert "reports_only_for_sysadmins" not in caplog.text + + response = admin_ctrl_fixture.manager.admin_view_controller(None, None) + assert 200 == response.status_code + html = response.get_data(as_text=True) + + match = html_feature_flags_re.search(html) + assert match is not None + + feature_flags: str = match.groups(0)[0] # type: ignore[assignment] + assert '"enableAutoList": true' in feature_flags + assert '"showCircEventsDownload": true' in feature_flags + assert '"reportsOnlyForSysadmins": false' in feature_flags diff --git a/tests/manager/service/test_configuration.py b/tests/manager/service/test_configuration.py index 068b610672..c6e02be149 100644 --- a/tests/manager/service/test_configuration.py +++ b/tests/manager/service/test_configuration.py @@ -7,6 +7,7 @@ from palace.manager.core.config import CannotLoadConfiguration from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.logging.configuration import LogLevel from palace.manager.util.settings import ServiceConfigurationWithLimitedEnvOverride if TYPE_CHECKING: @@ -29,13 +30,17 @@ def __init__(self, type: str, monkeypatch: MonkeyPatch, fs: FakeFilesystem): self.fs = fs # Make sure the environment is empty - self.monkeypatch.delenv("MOCK_STRING_WITHOUT_DEFAULT", raising=False) - self.monkeypatch.delenv("MOCK_INT_TYPE", raising=False) - self.monkeypatch.delenv("MOCK_STRING_WITH_DEFAULT", raising=False) + self.reset( + ["MOCK_STRING_WITHOUT_DEFAULT", "MOCK_INT_TYPE", "MOCK_STRING_WITH_DEFAULT"] + ) # Make sure the .env file is empty self.env_file = fs.create_file(".env", contents="") + def reset(self, keys: list[str]): + for key in keys: + self.monkeypatch.delenv(key, raising=False) + def set(self, key: str, value: str): if self.type == "env": self.set_env(key, value) @@ -174,14 +179,75 @@ def test_environment_override_warning_fields( caplog: pytest.LogCaptureFixture, service_configuration_fixture: ServiceConfigurationFixture, ): + caplog.set_level(LogLevel.warning) + class MockConfiguration(ServiceConfigurationWithLimitedEnvOverride): - field: bool = True - warning_field1: bool = True + field1: bool = True warning_field2: bool = True class Config: env_prefix = "MOCK_" environment_override_warning_fields = { - "warning_field1", "warning_field2", } + + service_configuration_fixture.reset(["MOCK_FIELD1", "MOCK_WARNING_FIELD2"]) + config1 = MockConfiguration() + + assert config1.field1 is True + assert config1.warning_field2 is True + assert ( + "Some `environment_override_warning_fields` are overridden in the environment." + not in caplog.text + ) + assert "warning_field2" not in caplog.text + assert "field1" not in caplog.text + + service_configuration_fixture.set("MOCK_FIELD1", "false") + service_configuration_fixture.set("MOCK_WARNING_FIELD2", "false") + config2 = MockConfiguration() + + assert config2.field1 is False + assert config2.warning_field2 is True + assert ( + "Some `environment_override_warning_fields` are overridden in the environment." + in caplog.text + ) + assert "warning_field2" in caplog.text + assert "field1" not in caplog.text + + service_configuration_fixture.reset(["MOCK_FIELD1", "MOCK_WARNING_FIELD2"]) + + def test_environment_override_error_field( + self, + service_configuration_fixture: ServiceConfigurationFixture, + ): + class MockConfiguration(ServiceConfigurationWithLimitedEnvOverride): + field1: bool = True + error_field2: bool = True + + class Config: + env_prefix = "MOCK_" + environment_override_error_fields = { + "error_field2", + } + + service_configuration_fixture.reset(["MOCK_FIELD1", "MOCK_ERROR_FIELD2"]) + config1 = MockConfiguration() + + assert config1.field1 is True + assert config1.error_field2 is True + + service_configuration_fixture.set("MOCK_FIELD1", "false") + service_configuration_fixture.set("MOCK_ERROR_FIELD2", "false") + with pytest.raises(CannotLoadConfiguration) as exc_info: + MockConfiguration() + + assert ( + "Some `environment_override_error_fields` are overridden in the environment." + in str(exc_info.value) + ) + assert "error_field2" in str(exc_info.value) + assert "field1" not in str(exc_info.value) + + service_configuration_fixture.reset(["MOCK_FIELD1", "MOCK_ERROR_FIELD2"]) From 179766e309a0493eeeda9183a08729e34acb3385 Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Wed, 12 Jun 2024 09:29:09 -0400 Subject: [PATCH 5/8] Appease mypy. --- tests/manager/api/admin/controller/test_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/manager/api/admin/controller/test_view.py b/tests/manager/api/admin/controller/test_view.py index aba91bac75..1a5814e4d9 100644 --- a/tests/manager/api/admin/controller/test_view.py +++ b/tests/manager/api/admin/controller/test_view.py @@ -187,7 +187,7 @@ def test_feature_flags_defaults( match = html_feature_flags_re.search(html) assert match is not None - feature_flags = match.groups(0)[0] + feature_flags: str = match.groups(0)[0] # type: ignore[assignment] assert '"enableAutoList": true' in feature_flags assert '"showCircEventsDownload": true' in feature_flags assert '"reportsOnlyForSysadmins": true' in feature_flags From 13a21c4544a3dd44e006d2a0c293b4ec71f3d556 Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Wed, 12 Jun 2024 10:51:59 -0400 Subject: [PATCH 6/8] Review feedback: remove commented code. --- src/palace/manager/api/admin/controller/view.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/palace/manager/api/admin/controller/view.py b/src/palace/manager/api/admin/controller/view.py index ab6a819b59..54c1de8be6 100644 --- a/src/palace/manager/api/admin/controller/view.py +++ b/src/palace/manager/api/admin/controller/view.py @@ -67,9 +67,6 @@ def __call__(self, collection, book, path=None): admin_js = AdminClientConfig.lookup_asset_url(key="admin_js") admin_css = AdminClientConfig.lookup_asset_url(key="admin_css") - # # We always have local_analytics - # show_circ_events_download = True - response = Response( flask.render_template_string( admin_template, From 4b9c2726ede2aa348c6e0a8d79b477d2fbc24e41 Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Wed, 12 Jun 2024 10:50:32 -0400 Subject: [PATCH 7/8] Review feedback: put new settings class with parent. --- src/palace/manager/api/admin/config.py | 4 +++- src/palace/manager/service/analytics/configuration.py | 4 +++- src/palace/manager/service/celery/configuration.py | 4 +++- src/palace/manager/service/configuration/__init__.py | 1 + .../configuration/limited_env_override.py} | 4 +++- .../service_configuration.py} | 0 src/palace/manager/service/email/configuration.py | 4 +++- src/palace/manager/service/fcm/configuration.py | 4 +++- src/palace/manager/service/logging/configuration.py | 4 +++- src/palace/manager/service/search/configuration.py | 4 +++- src/palace/manager/service/sitewide.py | 4 +++- src/palace/manager/service/storage/configuration.py | 4 +++- tests/fixtures/config.py | 4 +++- tests/manager/service/storage/test_s3.py | 4 +++- tests/manager/service/test_configuration.py | 8 ++++++-- 15 files changed, 43 insertions(+), 14 deletions(-) create mode 100644 src/palace/manager/service/configuration/__init__.py rename src/palace/manager/{util/settings.py => service/configuration/limited_env_override.py} (98%) rename src/palace/manager/service/{configuration.py => configuration/service_configuration.py} (100%) diff --git a/src/palace/manager/api/admin/config.py b/src/palace/manager/api/admin/config.py index 570a4ea3fe..3ac5c1296d 100644 --- a/src/palace/manager/api/admin/config.py +++ b/src/palace/manager/api/admin/config.py @@ -5,10 +5,12 @@ from pydantic import Field from requests import RequestException +from palace.manager.service.configuration.limited_env_override import ( + ServiceConfigurationWithLimitedEnvOverride, +) from palace.manager.util.flask_util import _snake_to_camel_case from palace.manager.util.http import HTTP, RequestNetworkException from palace.manager.util.log import LoggerMixin -from palace.manager.util.settings import ServiceConfigurationWithLimitedEnvOverride class AdminClientFeatureFlags(ServiceConfigurationWithLimitedEnvOverride): diff --git a/src/palace/manager/service/analytics/configuration.py b/src/palace/manager/service/analytics/configuration.py index ef3e87500d..56895b5bd0 100644 --- a/src/palace/manager/service/analytics/configuration.py +++ b/src/palace/manager/service/analytics/configuration.py @@ -1,4 +1,6 @@ -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class AnalyticsConfiguration(ServiceConfiguration): diff --git a/src/palace/manager/service/celery/configuration.py b/src/palace/manager/service/celery/configuration.py index ebe531e1cb..fe686d5bd8 100644 --- a/src/palace/manager/service/celery/configuration.py +++ b/src/palace/manager/service/celery/configuration.py @@ -2,7 +2,9 @@ from pydantic import RedisDsn -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class CeleryConfiguration(ServiceConfiguration): diff --git a/src/palace/manager/service/configuration/__init__.py b/src/palace/manager/service/configuration/__init__.py new file mode 100644 index 0000000000..9d48db4f9f --- /dev/null +++ b/src/palace/manager/service/configuration/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/src/palace/manager/util/settings.py b/src/palace/manager/service/configuration/limited_env_override.py similarity index 98% rename from src/palace/manager/util/settings.py rename to src/palace/manager/service/configuration/limited_env_override.py index 31a9eac63c..1dc7f87948 100644 --- a/src/palace/manager/util/settings.py +++ b/src/palace/manager/service/configuration/limited_env_override.py @@ -6,7 +6,9 @@ from pydantic.fields import ModelField from palace.manager.core.config import CannotLoadConfiguration -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) from palace.manager.util.log import LoggerMixin diff --git a/src/palace/manager/service/configuration.py b/src/palace/manager/service/configuration/service_configuration.py similarity index 100% rename from src/palace/manager/service/configuration.py rename to src/palace/manager/service/configuration/service_configuration.py diff --git a/src/palace/manager/service/email/configuration.py b/src/palace/manager/service/email/configuration.py index 5f160f9b0c..cfe2800df3 100644 --- a/src/palace/manager/service/email/configuration.py +++ b/src/palace/manager/service/email/configuration.py @@ -1,6 +1,8 @@ from pydantic import EmailStr, PositiveInt -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class EmailConfiguration(ServiceConfiguration): diff --git a/src/palace/manager/service/fcm/configuration.py b/src/palace/manager/service/fcm/configuration.py index d2b722408f..bf5309ca41 100644 --- a/src/palace/manager/service/fcm/configuration.py +++ b/src/palace/manager/service/fcm/configuration.py @@ -1,6 +1,8 @@ from pathlib import Path -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class FcmConfiguration(ServiceConfiguration): diff --git a/src/palace/manager/service/logging/configuration.py b/src/palace/manager/service/logging/configuration.py index e47d199f5d..5c8b5c6502 100644 --- a/src/palace/manager/service/logging/configuration.py +++ b/src/palace/manager/service/logging/configuration.py @@ -9,7 +9,9 @@ from pydantic import PositiveInt, validator from watchtower import DEFAULT_LOG_STREAM_NAME -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) # TODO: Remove this when we drop support for Python 3.10 if sys.version_info >= (3, 11): diff --git a/src/palace/manager/service/search/configuration.py b/src/palace/manager/service/search/configuration.py index e953259f55..d9c9b6b8d5 100644 --- a/src/palace/manager/service/search/configuration.py +++ b/src/palace/manager/service/search/configuration.py @@ -1,6 +1,8 @@ from pydantic import AnyHttpUrl -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class SearchConfiguration(ServiceConfiguration): diff --git a/src/palace/manager/service/sitewide.py b/src/palace/manager/service/sitewide.py index 771866097c..3fad5d7328 100644 --- a/src/palace/manager/service/sitewide.py +++ b/src/palace/manager/service/sitewide.py @@ -4,7 +4,9 @@ from pydantic import AnyHttpUrl, NonNegativeInt, validator -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class SitewideConfiguration(ServiceConfiguration): diff --git a/src/palace/manager/service/storage/configuration.py b/src/palace/manager/service/storage/configuration.py index bf736cc6d6..bc4334200e 100644 --- a/src/palace/manager/service/storage/configuration.py +++ b/src/palace/manager/service/storage/configuration.py @@ -1,7 +1,9 @@ import boto3 from pydantic import AnyHttpUrl, parse_obj_as, validator -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) class StorageConfiguration(ServiceConfiguration): diff --git a/tests/fixtures/config.py b/tests/fixtures/config.py index b2d1518aee..e45bb2a93c 100644 --- a/tests/fixtures/config.py +++ b/tests/fixtures/config.py @@ -6,7 +6,9 @@ from pydantic import AnyUrl, Extra from typing_extensions import Self -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) @dataclasses.dataclass diff --git a/tests/manager/service/storage/test_s3.py b/tests/manager/service/storage/test_s3.py index 6ac9fe00de..8e4ac42f07 100644 --- a/tests/manager/service/storage/test_s3.py +++ b/tests/manager/service/storage/test_s3.py @@ -12,7 +12,9 @@ from pydantic import AnyHttpUrl from palace.manager.core.config import CannotLoadConfiguration -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) from palace.manager.service.storage.container import Storage from palace.manager.service.storage.s3 import S3Service from tests.fixtures.config import FixtureTestUrlConfiguration diff --git a/tests/manager/service/test_configuration.py b/tests/manager/service/test_configuration.py index c6e02be149..21b08aab16 100644 --- a/tests/manager/service/test_configuration.py +++ b/tests/manager/service/test_configuration.py @@ -6,9 +6,13 @@ from pyfakefs.fake_filesystem import FakeFilesystem from palace.manager.core.config import CannotLoadConfiguration -from palace.manager.service.configuration import ServiceConfiguration +from palace.manager.service.configuration.limited_env_override import ( + ServiceConfigurationWithLimitedEnvOverride, +) +from palace.manager.service.configuration.service_configuration import ( + ServiceConfiguration, +) from palace.manager.service.logging.configuration import LogLevel -from palace.manager.util.settings import ServiceConfigurationWithLimitedEnvOverride if TYPE_CHECKING: from pytest import MonkeyPatch From 6626f10cd83a1c086600240d44bc1cbe153bf1ca Mon Sep 17 00:00:00 2001 From: Tim DiLauro Date: Wed, 12 Jun 2024 11:45:13 -0400 Subject: [PATCH 8/8] Fix some more types. --- .../service/configuration/limited_env_override.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/palace/manager/service/configuration/limited_env_override.py b/src/palace/manager/service/configuration/limited_env_override.py index 1dc7f87948..c85fe7644f 100644 --- a/src/palace/manager/service/configuration/limited_env_override.py +++ b/src/palace/manager/service/configuration/limited_env_override.py @@ -1,5 +1,4 @@ import functools -from collections.abc import Callable from typing import Any from pydantic.env_settings import BaseSettings, SettingsSourceCallable @@ -35,9 +34,9 @@ class Config: @classmethod def customise_sources( cls, - init_settings, - env_settings, - file_secret_settings, + init_settings: SettingsSourceCallable, + env_settings: SettingsSourceCallable, + file_secret_settings: SettingsSourceCallable, ) -> tuple[SettingsSourceCallable, ...]: # We have to wrap the environment settings source in our own function # so that we can report on/strip out fields that are not overridable @@ -49,13 +48,13 @@ def customise_sources( ) -def _env_var_for(field: ModelField) -> str | None: - env_prefix = field.model_config.env_prefix # type: ignore[attr-defined] +def _env_var_for(field: ModelField) -> str: + env_prefix = field.model_config.env_prefix or "" # type: ignore[attr-defined] return (env_prefix + field.name).upper() def _restrict_environment( - env_settings: Callable[[BaseSettings], dict[str, Any]], settings: BaseSettings + env_settings: SettingsSourceCallable, settings: BaseSettings ) -> dict[str, Any]: """Limit environment variables to those not restricted by the `environment_override_*` settings.