Move storage integrations to be configured as services (PP-95)

README.md

@@ -151,17 +151,42 @@ CREATE USER palace with password 'test';
 grant all privileges on database circ to palace;
 ```
 
-#### Environment variables
+### Environment variables
 
-##### Database
+#### Database
 
 To let the application know which database to use, set the `SIMPLIFIED_PRODUCTION_DATABASE` environment variable.
 
 ```sh
 export SIMPLIFIED_PRODUCTION_DATABASE="postgresql://palace:test@localhost:5432/circ"
 ```
 
-##### Patron `Basic Token` authentication
+#### Storage Service
+
+The application optionally uses a s3 compatible storage service to store files. To configure the application to use
+a storage service, you can set the following environment variables:
+
+- `PALACE_STORAGE_PUBLIC_ACCESS_BUCKET`: Required if you want to use the storage service to serve files directly to
+  users. This is the name of the bucket that will be used to serve files. This bucket should be configured to allow
+  public access to the files.
+- `PALACE_STORAGE_ANALYTICS_BUCKET`: Required if you want to use the storage service to store analytics data.
+- `PALACE_STORAGE_ACCESS_KEY`: The access key (optional).
+    - If this key is set it will be passed to boto3 when connecting to the storage service.
+    - If it is not set boto3 will attempt to find credentials as outlined in their
+      [documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials).
+- `PALACE_STORAGE_SECRET_KEY`: The secret key (optional).
+- `PALACE_STORAGE_REGION`: The AWS region of the storage service (optional).
+- `PALACE_STORAGE_ENDPOINT_URL`: The endpoint of the storage service (optional). This is used if you are using a
+  s3 compatible storage service like [minio](https://min.io/).
+- `PALACE_STORAGE_URL_TEMPLATE`: The url template to use when generating urls for files stored in the storage service
+  (optional).
+    - The default value is `https://{bucket}.s3.{region}.amazonaws.com/{key}`.
+    - The following variables can be used in the template:
+        - `{bucket}`: The name of the bucket.
+        - `{key}`: The key of the file.
+        - `{region}`: The region of the storage service.
+
+#### Patron `Basic Token` authentication
 
 Enables/disables patron "basic token" authentication through setting the designated environment variable to any
 (case-insensitive) value of "true"/"yes"/"on"/"1" or "false"/"no"/"off"/"0", respectively.
@@ -172,7 +197,7 @@ If the value is the empty string or the variable is not present in the environme
 export SIMPLIFIED_ENABLE_BASIC_TOKEN_AUTH=true
 ```
 
-##### Firebase Cloud Messaging
+#### Firebase Cloud Messaging
 
 For Firebase Cloud Messaging (FCM) support (e.g., for notifications), `one` (and only one) of the following should be set:
 - `SIMPLIFIED_FCM_CREDENTIALS_JSON` - the JSON-format Google Cloud Platform (GCP) service account key  or
@@ -191,7 +216,7 @@ export SIMPLIFIED_FCM_CREDENTIALS_FILE="/opt/credentials/fcm_credentials.json"
 The FCM credentials can be downloaded once a Google Service account has been created.
 More details in the [FCM documentation](https://firebase.google.com/docs/admin/setup#set-up-project-and-service-account)
 
-### Email sending
+#### Email
 
 To use the features that require sending emails, for example to reset the password for logged-out users, you will need
 to have a working SMTP server and set some environment variables:
@@ -204,7 +229,7 @@ export SIMPLIFIED_MAIL_PASSWORD=password
 export [email protected]
 ```
 
-### Running the Application
+## Running the Application
 
 As mentioned in the [pyenv](#pyenv) section, the `poetry` tool should be executed under a virtual environment
 in order to guarantee that it will use the Python version you expect. To use a particular Python version,

api/admin/config.py

@@ -18,7 +18,7 @@ class Configuration:
 
     APP_NAME = "Palace Collection Manager"
     PACKAGE_NAME = "@thepalaceproject/circulation-admin"
-    PACKAGE_VERSION = "1.8.0"
+    PACKAGE_VERSION = "1.9.0"
 
     STATIC_ASSETS = {
         "admin_js": "circulation-admin.js",

api/admin/controller/__init__.py

@@ -50,7 +50,6 @@ def setup_admin_controllers(manager: CirculationManager):
     from api.admin.controller.sitewide_settings import (
         SitewideConfigurationSettingsController,
     )
-    from api.admin.controller.storage_services import StorageServicesController
     from api.admin.controller.timestamps import TimestampsController
     from api.admin.controller.view import ViewController
     from api.admin.controller.work_editor import WorkController
@@ -100,7 +99,6 @@ def setup_admin_controllers(manager: CirculationManager):
         SearchServiceSelfTestsController(manager)
     )
     manager.admin_search_services_controller = SearchServicesController(manager)
-    manager.admin_storage_services_controller = StorageServicesController(manager)
     manager.admin_catalog_services_controller = CatalogServicesController(manager)
     manager.admin_announcement_service = AnnouncementSettings(manager)
     manager.admin_search_controller = AdminSearchController(manager)

api/admin/controller/analytics_services.py

@@ -6,8 +6,7 @@
 from api.google_analytics_provider import GoogleAnalyticsProvider
 from api.s3_analytics_provider import S3AnalyticsProvider
 from core.local_analytics_provider import LocalAnalyticsProvider
-from core.model import ExternalIntegration, ExternalIntegrationLink
-from core.s3 import S3UploaderConfiguration
+from core.model import ExternalIntegration
 from core.util import first_or_default
 from core.util.problem_detail import ProblemDetail
 
@@ -33,11 +32,6 @@ def update_protocol_settings(self):
             ]
         )
 
-        if s3_analytics_provider:
-            s3_analytics_provider[
-                "settings"
-            ] = S3AnalyticsProvider.get_storage_settings(self._db)
-
     def process_analytics_services(self):
         if flask.request.method == "GET":
             return self.process_get()
@@ -101,14 +95,6 @@ def process_post(self):
 
         service.name = name
 
-        external_integration_link = self._set_storage_external_integration_link(
-            service,
-            ExternalIntegrationLink.ANALYTICS,
-            S3UploaderConfiguration.ANALYTICS_BUCKET_KEY,
-        )
-        if isinstance(external_integration_link, ProblemDetail):
-            return external_integration_link
-
         if is_new:
             return Response(str(service.id), 201)
         else:

api/admin/controller/catalog_services.py

@@ -6,15 +6,12 @@
 from api.admin.problem_details import (
     CANNOT_CHANGE_PROTOCOL,
     INTEGRATION_NAME_ALREADY_IN_USE,
-    MISSING_INTEGRATION,
     MISSING_SERVICE,
     MULTIPLE_SERVICES_FOR_LIBRARY,
     UNKNOWN_PROTOCOL,
 )
 from core.marc import MARCExporter
-from core.model import ExternalIntegration, get_one, get_one_or_create
-from core.model.configuration import ExternalIntegrationLink
-from core.s3 import S3UploaderConfiguration
+from core.model import ExternalIntegration, get_one
 from core.util.problem_detail import ProblemDetail
 
 
@@ -25,10 +22,6 @@ def __init__(self, manager):
         self.protocols = self._get_integration_protocols(
             service_apis, protocol_name_attr="NAME"
         )
-        self.update_protocol_settings()
-
-    def update_protocol_settings(self):
-        self.protocols[0]["settings"] = [MARCExporter.get_storage_settings(self._db)]
 
     def process_catalog_services(self):
         self.require_system_admin()
@@ -42,7 +35,6 @@ def process_get(self):
         services = self._get_integration_info(
             ExternalIntegration.CATALOG_GOAL, self.protocols
         )
-        self.update_protocol_settings()
         return dict(
             catalog_services=services,
             protocols=self.protocols,
@@ -91,10 +83,6 @@ def process_post(self):
         if isinstance(result, ProblemDetail):
             return result
 
-        external_integration_link = self._set_external_integration_link(service)
-        if isinstance(external_integration_link, ProblemDetail):
-            return external_integration_link
-
         library_error = self.check_libraries(service)
         if library_error:
             self._db.rollback()
@@ -105,39 +93,6 @@ def process_post(self):
         else:
             return Response(str(service.id), 200)
 
-    def _set_external_integration_link(self, service):
-        """Either set or delete the external integration link between the
-        service and the storage integration.
-        """
-        mirror_integration_id = flask.request.form.get("mirror_integration_id")
-
-        # If no storage integration was selected, then delete the existing
-        # external integration link.
-        current_integration_link, ignore = get_one_or_create(
-            self._db,
-            ExternalIntegrationLink,
-            library_id=None,
-            external_integration_id=service.id,
-            purpose=ExternalIntegrationLink.MARC,
-        )
-
-        if mirror_integration_id == self.NO_MIRROR_INTEGRATION:
-            if current_integration_link:
-                self._db.delete(current_integration_link)
-        else:
-            storage_integration = get_one(
-                self._db, ExternalIntegration, id=mirror_integration_id
-            )
-            # Only get storage integrations that have a MARC file option set
-            if (
-                not storage_integration
-                or not storage_integration.setting(
-                    S3UploaderConfiguration.MARC_BUCKET_KEY
-                ).value
-            ):
-                return MISSING_INTEGRATION
-            current_integration_link.other_integration_id = storage_integration.id
-
     def validate_form_fields(self, protocol):
         """Verify that the protocol which the user has selected is in the list
         of recognized protocol options."""

api/admin/routes.py

@@ -493,22 +493,6 @@ def search_service_self_tests(identifier):
     )
 
 
-@app.route("/admin/storage_services", methods=["GET", "POST"])
-@returns_json_or_response_or_problem_detail
-@requires_admin
-@requires_csrf_token
-def storage_services():
-    return app.manager.admin_storage_services_controller.process_services()
-
-
-@app.route("/admin/storage_service/<service_id>", methods=["DELETE"])
-@returns_json_or_response_or_problem_detail
-@requires_admin
-@requires_csrf_token
-def storage_service(service_id):
-    return app.manager.admin_storage_services_controller.process_delete(service_id)
-
-
 @app.route("/admin/catalog_services", methods=["GET", "POST"])
 @returns_json_or_response_or_problem_detail
 @requires_admin

api/app.py

@@ -17,6 +17,7 @@
     SessionManager,
     pg_advisory_lock,
 )
+from core.service.container import container_instance
 from core.util import LanguageCodes
 from core.util.cache import CachedData
 from scripts import InstanceInitializationScript
@@ -72,8 +73,9 @@ def initialize_circulation_manager():
         pass
     else:
         if getattr(app, "manager", None) is None:
+            container = container_instance()
             try:
-                app.manager = CirculationManager(app._db)
+                app.manager = CirculationManager(app._db, container)
             except Exception:
                 logging.exception("Error instantiating circulation manager!")
                 raise

api/circulation.py

@@ -15,7 +15,6 @@
     Generic,
     List,
     Literal,
-    Optional,
     Tuple,
     Type,
     TypeVar,
@@ -25,7 +24,7 @@
 from flask import Response
 from flask_babel import lazy_gettext as _
 from pydantic import PositiveInt
-from sqlalchemy.orm import Query, Session
+from sqlalchemy.orm import Query
 
 from core.analytics import Analytics
 from core.config import CannotLoadConfiguration

api/controller.py

@@ -87,6 +87,7 @@
 from core.opds2 import AcquisitonFeedOPDS2
 from core.opensearch import OpenSearchDocument
 from core.query.playtime_entries import PlaytimeEntries
+from core.service.container import Services
 from core.user_profile import ProfileController as CoreProfileController
 from core.util.authentication_for_opds import AuthenticationForOPDSDocument
 from core.util.datetime_helpers import utc_now
@@ -166,7 +167,6 @@
     from api.admin.controller.sitewide_settings import (
         SitewideConfigurationSettingsController,
     )
-    from api.admin.controller.storage_services import StorageServicesController
     from api.admin.controller.timestamps import TimestampsController
     from api.admin.controller.view import ViewController
     from api.admin.controller.work_editor import WorkController as AdminWorkController
@@ -220,14 +220,14 @@ class CirculationManager:
     admin_logging_services_controller: LoggingServicesController
     admin_search_service_self_tests_controller: SearchServiceSelfTestsController
     admin_search_services_controller: SearchServicesController
-    admin_storage_services_controller: StorageServicesController
     admin_catalog_services_controller: CatalogServicesController
     admin_announcement_service: AnnouncementSettings
     admin_search_controller: AdminSearchController
     admin_view_controller: ViewController
 
-    def __init__(self, _db):
+    def __init__(self, _db, services: Services):
         self._db = _db
+        self.services = services
         self.site_configuration_last_update = (
             Configuration.site_configuration_last_update(self._db, timeout=0)
         )

api/discovery/registration_script.py

@@ -67,7 +67,7 @@ def do_run(
         # Set up an application context so we have access to url_for.
         from api.app import app
 
-        app.manager = manager or CirculationManager(self._db)
+        app.manager = manager or CirculationManager(self._db, self.services)
         base_url = ConfigurationSetting.sitewide(
             self._db, Configuration.BASE_URL_KEY
         ).value