From ea60979a2f03f461a2606f94f5d481c79d4c9bc1 Mon Sep 17 00:00:00 2001 From: Jonathan Green Date: Sat, 5 Oct 2024 18:47:01 -0300 Subject: [PATCH] OPDS ODL API changes for UL Feed (PP-1769) (#2095) --- pyproject.toml | 2 + src/palace/manager/api/circulation_manager.py | 6 +- .../api/controller/odl_notification.py | 77 ++- src/palace/manager/api/odl/api.py | 494 ++++++-------- src/palace/manager/api/odl/auth.py | 200 ++++-- src/palace/manager/api/odl/importer.py | 6 +- src/palace/manager/opds/__init__.py | 0 src/palace/manager/opds/authentication.py | 64 ++ src/palace/manager/opds/base.py | 150 ++++ src/palace/manager/opds/lcp/__init__.py | 0 src/palace/manager/opds/lcp/license.py | 120 ++++ src/palace/manager/opds/lcp/status.py | 113 ++++ src/palace/manager/opds/odl/__init__.py | 0 src/palace/manager/opds/odl/info.py | 78 +++ src/palace/manager/opds/odl/odl.py | 34 + src/palace/manager/opds/opds.py | 22 + src/palace/manager/sqlalchemy/constants.py | 1 + tests/files/opds/lcp/license/fb.json | 48 ++ tests/files/opds/lcp/license/ul.json | 51 ++ tests/files/opds/lcp/status/fb-active.json | 39 ++ .../files/opds/lcp/status/fb-book-adobe.json | 57 ++ .../opds/lcp/status/fb-early-return.json | 1 + tests/files/opds/lcp/status/ul-active.json | 1 + tests/files/opds/lcp/status/ul-returned.json | 1 + .../odl/info/feedbooks-ab-checked-out.json | 36 + .../odl/info/feedbooks-ab-loan-limited.json | 36 + .../info/feedbooks-ab-not-checked-out.json | 28 + .../opds/odl/info/feedbooks-book-adept.json | 28 + .../odl/info/feedbooks-book-unavailable.json | 28 + tests/files/opds/odl/info/ul-ab.json | 1 + tests/files/opds/odl/info/ul-book.json | 1 + tests/fixtures/odl.py | 108 ++- .../manager/api/controller/test_odl_notify.py | 190 +++--- tests/manager/api/odl/test_api.py | 638 ++++++++---------- tests/manager/api/odl/test_auth.py | 232 ++++--- tests/manager/api/odl/test_importer.py | 10 +- tests/manager/opds/__init__.py | 0 tests/manager/opds/lcp/__init__.py | 0 tests/manager/opds/lcp/test_license.py | 21 + tests/manager/opds/lcp/test_status.py | 24 + tests/manager/opds/odl/__init__.py | 0 tests/manager/opds/odl/test_info.py | 29 + tests/manager/opds/test_base.py | 161 +++++ tests/mocks/mock.py | 13 +- tests/mocks/odl.py | 28 +- 45 files changed, 2249 insertions(+), 928 deletions(-) create mode 100644 src/palace/manager/opds/__init__.py create mode 100644 src/palace/manager/opds/authentication.py create mode 100644 src/palace/manager/opds/base.py create mode 100644 src/palace/manager/opds/lcp/__init__.py create mode 100644 src/palace/manager/opds/lcp/license.py create mode 100644 src/palace/manager/opds/lcp/status.py create mode 100644 src/palace/manager/opds/odl/__init__.py create mode 100644 src/palace/manager/opds/odl/info.py create mode 100644 src/palace/manager/opds/odl/odl.py create mode 100644 src/palace/manager/opds/opds.py create mode 100644 tests/files/opds/lcp/license/fb.json create mode 100644 tests/files/opds/lcp/license/ul.json create mode 100644 tests/files/opds/lcp/status/fb-active.json create mode 100644 tests/files/opds/lcp/status/fb-book-adobe.json create mode 100644 tests/files/opds/lcp/status/fb-early-return.json create mode 100644 tests/files/opds/lcp/status/ul-active.json create mode 100644 tests/files/opds/lcp/status/ul-returned.json create mode 100644 tests/files/opds/odl/info/feedbooks-ab-checked-out.json create mode 100644 tests/files/opds/odl/info/feedbooks-ab-loan-limited.json create mode 100644 tests/files/opds/odl/info/feedbooks-ab-not-checked-out.json create mode 100644 tests/files/opds/odl/info/feedbooks-book-adept.json create mode 100644 tests/files/opds/odl/info/feedbooks-book-unavailable.json create mode 100644 tests/files/opds/odl/info/ul-ab.json create mode 100644 tests/files/opds/odl/info/ul-book.json create mode 100644 tests/manager/opds/__init__.py create mode 100644 tests/manager/opds/lcp/__init__.py create mode 100644 tests/manager/opds/lcp/test_license.py create mode 100644 tests/manager/opds/lcp/test_status.py create mode 100644 tests/manager/opds/odl/__init__.py create mode 100644 tests/manager/opds/odl/test_info.py create mode 100644 tests/manager/opds/test_base.py diff --git a/pyproject.toml b/pyproject.toml index 831fca98fb..b48d6e4244 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ module = [ "palace.manager.api.controller.circulation_manager", "palace.manager.api.controller.loan", "palace.manager.api.controller.marc", + "palace.manager.api.controller.odl_notification", "palace.manager.api.discovery.*", "palace.manager.api.enki", "palace.manager.api.lcp.hash", @@ -99,6 +100,7 @@ module = [ "palace.manager.core.selftest", "palace.manager.feed.*", "palace.manager.integration.*", + "palace.manager.opds.*", "palace.manager.scripts.initialization", "palace.manager.scripts.rotate_jwe_key", "palace.manager.scripts.search", diff --git a/src/palace/manager/api/circulation_manager.py b/src/palace/manager/api/circulation_manager.py index aabc6589f9..affdc6cb1f 100644 --- a/src/palace/manager/api/circulation_manager.py +++ b/src/palace/manager/api/circulation_manager.py @@ -357,7 +357,11 @@ def setup_one_time_controllers(self): self.profiles = ProfileController(self) self.patron_devices = DeviceTokensController(self) self.version = ApplicationVersionController() - self.odl_notification_controller = ODLNotificationController(self) + self.odl_notification_controller = ODLNotificationController( + self._db, + self, + self.services.integration_registry.license_providers(), + ) self.patron_auth_token = PatronAuthTokenController(self) self.playtime_entries = PlaytimeEntriesController(self) diff --git a/src/palace/manager/api/controller/odl_notification.py b/src/palace/manager/api/controller/odl_notification.py index a5a8ecefef..faea4135a6 100644 --- a/src/palace/manager/api/controller/odl_notification.py +++ b/src/palace/manager/api/controller/odl_notification.py @@ -1,42 +1,81 @@ from __future__ import annotations -import json +from typing import TYPE_CHECKING import flask from flask import Response from flask_babel import lazy_gettext as _ +from pydantic import ValidationError +from sqlalchemy.orm import Session -from palace.manager.api.controller.circulation_manager import ( - CirculationManagerController, -) from palace.manager.api.odl.api import OPDS2WithODLApi from palace.manager.api.problem_details import ( INVALID_LOAN_FOR_ODL_NOTIFICATION, NO_ACTIVE_LOAN, ) +from palace.manager.core.problem_details import INVALID_INPUT +from palace.manager.opds.lcp.status import LoanStatus +from palace.manager.service.integration_registry.license_providers import ( + LicenseProvidersRegistry, +) +from palace.manager.sqlalchemy.model.library import Library from palace.manager.sqlalchemy.model.patron import Loan from palace.manager.sqlalchemy.util import get_one +from palace.manager.util.log import LoggerMixin +from palace.manager.util.problem_detail import ProblemDetail + +if TYPE_CHECKING: + from palace.manager.api.circulation_manager import CirculationManager -class ODLNotificationController(CirculationManagerController): +class ODLNotificationController(LoggerMixin): """Receive notifications from an ODL distributor when the status of a loan changes. """ - def notify(self, loan_id): - library = flask.request.library - status_doc = flask.request.data - loan = get_one(self._db, Loan, id=loan_id) + def __init__( + self, + db: Session, + manager: CirculationManager, + registry: LicenseProvidersRegistry, + ) -> None: + self.db = db + self.manager = manager + self.registry = registry - if not loan: - return NO_ACTIVE_LOAN.detailed(_("No loan was found for this identifier.")) - - collection = loan.license_pool.collection - if collection.protocol != OPDS2WithODLApi.label(): - return INVALID_LOAN_FOR_ODL_NOTIFICATION - - api = self.manager.circulation_apis[library.id].api_for_license_pool( + def get_api(self, library: Library, loan: Loan) -> OPDS2WithODLApi: + return self.manager.circulation_apis[library.id].api_for_license_pool( # type: ignore[no-any-return] loan.license_pool ) - api.update_loan(loan, json.loads(status_doc)) - return Response(_("Success"), 200) + + def notify(self, loan_id: int) -> Response | ProblemDetail: + library = flask.request.library # type: ignore[attr-defined] + status_doc_json = flask.request.data + loan = get_one(self.db, Loan, id=loan_id) + + try: + status_doc = LoanStatus.model_validate_json(status_doc_json) + except ValidationError as e: + self.log.exception(f"Unable to parse loan status document. {e}") + return INVALID_INPUT + + # We don't have a record of this loan. This likely means that the loan has been returned + # and our local record has been deleted. This is expected, except in the case where the + # distributor thinks the loan is still active. + if loan is None and status_doc.active: + return NO_ACTIVE_LOAN.detailed( + _("No loan was found for this identifier."), status_code=404 + ) + + if loan: + integration = loan.license_pool.collection.integration_configuration + if ( + not integration.protocol + or self.registry.get(integration.protocol) != OPDS2WithODLApi + ): + return INVALID_LOAN_FOR_ODL_NOTIFICATION + + api = self.get_api(library, loan) + api.update_loan(loan, status_doc) + + return Response(status=204) diff --git a/src/palace/manager/api/odl/api.py b/src/palace/manager/api/odl/api.py index d20d91c0ac..37f6acf116 100644 --- a/src/palace/manager/api/odl/api.py +++ b/src/palace/manager/api/odl/api.py @@ -4,12 +4,13 @@ import datetime import json import uuid -from functools import cached_property +from collections.abc import Callable +from functools import cached_property, partial from typing import Any, Literal -import dateutil from dependency_injector.wiring import Provide, inject from flask import url_for +from pydantic import ValidationError from sqlalchemy import or_ from sqlalchemy.orm import Session from uritemplate import URITemplate @@ -23,12 +24,14 @@ LoanInfo, PatronActivityCirculationAPI, RedirectFulfillment, + UrlFulfillment, ) from palace.manager.api.circulation_exceptions import ( AlreadyCheckedOut, AlreadyOnHold, CannotFulfill, CannotLoan, + CannotReturn, CurrentlyAvailable, FormatNotAvailable, HoldOnUnlimitedAccess, @@ -41,18 +44,18 @@ PatronLoanLimitReached, ) from palace.manager.api.lcp.hash import Hasher, HasherFactory -from palace.manager.api.odl.auth import ODLAuthenticatedGet +from palace.manager.api.odl.auth import OdlAuthenticatedRequest, OpdsWithOdlException from palace.manager.api.odl.constants import FEEDBOOKS_AUDIO from palace.manager.api.odl.settings import ( OPDS2AuthType, OPDS2WithODLLibrarySettings, OPDS2WithODLSettings, ) -from palace.manager.core.lcp.credential import ( - LCPCredentialFactory, - LCPHashedPassphrase, - LCPUnhashedPassphrase, -) +from palace.manager.core.exceptions import PalaceValueError +from palace.manager.core.lcp.credential import LCPCredentialFactory +from palace.manager.opds.base import BaseLink +from palace.manager.opds.lcp.license import LicenseDocument +from palace.manager.opds.lcp.status import LoanStatus from palace.manager.service.container import Services from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.datasource import DataSource @@ -70,7 +73,7 @@ class OPDS2WithODLApi( - ODLAuthenticatedGet, + OdlAuthenticatedRequest, PatronActivityCirculationAPI[OPDS2WithODLSettings, OPDS2WithODLLibrarySettings], ): """ODL (Open Distribution to Libraries) is a specification that allows @@ -82,35 +85,6 @@ class OPDS2WithODLApi( SET_DELIVERY_MECHANISM_AT = BaseCirculationAPI.FULFILL_STEP - # Possible status values in the License Status Document: - - # The license is available but the user hasn't fulfilled it yet. - READY_STATUS = "ready" - - # The license is available and has been fulfilled on at least one device. - ACTIVE_STATUS = "active" - - # The license has been revoked by the distributor. - REVOKED_STATUS = "revoked" - - # The license has been returned early by the user. - RETURNED_STATUS = "returned" - - # The license was returned early and was never fulfilled. - CANCELLED_STATUS = "cancelled" - - # The license has expired. - EXPIRED_STATUS = "expired" - - STATUS_VALUES = [ - READY_STATUS, - ACTIVE_STATUS, - REVOKED_STATUS, - RETURNED_STATUS, - CANCELLED_STATUS, - EXPIRED_STATUS, - ] - @classmethod def settings_class(cls) -> type[OPDS2WithODLSettings]: return OPDS2WithODLSettings @@ -188,96 +162,44 @@ def _url_for(self, *args: Any, **kwargs: Any) -> str: """Wrapper around flask's url_for to be overridden for tests.""" return url_for(*args, **kwargs) - def get_license_status_document(self, loan: Loan) -> dict[str, Any]: - """Get the License Status Document for a loan. - - For a new loan, create a local loan with no external identifier and - pass it in to this method. - - This will create the remote loan if one doesn't exist yet. The loan's - internal database id will be used to receive notifications from the - distributor when the loan's status changes. - """ - _db = Session.object_session(loan) - - if loan.external_identifier: - url = loan.external_identifier - else: - id = loan.license.identifier - checkout_id = str(uuid.uuid1()) - if self.collection is None: - raise ValueError(f"Collection not found: {self.collection_id}") - default_loan_period = self.collection.default_loan_period( - loan.patron.library - ) - - expires = utc_now() + datetime.timedelta(days=default_loan_period) - # The patron UUID is generated randomly on each loan, so the distributor - # doesn't know when multiple loans come from the same patron. - patron_id = str(uuid.uuid1()) - - library_short_name = loan.patron.library.short_name - - db = Session.object_session(loan) - patron = loan.patron - hasher = self._get_hasher() - - unhashed_pass: LCPUnhashedPassphrase = ( - self._credential_factory.get_patron_passphrase(db, patron) - ) - hashed_pass: LCPHashedPassphrase = unhashed_pass.hash(hasher) - self._credential_factory.set_hashed_passphrase(db, patron, hashed_pass) - encoded_pass: str = base64.b64encode(binascii.unhexlify(hashed_pass.hashed)) - - notification_url = self._url_for( - "odl_notify", - library_short_name=library_short_name, - loan_id=loan.id, - _external=True, - ) - - checkout_url = str(loan.license.checkout_url) - url_template = URITemplate(checkout_url) - url = url_template.expand( - id=str(id), - checkout_id=checkout_id, - patron_id=patron_id, - expires=expires.isoformat(), - notification_url=notification_url, - passphrase=encoded_pass, - hint=self.settings.passphrase_hint, - hint_url=self.settings.passphrase_hint_url, - ) - + def _request_loan_status( + self, method: str, url: str, ignored_problem_types: list[str] | None = None + ) -> LoanStatus: try: - response = self._get(url, allowed_response_codes=["2xx"]) - except BadResponseException as e: - response = e.response - header_string = ", ".join( - {f"{k}: {v}" for k, v in response.headers.items()} - ) - response_string = ( - response.text - if len(response.text) < 100 - else response.text[:100] + "..." - ) - self.log.error( - f"Error getting License Status Document for loan ({loan.id}): Url '{url}' returned " - f"status code {response.status_code}. Expected 2XX. Response headers: {header_string}. " - f"Response content: {response_string}." + response = self._request(method, url, allowed_response_codes=["2xx"]) + status_doc = LoanStatus.model_validate_json(response.content) + except ValidationError as e: + self.log.exception( + f"Error validating Loan Status Document. '{url}' returned and invalid document. {e}" ) - raise - try: - status_doc = json.loads(response.content) - except ValueError as e: raise RemoteIntegrationException( - url, "License Status Document was not valid JSON." + url, "Loan Status Document not valid." ) from e - if status_doc.get("status") not in self.STATUS_VALUES: - raise RemoteIntegrationException( - url, "License Status Document had an unknown status value." - ) - return status_doc # type: ignore[no-any-return] + except BadResponseException as e: + response = e.response + error_message = f"Error requesting Loan Status Document. '{url}' returned status code {response.status_code}." + if isinstance(e, OpdsWithOdlException): + # It this problem type is explicitly ignored, we just raise the exception instead of proceeding with + # logging the information about it. The caller will handle the exception. + if ignored_problem_types and e.type in ignored_problem_types: + raise + error_message += f" Problem Detail: '{e.type}' - {e.title}" + if e.detail: + error_message += f" - {e.detail}" + else: + header_string = ", ".join( + {f"{k}: {v}" for k, v in response.headers.items()} + ) + response_string = ( + response.text + if len(response.text) < 100 + else response.text[:100] + "..." + ) + error_message += f" Response headers: {header_string}. Response content: {response_string}." + self.log.exception(error_message) + raise + + return status_doc def checkin(self, patron: Patron, pin: str, licensepool: LicensePool) -> None: """Return a loan early.""" @@ -300,46 +222,46 @@ def checkin(self, patron: Patron, pin: str, licensepool: LicensePool) -> None: def _checkin(self, loan: Loan) -> bool: _db = Session.object_session(loan) - doc = self.get_license_status_document(loan) - status = doc.get("status") - if status in [ - self.REVOKED_STATUS, - self.RETURNED_STATUS, - self.CANCELLED_STATUS, - self.EXPIRED_STATUS, - ]: - # This loan was already returned early or revoked by the distributor, or it expired. - self.update_loan(loan, doc) - raise NotCheckedOut() - - return_url = None - links = doc.get("links", []) - for link in links: - if link.get("rel") == "return": - return_url = link.get("href") - break - - if not return_url: - # The distributor didn't provide a link to return this loan. - # This may be because the book has already been fulfilled and - # must be returned through the DRM system. If that's true, the - # app will already be doing that on its own, so we'll silently - # do nothing. + if loan.external_identifier is None: + # We can't return a loan that doesn't have an external identifier. This should never happen + # but if it does, we log an error and delete the loan, so it doesn't stay on the patrons + # bookshelf forever. + self.log.error(f"Loan {loan.id} has no external identifier.") return False - # Hit the distributor's return link. - self._get(return_url) - # Get the status document again to make sure the return was successful, - # and if so update the pool availability and delete the local loan. - self.update_loan(loan) - - # At this point, if the loan still exists, something went wrong. - # However, it might be because the loan has already been fulfilled - # and must be returned through the DRM system, which the app will - # do on its own, so we can ignore the problem. - new_loan = get_one(_db, Loan, id=loan.id) - if new_loan: + doc = self._request_loan_status("GET", loan.external_identifier) + if not doc.active: + self.log.warning( + f"Loan {loan.id} was already returned early, revoked by the distributor, or it expired." + ) + self.update_loan(loan, doc) return False + + return_link = doc.links.get(rel="return", type=LoanStatus.content_type()) + if not return_link: + # The distributor didn't provide a link to return this loan. This means that the distributor + # does not support early returns, and the patron will have to wait until the loan expires. + raise CannotReturn() + + # The parameters for this link (if its templated) are defined here: + # https://readium.org/lcp-specs/releases/lsd/latest.html#34-returning-a-publication + # None of them are required, and often the link is not templated. But in the case + # of the open source LCP server, the link is templated, so we need to process the + # template before we can make the request. + return_url = return_link.href_templated({"name": "Palace Manager"}) + + # Hit the distributor's return link, and if it's successful, update the pool + # availability and delete the local loan. + doc = self._request_loan_status("PUT", return_url) + if doc.active: + # If the distributor says the loan is still active, we didn't return it, and + # something went wrong. We log an error and don't delete the loan, so the patron + # can try again later. + self.log.error( + f"Loan {loan.id} was not returned. The distributor says it's still active. {doc.model_dump_json()}" + ) + raise CannotReturn() + self.update_loan(loan, doc) return True def checkout( @@ -422,26 +344,59 @@ def _checkout( raise NoAvailableCopies() loan, ignore = license.loan_to(patron) + identifier = loan.license.identifier + checkout_id = str(uuid.uuid4()) + if self.collection is None: + raise PalaceValueError(f"Collection not found: {self.collection_id}") + default_loan_period = self.collection.default_loan_period(loan.patron.library) + + requested_expiry = utc_now() + datetime.timedelta(days=default_loan_period) + patron_id = patron.identifier_to_remote_service(licensepool.data_source) + library_short_name = loan.patron.library.short_name + + db = Session.object_session(loan) + patron = loan.patron + hasher = self._get_hasher() + + unhashed_pass = self._credential_factory.get_patron_passphrase(db, patron) + hashed_pass = unhashed_pass.hash(hasher) + self._credential_factory.set_hashed_passphrase(db, patron, hashed_pass) + encoded_pass = base64.b64encode(binascii.unhexlify(hashed_pass.hashed)) + + notification_url = self._url_for( + "odl_notify", + library_short_name=library_short_name, + loan_id=loan.id, + _external=True, + ) + + # We should never be able to get here if the license doesn't have a checkout_url, but + # we assert it anyway, to be sure we fail fast if it happens. + assert license.checkout_url is not None + url_template = URITemplate(license.checkout_url) + checkout_url = url_template.expand( + id=str(identifier), + checkout_id=checkout_id, + patron_id=patron_id, + expires=requested_expiry.isoformat(), + notification_url=notification_url, + passphrase=encoded_pass, + hint=self.settings.passphrase_hint, + hint_url=self.settings.passphrase_hint_url, + ) + try: - doc = self.get_license_status_document(loan) + doc = self._request_loan_status( + "POST", + checkout_url, + ignored_problem_types=[ + "http://opds-spec.org/odl/error/checkout/unavailable" + ], + ) except BadResponseException as e: _db.delete(loan) - response = e.response - # DeMarque sends "application/api-problem+json", but the ODL spec says we should - # expect "application/problem+json", so we need to check for both. - if response.headers.get("Content-Type") in [ - "application/api-problem+json", - "application/problem+json", - ]: - try: - json_response = response.json() - except ValueError: - json_response = {} - - if ( - json_response.get("type") - == "http://opds-spec.org/odl/error/checkout/unavailable" - ): + if isinstance(e, OpdsWithOdlException): + if e.type == "http://opds-spec.org/odl/error/checkout/unavailable": # TODO: This would be a good place to do an async availability update, since we know # the book is unavailable, when we thought it was available. For now, we know that # the license has no checkouts_available, so we do that update. @@ -450,35 +405,48 @@ def _checkout( raise NoAvailableCopies() raise - status = doc.get("status") - - if status not in [self.READY_STATUS, self.ACTIVE_STATUS]: + if not doc.active: # Something went wrong with this loan and we don't actually # have the book checked out. This should never happen. # Remove the loan we created. _db.delete(loan) raise CannotLoan() - links = doc.get("links", []) - external_identifier = None - for link in links: - if link.get("rel") == "self": - external_identifier = link.get("href") - break - if not external_identifier: + # We save the link to the loan status document in the loan's external_identifier field, so + # we are able to retrieve it later. + loan_status_document_link: BaseLink | None = doc.links.get( + rel="self", type=LoanStatus.content_type() + ) + + # The ODL spec requires that a 'self' link be present in the links section of the response. + # See: https://drafts.opds.io/odl-1.0.html#54-interacting-with-a-checkout-link + # However, the open source LCP license status server does not provide this link, so we make + # an extra request to try to get the information we need from the 'status' link in the license + # document, which the LCP server does provide. + # TODO: Raise this issue with LCP server maintainers, and try to get a fix in place. + # once that is done, we should be able to remove this fallback. + if not loan_status_document_link: + license_document_link = doc.links.get( + rel="license", type=LicenseDocument.content_type() + ) + if license_document_link: + response = self._request( + "GET", license_document_link.href, allowed_response_codes=["2xx"] + ) + license_doc = LicenseDocument.model_validate_json(response.content) + loan_status_document_link = license_doc.links.get( + rel="status", type=LoanStatus.content_type() + ) + + if not loan_status_document_link: _db.delete(loan) raise CannotLoan() - start = utc_now() - expires = doc.get("potential_rights", {}).get("end") - if expires: - expires = dateutil.parser.parse(expires) - # We need to set the start and end dates on our local loan since # the code that calls this only sets them when a new loan is created. - loan.start = start - loan.end = expires - loan.external_identifier = external_identifier + loan.start = utc_now() + loan.end = doc.potential_rights.end + loan.external_identifier = loan_status_document_link.href # We also need to update the remaining checkouts for the license. loan.license.checkout() @@ -508,52 +476,9 @@ def fulfill( return self._fulfill(loan, delivery_mechanism) @staticmethod - def _find_content_link_and_type( - links: list[dict[str, str]], - drm_scheme: str | None, - ) -> tuple[str | None, str | None]: - """Find a content link with the type information corresponding to the selected delivery mechanism. - - :param links: List of dict-like objects containing information about available links in the LCP license file - :param drm_scheme: Selected delivery mechanism DRM scheme - - :return: Two-tuple containing a content link and content type - """ - candidates = [] - for link in links: - # Depending on the format being served, the crucial information - # may be in 'manifest' or in 'license'. - if link.get("rel") not in ("manifest", "license"): - continue - href = link.get("href") - type = link.get("type") - candidates.append((href, type)) - - if len(candidates) == 0: - # No candidates - return None, None - - # For DeMarque audiobook content, we need to translate the type property - # to reflect what we have stored in our delivery mechanisms. - if drm_scheme == DeliveryMechanism.FEEDBOOKS_AUDIOBOOK_DRM: - drm_scheme = FEEDBOOKS_AUDIO - - return next(filter(lambda x: x[1] == drm_scheme, candidates), (None, None)) - - def _unlimited_access_fulfill( - self, loan: Loan, delivery_mechanism: LicensePoolDeliveryMechanism - ) -> Fulfillment: - licensepool = loan.license_pool - fulfillment = self._find_matching_delivery_mechanism( - delivery_mechanism.delivery_mechanism, licensepool - ) - content_link = fulfillment.resource.representation.public_url - content_type = fulfillment.resource.representation.media_type - return RedirectFulfillment(content_link, content_type) - - def _find_matching_delivery_mechanism( - self, requested_delivery_mechanism: DeliveryMechanism, licensepool: LicensePool - ) -> LicensePoolDeliveryMechanism: + def _check_delivery_mechanism_available( + requested_delivery_mechanism: DeliveryMechanism, licensepool: LicensePool + ) -> None: fulfillment = next( ( lpdm @@ -564,15 +489,30 @@ def _find_matching_delivery_mechanism( ) if fulfillment is None: raise FormatNotAvailable() - return fulfillment - def _lcp_fulfill( + def _unlimited_access_fulfill( + self, loan: Loan, delivery_mechanism: LicensePoolDeliveryMechanism + ) -> Fulfillment: + licensepool = loan.license_pool + self._check_delivery_mechanism_available( + delivery_mechanism.delivery_mechanism, licensepool + ) + content_link = delivery_mechanism.resource.representation.public_url + content_type = delivery_mechanism.resource.representation.media_type + return RedirectFulfillment(content_link, content_type) + + def _license_fulfill( self, loan: Loan, delivery_mechanism: LicensePoolDeliveryMechanism ) -> Fulfillment: - doc = self.get_license_status_document(loan) - status = doc.get("status") + # We are unable to fulfill a loan that doesn't have its external identifier set, + # since we use this to get to the checkout link. It shouldn't be possible to get + # into this state. + license_status_url = loan.external_identifier + assert license_status_url is not None + + doc = self._request_loan_status("GET", license_status_url) - if status not in [self.READY_STATUS, self.ACTIVE_STATUS]: + if not doc.active: # This loan isn't available for some reason. It's possible # the distributor revoked it or the patron already returned it # through the DRM system, and we didn't get a notification @@ -580,27 +520,36 @@ def _lcp_fulfill( self.update_loan(loan, doc) raise CannotFulfill() - expires = doc.get("potential_rights", {}).get("end") - expires = dateutil.parser.parse(expires) - - links = doc.get("links", []) - - content_link, content_type = self._find_content_link_and_type( - links, delivery_mechanism.delivery_mechanism.drm_scheme - ) + drm_scheme = delivery_mechanism.delivery_mechanism.drm_scheme + fulfill_cls: Callable[[str, str | None], UrlFulfillment] + if drm_scheme == DeliveryMechanism.NO_DRM: + # If we have no DRM, we can just redirect to the content link and let the patron download the book. + fulfill_link = doc.links.get( + rel="publication", + type=delivery_mechanism.delivery_mechanism.content_type, + ) + fulfill_cls = RedirectFulfillment + elif drm_scheme == DeliveryMechanism.FEEDBOOKS_AUDIOBOOK_DRM: + # For DeMarque audiobook content using "FEEDBOOKS_AUDIOBOOK_DRM", the link + # we are looking for is stored in the 'manifest' rel. + fulfill_link = doc.links.get(rel="manifest", type=FEEDBOOKS_AUDIO) + fulfill_cls = partial(FetchFulfillment, allowed_response_codes=["2xx"]) + else: + # We are getting content via a license document, so we need to find the link + # that corresponds to the delivery mechanism we are using. + fulfill_link = doc.links.get(rel="license", type=drm_scheme) + fulfill_cls = partial(FetchFulfillment, allowed_response_codes=["2xx"]) - if content_link is None or content_type is None: + if fulfill_link is None: raise CannotFulfill() - return FetchFulfillment( - content_link, content_type, allowed_response_codes=["2xx"] - ) + return fulfill_cls(fulfill_link.href, fulfill_link.type) def _bearer_token_fulfill( self, loan: Loan, delivery_mechanism: LicensePoolDeliveryMechanism ) -> Fulfillment: licensepool = loan.license_pool - fulfillment_mechanism = self._find_matching_delivery_mechanism( + self._check_delivery_mechanism_available( delivery_mechanism.delivery_mechanism, licensepool ) @@ -621,7 +570,7 @@ def _bearer_token_fulfill( token_type="Bearer", access_token=self._session_token.token, expires_in=(int((self._session_token.expires - utc_now()).total_seconds())), - location=fulfillment_mechanism.resource.url, + location=delivery_mechanism.resource.url, ) return DirectFulfillment( @@ -644,7 +593,7 @@ def _fulfill( else: return self._unlimited_access_fulfill(loan, delivery_mechanism) else: - return self._lcp_fulfill(loan, delivery_mechanism) + return self._license_fulfill(loan, delivery_mechanism) def _count_holds_before(self, holdinfo: HoldInfo, pool: LicensePool) -> int: # Count holds on the license pool that started before this hold and @@ -958,30 +907,13 @@ def patron_activity( for hold in remaining_holds ] - def update_loan(self, loan: Loan, status_doc: dict[str, Any] | None = None) -> None: + def update_loan(self, loan: Loan, status_doc: LoanStatus) -> None: """Check a loan's status, and if it is no longer active, delete the loan and update its pool's availability. """ _db = Session.object_session(loan) - if not status_doc: - status_doc = self.get_license_status_document(loan) - - status = status_doc.get("status") - # We already check that the status is valid in get_license_status_document, - # but if the document came from a notification it hasn't been checked yet. - if status not in self.STATUS_VALUES: - raise RemoteIntegrationException( - str(loan.license.checkout_url), - "The License Status Document had an unknown status value.", - ) - - if status in [ - self.REVOKED_STATUS, - self.RETURNED_STATUS, - self.CANCELLED_STATUS, - self.EXPIRED_STATUS, - ]: + if not status_doc.active: # This loan is no longer active. Update the pool's availability # and delete the loan. diff --git a/src/palace/manager/api/odl/auth.py b/src/palace/manager/api/odl/auth.py index e5ffbd8e0e..7c08426f50 100644 --- a/src/palace/manager/api/odl/auth.py +++ b/src/palace/manager/api/odl/auth.py @@ -3,13 +3,17 @@ from datetime import datetime, timedelta from typing import Any, Literal, NamedTuple -from pydantic import AnyUrl, BaseModel, PositiveInt, ValidationError +from pydantic import BaseModel, PositiveInt, ValidationError from requests import Response +from typing_extensions import Self from palace.manager.api.odl.settings import OPDS2AuthType from palace.manager.core.exceptions import IntegrationException, PalaceValueError +from palace.manager.opds.authentication import AuthenticationDocument from palace.manager.util.datetime_helpers import utc_now -from palace.manager.util.http import HTTP, BearerAuth +from palace.manager.util.http import HTTP, BadResponseException, BearerAuth +from palace.manager.util.log import LoggerMixin +from palace.manager.util.problem_detail import ProblemDetail class TokenTuple(NamedTuple): @@ -23,39 +27,65 @@ class TokenGrant(BaseModel): expires_in: PositiveInt -class AuthenticationLink(BaseModel): - rel: str - href: str +class OpdsWithOdlException(BadResponseException): + """ + ODL and Readium LCP specify that all errors should be returned as Problem + Detail documents. This isn't always the case, but we try to use this information + when we can. + """ + + def __init__( + self, + type: str, + title: str, + status: int, + detail: str | None, + response: Response, + ) -> None: + super().__init__(url_or_service=response.url, message=title, response=response) + self.type = type + self.title = title + self.status = status + self.detail = detail + @property + def problem_detail(self) -> ProblemDetail: + return ProblemDetail( + uri=self.type, + status_code=self.status, + title=self.title, + detail=self.detail, + ) -class Authentication(BaseModel): - type: str - links: list[AuthenticationLink] - - def by_rel(self, rel: str) -> list[AuthenticationLink]: - return [link for link in self.links if link.rel == rel] - + @classmethod + def from_response(cls, response: Response) -> Self | None: + # Wrap the response in a OpdsWithOdlException if it is a problem detail document. + # + # DeMarque sends "application/api-problem+json", but the ODL spec says we should + # expect "application/problem+json", so we need to check for both. + if response.headers.get("Content-Type") not in [ + "application/api-problem+json", + "application/problem+json", + ]: + return None -class AuthenticationDocument(BaseModel): - id: AnyUrl - title: str - authentication: list[Authentication] + try: + json_response = response.json() + except ValueError: + json_response = {} - def by_type(self, auth_type: str) -> list[Authentication]: - return [auth for auth in self.authentication if auth.type == auth_type] + type = json_response.get("type") + title = json_response.get("title") + status = json_response.get("status") or response.status_code + detail = json_response.get("detail") - def link_href_by_type_and_rel(self, auth_type: str, rel: str) -> list[str]: - return [ - link.href for auth in self.by_type(auth_type) for link in auth.by_rel(rel) - ] + if type is None or title is None: + return None + return cls(type, title, status, detail, response) -class ODLAuthenticatedGet(ABC): - AUTH_DOC_CONTENT_TYPES = [ - "application/opds-authentication+json", - "application/vnd.opds.authentication.v1.0+json", - ] +class OdlAuthenticatedRequest(LoggerMixin, ABC): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._session_token: TokenTuple | None = None @@ -77,21 +107,36 @@ def _auth_type(self) -> OPDS2AuthType: ... @abstractmethod def _feed_url(self) -> str: ... - @staticmethod - def _no_auth_get( - url: str, headers: Mapping[str, str] | None = None, **kwargs: Any + def _no_auth_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, ) -> Response: - return HTTP.get_with_timeout(url, headers=headers, **kwargs) - - def _basic_auth_get( - self, url: str, headers: Mapping[str, str] | None = None, **kwargs: Any + return HTTP.request_with_timeout(method, url, headers=headers, **kwargs) + + def _basic_auth_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, ) -> Response: - return HTTP.get_with_timeout( - url, headers=headers, auth=(self._username, self._password), **kwargs + return HTTP.request_with_timeout( + method, + url, + headers=headers, + auth=(self._username, self._password), + **kwargs, ) - def _oauth_get( - self, url: str, headers: Mapping[str, str] | None = None, **kwargs: Any + def _oauth_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, ) -> Response: # If the request restricts allowed response codes, we need to add 401 to the allowed response codes # so that we can handle the 401 response and refresh the token, if necessary. @@ -112,10 +157,10 @@ def _oauth_get( token_refreshed = True # Make a request, refreshing the token if we get a 401 response, and we haven't already refreshed the token. - resp = self._session_token_get(url, headers=headers, **kwargs) + resp = self._session_token_request(method, url, headers=headers, **kwargs) if resp.status_code == 401 and not token_refreshed: self._refresh_token() - resp = self._session_token_get(url, headers=headers, **kwargs) + resp = self._session_token_request(method, url, headers=headers, **kwargs) # If we got a 401 response and we modified the allowed response codes, we process the response # with the original allowed response codes, so the calling function can handle the 401 response. @@ -125,18 +170,33 @@ def _oauth_get( return resp - def _get( - self, url: str, headers: Mapping[str, str] | None = None, **kwargs: Any + def _request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, ) -> Response: - match self._auth_type: - case OPDS2AuthType.BASIC: - return self._basic_auth_get(url, headers, **kwargs) - case OPDS2AuthType.OAUTH: - return self._oauth_get(url, headers, **kwargs) - case OPDS2AuthType.NONE: - return self._no_auth_get(url, headers=headers, **kwargs) - case _: - raise PalaceValueError(f"Invalid OPDS2AuthType: '{self._auth_type}'") + try: + match self._auth_type: + case OPDS2AuthType.BASIC: + return self._basic_auth_request( + method, url, headers=headers, **kwargs + ) + case OPDS2AuthType.OAUTH: + return self._oauth_request(method, url, headers=headers, **kwargs) + case OPDS2AuthType.NONE: + return self._no_auth_request(method, url, headers=headers, **kwargs) + case _: + raise PalaceValueError( + f"Invalid OPDS2AuthType: '{self._auth_type}'" + ) + except BadResponseException as e: + response = e.response + # Create a OpdsWithOdlException if the response is a problem detail document. + if opds_exception := OpdsWithOdlException.from_response(response): + raise opds_exception from e + raise @staticmethod def _get_oauth_url_from_auth_document(auth_document_str: str) -> str: @@ -150,18 +210,21 @@ def _get_oauth_url_from_auth_document(auth_document_str: str) -> str: debug_message=f"Auth document: {auth_document_str}", ) from e - auth_links = auth_document.link_href_by_type_and_rel( - "http://opds-spec.org/auth/oauth/client_credentials", "authenticate" - ) - - if len(auth_links) != 1: + try: + return ( + auth_document.by_type( + "http://opds-spec.org/auth/oauth/client_credentials" + ) + .links.get(rel="authenticate", raising=True) + .href + ) + except PalaceValueError: raise IntegrationException( - "Unable to find exactly one valid authentication link", - debug_message=f"Found {len(auth_links)} authentication links. Auth document: {auth_document_str}", + "Unable to find valid authentication link for " + "'http://opds-spec.org/auth/oauth/client_credentials' with rel 'authenticate'", + debug_message=f"Auth document: {auth_document_str}", ) - return auth_links[0] - @staticmethod def _oauth_session_token_refresh( auth_url: str, username: str, password: str @@ -192,18 +255,27 @@ def _oauth_session_token_refresh( def _fetch_auth_document(self) -> str: resp = HTTP.get_with_timeout(self._feed_url) content_type = resp.headers.get("Content-Type") - if resp.status_code != 401 or content_type not in self.AUTH_DOC_CONTENT_TYPES: + if ( + resp.status_code != 401 + or content_type not in AuthenticationDocument.content_types() + ): raise IntegrationException( "Unable to fetch OPDS authentication document. Incorrect status code or content type.", debug_message=f"Status code: '{resp.status_code}' Content-type: '{content_type}' Response: {resp.text}", ) return resp.text - def _session_token_get( - self, url: str, headers: Mapping[str, str] | None = None, **kwargs: Any + def _session_token_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, ) -> Response: auth = BearerAuth(self._session_token.token) if self._session_token else None - return HTTP.get_with_timeout(url, headers=headers, auth=auth, **kwargs) + return HTTP.request_with_timeout( + method, url, headers=headers, auth=auth, **kwargs + ) def _refresh_token(self) -> None: if self._token_url is None: diff --git a/src/palace/manager/api/odl/importer.py b/src/palace/manager/api/odl/importer.py index 014966771b..74906c7d33 100644 --- a/src/palace/manager/api/odl/importer.py +++ b/src/palace/manager/api/odl/importer.py @@ -12,7 +12,7 @@ from webpub_manifest_parser.opds2.registry import OPDS2LinkRelationsRegistry from palace.manager.api.odl.api import OPDS2WithODLApi -from palace.manager.api.odl.auth import ODLAuthenticatedGet +from palace.manager.api.odl.auth import OdlAuthenticatedRequest from palace.manager.api.odl.constants import FEEDBOOKS_AUDIO from palace.manager.api.odl.settings import OPDS2AuthType, OPDS2WithODLSettings from palace.manager.core.metadata_layer import FormatData, LicenseData, Metadata @@ -423,7 +423,7 @@ def get_license_data( return parsed_license -class OPDS2WithODLImportMonitor(ODLAuthenticatedGet, OPDS2ImportMonitor): +class OPDS2WithODLImportMonitor(OdlAuthenticatedRequest, OPDS2ImportMonitor): """Import information from an ODL feed.""" PROTOCOL = OPDS2WithODLApi.label() @@ -467,4 +467,4 @@ def _get( kwargs["allowed_response_codes"] = ["2xx", "3xx"] if not url.startswith("http"): url = urljoin(self._feed_base_url, url) - return super()._get(url, headers, **kwargs) + return super()._request("GET", url, headers, **kwargs) diff --git a/src/palace/manager/opds/__init__.py b/src/palace/manager/opds/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/palace/manager/opds/authentication.py b/src/palace/manager/opds/authentication.py new file mode 100644 index 0000000000..85b81bb5e0 --- /dev/null +++ b/src/palace/manager/opds/authentication.py @@ -0,0 +1,64 @@ +""" +Models for the Authentication for OPDS 1.0 specification. +https://drafts.opds.io/authentication-for-opds-1.0 +""" + +from pydantic import BaseModel, Field, field_validator + +from palace.manager.core.exceptions import PalaceValueError +from palace.manager.opds.base import ListOfLinks +from palace.manager.opds.opds import Link + + +class AuthenticationLabels(BaseModel): + login: str + password: str + + +class Authentication(BaseModel): + type: str + labels: AuthenticationLabels | None = None + links: ListOfLinks[Link] + + +class AuthenticationDocument(BaseModel): + @staticmethod + def content_types() -> list[str]: + return [ + "application/opds-authentication+json", + "application/vnd.opds.authentication.v1.0+json", + ] + + @classmethod + def content_type(cls) -> str: + return cls.content_types()[0] + + id: str + title: str + authentication: list[Authentication] + description: str | None = None + links: ListOfLinks[Link] = Field(default_factory=ListOfLinks) + + @field_validator("authentication") + @classmethod + def _validate_authentication( + cls, value: list[Authentication] + ) -> list[Authentication]: + if not value: + raise ValueError( + "Authentication document must have at least one authentication object." + ) + + auth_types = set() + for auth in value: + if auth.type in auth_types: + raise ValueError(f"Duplicate authentication type '{auth.type}'.") + auth_types.add(auth.type) + + return value + + def by_type(self, auth_type: str) -> Authentication: + for auth in self.authentication: + if auth.type == auth_type: + return auth + raise PalaceValueError(f"Unable to find authentication for '{auth_type}'") diff --git a/src/palace/manager/opds/base.py b/src/palace/manager/opds/base.py new file mode 100644 index 0000000000..2ea716cb79 --- /dev/null +++ b/src/palace/manager/opds/base.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import typing +from functools import cached_property +from typing import Any, TypeVar + +from pydantic import BaseModel, ConfigDict, GetCoreSchemaHandler +from pydantic_core import core_schema +from uritemplate import URITemplate, variable + +from palace.manager.core.exceptions import PalaceValueError + +T = TypeVar("T") + + +def obj_or_set_to_set(value: T | set[T] | frozenset[T] | None) -> frozenset[T]: + """Convert object or set of objects to a set of objects.""" + if value is None: + return frozenset() + if isinstance(value, set): + return frozenset(value) + elif isinstance(value, frozenset): + return value + return frozenset({value}) + + +class BaseOpdsModel(BaseModel): + """Base class for OPDS models.""" + + model_config = ConfigDict( + populate_by_name=True, + frozen=True, + ) + + +class BaseLink(BaseOpdsModel): + """The various models all have links with this same basic structure, but + with additional fields, so we define this base class to avoid repeating + the same fields in each model, and so we can use the same basic validation + for them all. + """ + + href: str + rel: set[str] | str + templated: bool = False + type: str | None = None + + @cached_property + def rels(self) -> frozenset[str]: + return obj_or_set_to_set(self.rel) + + def href_templated(self, var_dict: variable.VariableValueDict | None = None) -> str: + """ + Return the URL with template variables expanded, if necessary. + """ + if not self.templated: + return self.href + template = URITemplate(self.href) + return template.expand(var_dict) + + +LinkT = TypeVar("LinkT", bound="BaseLink") + + +class ListOfLinks(list[LinkT]): + """ + A generic list container for OPDS type links. + + Provides helper methods for finding a link by relation and type, and + provides validation to ensure that each href, rel, and type combination + in the list is unique. + """ + + @classmethod + def __get_pydantic_core_schema__( + cls, source_type: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + origin_type = typing.get_origin(source_type) + assert origin_type is ListOfLinks + [container_type] = typing.get_args(source_type) + return core_schema.no_info_after_validator_function( + cls._validate, + core_schema.list_schema(handler(container_type)), + ) + + @classmethod + def _validate(cls, value: list[LinkT]) -> ListOfLinks[LinkT]: + link_set = set() + links: ListOfLinks[LinkT] = ListOfLinks() + for link in value: + if (link.rels, link.href, link.type) in link_set: + raise PalaceValueError( + f"Duplicate link with relation '{link.rel}', type '{link.type}' and href '{link.href}'" + ) + link_set.add((link.rels, link.href, link.type)) + links.append(link) + return links + + @typing.overload + def get( + self, + *, + rel: str | None = ..., + type: str | None = ..., + raising: typing.Literal[True], + ) -> LinkT: ... + + @typing.overload + def get( + self, *, rel: str | None = ..., type: str | None = ..., raising: bool = ... + ) -> LinkT | None: ... + + def get( + self, *, rel: str | None = None, type: str | None = None, raising: bool = False + ) -> LinkT | None: + """ + Return the link with the specific relation and type. Raises an + exception if there are multiple links with the same relation and type. + """ + links = self.get_list(rel=rel, type=type) + if (num_links := len(links)) != 1 and raising: + if num_links == 0: + err = "No links found" + else: + err = "Multiple links found" + + match (rel, type): + case (None, None): + # Nothing to add to the error message + ... + case (_, None): + err += f" with rel='{rel}'" + case (None, _): + err += f" with type='{type}'" + case _: + err += f" with rel='{rel}' and type='{type}'" + raise PalaceValueError(err) + return next(iter(links), None) + + def get_list( + self, *, rel: str | None = None, type: str | None = None + ) -> list[LinkT]: + """ + Return links with the specific relation and type. + """ + return [ + link + for link in self + if (rel is None or rel in link.rels) and (type is None or type == link.type) + ] diff --git a/src/palace/manager/opds/lcp/__init__.py b/src/palace/manager/opds/lcp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/palace/manager/opds/lcp/license.py b/src/palace/manager/opds/lcp/license.py new file mode 100644 index 0000000000..2cff772c0d --- /dev/null +++ b/src/palace/manager/opds/lcp/license.py @@ -0,0 +1,120 @@ +from pydantic import ( + AwareDatetime, + Base64Bytes, + Field, + NonNegativeInt, + PositiveInt, + field_validator, +) + +from palace.manager.core.exceptions import PalaceValueError +from palace.manager.opds.base import BaseLink, BaseOpdsModel, ListOfLinks + + +class Link(BaseLink): + """ + https://readium.org/lcp-specs/releases/lcp/latest#35-pointing-to-external-resources-the-links-object + """ + + title: str | None = None + profile: str | None = None + length: PositiveInt | None = None + hash: Base64Bytes | None = None + + +class ContentKey(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lcp/latest#34-transmitting-keys-the-encryption-object + """ + + algorithm: str + encrypted_value: Base64Bytes + + +class UserKey(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lcp/latest#34-transmitting-keys-the-encryption-object + """ + + algorithm: str + text_hint: str + key_check: Base64Bytes + + +class Encryption(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lcp/latest#34-transmitting-keys-the-encryption-object + """ + + profile: str + content_key: ContentKey + user_key: UserKey + + +class Rights(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lcp/latest#36-identifying-rights-and-restrictions-the-rights-object + """ + + # This is aliased because 'copy' is a method on BaseModel. Print is + # aliased for consistency. + # NOTE: Although these look like they should be the same as the + # fields in the Protection model, they are not. The Protection model + # defines these as booleans, while rights defines them as the integer + # number of characters/pages allowed to be copied/printed. + allow_copy: NonNegativeInt | None = Field(None, alias="copy") + allow_print: NonNegativeInt | None = Field(None, alias="print") + start: AwareDatetime | None = None + end: AwareDatetime | None = None + + +class User(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lcp/latest#37-identifying-the-user-the-user-object + """ + + id: str | None = None + email: str | None = None + name: str | None = None + encrypted: list[str] = Field(default_factory=list) + + +class Signature(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lcp/latest#38-signing-the-license-the-signature-object + """ + + algorithm: str + certificate: Base64Bytes + value: Base64Bytes + + +class LicenseDocument(BaseOpdsModel): + """ + LCP License Document + + This document is defined here: + https://readium.org/lcp-specs/releases/lcp/latest#3-license-document + """ + + @staticmethod + def content_type() -> str: + return "application/vnd.readium.lcp.license.v1.0+json" + + id: str + issued: AwareDatetime + provider: str + updated: AwareDatetime | None = None + encryption: Encryption + links: ListOfLinks[Link] + rights: Rights | None = None + signature: Signature + + @field_validator("links") + @classmethod + def _validate_links(cls, value: ListOfLinks[Link]) -> ListOfLinks[Link]: + if value.get(rel="hint") is None: + raise PalaceValueError("links must contain a link with rel 'hint'") + if value.get(rel="publication") is None: + raise PalaceValueError("links must contain a link with rel 'publication'") + return value diff --git a/src/palace/manager/opds/lcp/status.py b/src/palace/manager/opds/lcp/status.py new file mode 100644 index 0000000000..5e4fc8d596 --- /dev/null +++ b/src/palace/manager/opds/lcp/status.py @@ -0,0 +1,113 @@ +import sys +from enum import auto +from functools import cached_property + +from pydantic import AwareDatetime, Field + +from palace.manager.opds.base import BaseLink, BaseOpdsModel, ListOfLinks + +# TODO: Remove this when we drop support for Python 3.10 +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from backports.strenum import StrEnum + + +class Link(BaseLink): + """ + https://readium.org/lcp-specs/releases/lsd/latest#25-links + """ + + title: str | None = None + profile: str | None = None + + +class Status(StrEnum): + """ + https://readium.org/lcp-specs/releases/lsd/latest.html#23-status-of-a-license + """ + + READY = auto() + ACTIVE = auto() + REVOKED = auto() + RETURNED = auto() + CANCELLED = auto() + EXPIRED = auto() + + +class Updated(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lsd/latest#24-timestamps + """ + + license: AwareDatetime + status: AwareDatetime + + +class PotentialRights(BaseOpdsModel): + """ " + https://readium.org/lcp-specs/releases/lsd/latest#26-potential-rights + """ + + end: AwareDatetime | None = None + + +class EventType(StrEnum): + """ + https://readium.org/lcp-specs/releases/lsd/latest#27-events + """ + + REGISTER = auto() + RENEW = auto() + RETURN = auto() + REVOKE = auto() + CANCEL = auto() + + +class Event(BaseOpdsModel): + """ + https://readium.org/lcp-specs/releases/lsd/latest#27-events + """ + + type: EventType + name: str + timestamp: AwareDatetime + + # The spec isn't clear if these fields are required, but DeMarque does not + # provide id in their event data. + id: str | None = None + device: str | None = None + + +class LoanStatus(BaseOpdsModel): + """ + This document is defined as part of the Readium LCP Specifications. + + Readium calls this the License Status Document (LSD), however, that + name conflates the concept of License. In the context of ODL and library + lends, it's really the loan status document, so we use that name here. + + The spec for it is located here: + https://readium.org/lcp-specs/releases/lsd/latest.html + + Technically the spec says that there must be at lease one link + with rel="license" but this is not always the case in practice, + especially when the license is returned or revoked. So we don't + enforce that here. + """ + + @staticmethod + def content_type() -> str: + return "application/vnd.readium.license.status.v1.0+json" + + id: str + status: Status + message: str + updated: Updated + links: ListOfLinks[Link] + potential_rights: PotentialRights = Field(default_factory=PotentialRights) + events: list[Event] = Field(default_factory=list) + + @cached_property + def active(self) -> bool: + return self.status in [Status.READY, Status.ACTIVE] diff --git a/src/palace/manager/opds/odl/__init__.py b/src/palace/manager/opds/odl/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/palace/manager/opds/odl/info.py b/src/palace/manager/opds/odl/info.py new file mode 100644 index 0000000000..832dd089d5 --- /dev/null +++ b/src/palace/manager/opds/odl/info.py @@ -0,0 +1,78 @@ +import sys +from enum import auto +from functools import cached_property + +from pydantic import AwareDatetime, Field, NonNegativeInt + +from palace.manager.opds.base import BaseOpdsModel, obj_or_set_to_set +from palace.manager.opds.odl.odl import Protection, Terms +from palace.manager.opds.opds import Price + +# TODO: Remove this when we drop support for Python 3.10 +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from backports.strenum import StrEnum + + +class Status(StrEnum): + """ + https://drafts.opds.io/odl-1.0.html#41-syntax + """ + + PREORDER = auto() + AVAILABLE = auto() + UNAVAILABLE = auto() + + +class Loan(BaseOpdsModel): + """ + https://drafts.opds.io/odl-1.0.html#41-syntax + """ + + href: str + id: str + # We alias 'patron' here because the ODL documentation + # requires the field to be named `patron_id` but + # DeMarque returns a field named `patron`. + patron_id: str = Field(validation_alias="patron") + expires: AwareDatetime + + +class Checkouts(BaseOpdsModel): + """ + https://drafts.opds.io/odl-1.0.html#41-syntax + """ + + left: NonNegativeInt | None = None + available: NonNegativeInt + active: list[Loan] = Field(default_factory=list) + + +class LicenseInfo(BaseOpdsModel): + """ + This document is defined in the ODL specification: + https://drafts.opds.io/odl-1.0.html#4-license-info-document + """ + + @staticmethod + def content_type() -> str: + return "application/vnd.odl.info+json" + + identifier: str + status: Status + checkouts: Checkouts + format: frozenset[str] | str + created: AwareDatetime | None = None + terms: Terms = Field(default_factory=Terms) + protection: Protection = Field(default_factory=Protection) + price: Price | None = None + source: str | None = None + + @cached_property + def formats(self) -> frozenset[str]: + return obj_or_set_to_set(self.format) + + @cached_property + def active(self) -> bool: + return self.status == Status.AVAILABLE diff --git a/src/palace/manager/opds/odl/odl.py b/src/palace/manager/opds/odl/odl.py new file mode 100644 index 0000000000..3f325c1034 --- /dev/null +++ b/src/palace/manager/opds/odl/odl.py @@ -0,0 +1,34 @@ +from functools import cached_property + +from pydantic import AwareDatetime, Field, NonNegativeInt + +from palace.manager.opds.base import BaseOpdsModel, obj_or_set_to_set + + +class Terms(BaseOpdsModel): + """ + https://drafts.opds.io/odl-1.0.html#33-terms + """ + + checkouts: NonNegativeInt | None = None + expires: AwareDatetime | None = None + concurrency: NonNegativeInt | None = None + length: NonNegativeInt | None = None + + +class Protection(BaseOpdsModel): + """ + https://drafts.opds.io/odl-1.0.html#34-protection + """ + + format: frozenset[str] | str = Field(default_factory=frozenset) + devices: int | None = None + # This is aliased because 'copy' is a method on BaseModel, the + # other fields are aliased for consistency. + allow_copy: bool = Field(True, alias="copy") + allow_print: bool = Field(True, alias="print") + allow_tts: bool = Field(True, alias="tts") + + @cached_property + def formats(self) -> frozenset[str]: + return obj_or_set_to_set(self.format) diff --git a/src/palace/manager/opds/opds.py b/src/palace/manager/opds/opds.py new file mode 100644 index 0000000000..166a57ed88 --- /dev/null +++ b/src/palace/manager/opds/opds.py @@ -0,0 +1,22 @@ +from pydantic import PositiveInt + +from palace.manager.opds.base import BaseLink, BaseOpdsModel + + +class Price(BaseOpdsModel): + """ + https://drafts.opds.io/opds-2.0#53-acquisition-links + """ + + currency: str + value: float + + +class Link(BaseLink): + """Link to another resource.""" + + title: str | None = None + height: PositiveInt | None = None + width: PositiveInt | None = None + bitrate: PositiveInt | None = None + duration: PositiveInt | None = None diff --git a/src/palace/manager/sqlalchemy/constants.py b/src/palace/manager/sqlalchemy/constants.py index 347fe69ad2..56630b29f6 100644 --- a/src/palace/manager/sqlalchemy/constants.py +++ b/src/palace/manager/sqlalchemy/constants.py @@ -127,6 +127,7 @@ class EditionConstants: additional_type_to_medium[v] = k additional_type_to_medium["http://schema.org/Book"] = BOOK_MEDIUM + additional_type_to_medium["http://schema.org/Audiobook"] = AUDIO_MEDIUM # Map the medium constants to the strings used when generating # permanent work IDs. diff --git a/tests/files/opds/lcp/license/fb.json b/tests/files/opds/lcp/license/fb.json new file mode 100644 index 0000000000..119b41ce5e --- /dev/null +++ b/tests/files/opds/lcp/license/fb.json @@ -0,0 +1,48 @@ +{ + "provider": "https://example.com", + "id": "123", + "issued": "2024-09-25T17:09:54Z", + "updated": "2024-09-27T18:57:58Z", + "encryption": { + "profile": "http://readium.org/lcp/profile-1.0", + "content_key": { + "algorithm": "http://www.w3.org/2001/04/xmlenc#aes256-cbc", + "encrypted_value": "cXdlcnR5IDEyMw==" + }, + "user_key": { + "algorithm": "http://www.w3.org/2001/04/xmlenc#sha256", + "text_hint": "Please contact your administrator to restore the passphrase", + "key_check": "cXdlcnR5IDEyMw==" + } + }, + "links": [ + { + "rel": "hint", + "href": "https://foo.com", + "type": "text/html" + }, + { + "rel": "status", + "href": "https://example.com/api/lcp/123/status", + "type": "application/vnd.readium.license.status.v1.0+json" + }, + { + "rel": "publication", + "href": "https://example.com/the/publication", + "type": "application/audiobook+lcp" + } + ], + "user": { + "id": "456" + }, + "rights": { + "print": 0, + "start": "2024-09-25T17:09:54Z", + "end": "2024-10-16T17:09:53Z" + }, + "signature": { + "certificate": "cXdlcnR5IDEyMw==", + "value": "cXdlcnR5IDEyMw==", + "algorithm": "http://www.w3.org/2001/04/xmldsig-more#ecdsa-sha256" + } +} \ No newline at end of file diff --git a/tests/files/opds/lcp/license/ul.json b/tests/files/opds/lcp/license/ul.json new file mode 100644 index 0000000000..9978b16c38 --- /dev/null +++ b/tests/files/opds/lcp/license/ul.json @@ -0,0 +1,51 @@ +{ + "provider": "https://app.example.com", + "id": "123-456", + "issued": "2024-09-26T17:53:16Z", + "encryption": { + "profile": "http://readium.org/lcp/basic-profile", + "content_key": { + "algorithm": "http://www.w3.org/2001/04/xmlenc#aes256-cbc", + "encrypted_value": "Sm9uYXRoYW4gR3JlZW4gcm9ja3MhICDwn5Sl" + }, + "user_key": { + "algorithm": "http://www.w3.org/2001/04/xmlenc#sha256", + "text_hint": "Look up... look wayyyyy up....", + "key_check": "Sm9uYXRoYW4gR3JlZW4gcm9ja3MhICDwn5Sl" + } + }, + "links": [ + { + "rel": "hint", + "href": "https://foo.com", + "type": "text/html" + }, + { + "rel": "status", + "href": "https://license.example.com/licenses/123-456/status", + "type": "application/vnd.readium.license.status.v1.0+json" + }, + { + "rel": "publication", + "href": "https://example.com/the/publication", + "type": "application/epub+zip", + "title": "Test Publication", + "length": 677442, + "hash": "681fc80896e1150a13c6d28466ccc43dd75e6a2a5b34c79b207a1043c5598610" + } + ], + "user": { + "id": "3e8dceae-c177-48e7-a2b2-5effa35eedb1" + }, + "rights": { + "print": 100000000, + "copy": 100000000, + "start": "2024-09-26T17:53:16Z", + "end": "2044-09-21T17:53:02Z" + }, + "signature": { + "certificate": "Sm9uYXRoYW4gR3JlZW4gcm9ja3MhICDwn5Sl", + "value": "Sm9uYXRoYW4gR3JlZW4gcm9ja3MhICDwn5Sl", + "algorithm": "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256" + } +} \ No newline at end of file diff --git a/tests/files/opds/lcp/status/fb-active.json b/tests/files/opds/lcp/status/fb-active.json new file mode 100644 index 0000000000..31a0c95c3c --- /dev/null +++ b/tests/files/opds/lcp/status/fb-active.json @@ -0,0 +1,39 @@ +{ + "id": "123-456-789", + "status": "active", + "message": "The license has been activated.", + "updated": { + "license": "2024-09-25T17:15:19Z", + "status": "2024-09-25T17:15:19Z" + }, + "links": [ + { + "href": "https://example.com/loan/status/?id=123&checkout_id=123-456-789", + "type": "application/vnd.readium.license.status.v1.0+json", + "rel": "self" + }, + { + "href": "https://example.com/loan/audiobook/manifest/?loan_uuid=123-456-789", + "type": "application/audiobook+json; protection=http://www.feedbooks.com/audiobooks/access-restriction", + "rel": "manifest" + }, + { + "href": "https://example.com/loan/audiobook/player/?loan_uuid=123-456-789", + "type": "text/html", + "rel": "publication" + }, + { + "href": "https://example.com/loan/lcp/?loan_uuid=123-456-789", + "type": "application/vnd.readium.lcp.license.v1.0+json", + "rel": "license" + }, + { + "href": "https://example.com/loan/do-return?uuid=123-456-789", + "type": "application/vnd.readium.license.status.v1.0+json", + "rel": "return" + } + ], + "potential_rights": { + "end": "2024-10-16T17:09:54Z" + } +} \ No newline at end of file diff --git a/tests/files/opds/lcp/status/fb-book-adobe.json b/tests/files/opds/lcp/status/fb-book-adobe.json new file mode 100644 index 0000000000..423855ed7b --- /dev/null +++ b/tests/files/opds/lcp/status/fb-book-adobe.json @@ -0,0 +1,57 @@ +{ + "id": "123", + "status": "active", + "message": "The license has been activated.", + "updated": { + "license": "2024-10-02T13:42:09Z", + "status": "2024-10-02T13:42:09Z" + }, + "links": [ + { + "href": "https://license.feedbooks.net/loan/status/?id=456&checkout_id=123", + "type": "application/vnd.readium.license.status.v1.0+json", + "rel": "self" + }, + { + "href": "https://license.feedbooks.net/loan/webreader/?loan_uuid=123", + "type": "text/html", + "rel": "publication" + }, + { + "href": "https://license.feedbooks.net/loan/webreader-token/{?token}", + "type": "text/html", + "rel": "publication", + "templated": true + }, + { + "href": "https://license.feedbooks.net/loan/acs/?loan_uuid=123", + "type": "application/vnd.adobe.adept+xml", + "rel": "license" + }, + { + "href": "https://license.feedbooks.net/loan/lcp/?loan_uuid=123", + "type": "application/vnd.readium.lcp.license.v1.0+json", + "rel": "license" + }, + { + "href": "https://r.cantook.com/read/{token}", + "type": "text/html", + "rel": [ + "publication", + "https://r.cantook.com" + ], + "templated": true, + "properties": { + "identifier": "789" + } + }, + { + "href": "https://license.feedbooks.net/loan/do-return?uuid=123", + "type": "application/vnd.readium.license.status.v1.0+json", + "rel": "return" + } + ], + "potential_rights": { + "end": "2024-10-23T13:39:11Z" + } +} \ No newline at end of file diff --git a/tests/files/opds/lcp/status/fb-early-return.json b/tests/files/opds/lcp/status/fb-early-return.json new file mode 100644 index 0000000000..4a336fd46a --- /dev/null +++ b/tests/files/opds/lcp/status/fb-early-return.json @@ -0,0 +1 @@ +{"id":"456","status":"returned","message":"The license has been returned early.","updated":{"license":"2024-10-02T13:43:41Z","status":"2024-10-02T13:43:41Z"},"links":[{"href":"https://license.feedbooks.net/loan/status/?id=123&checkout_id=456","type":"application/vnd.readium.license.status.v1.0+json","rel":"self"}],"potential_rights":{"end":"2024-10-23T13:39:11Z"},"events":[{"type":"return","name":"","timestamp":"2024-10-02T13:43:41Z","device":""}]} \ No newline at end of file diff --git a/tests/files/opds/lcp/status/ul-active.json b/tests/files/opds/lcp/status/ul-active.json new file mode 100644 index 0000000000..3e627f3e06 --- /dev/null +++ b/tests/files/opds/lcp/status/ul-active.json @@ -0,0 +1 @@ +{"id":"123-456-789","status":"active","updated":{"license":"2024-09-26T17:53:16Z","status":"2024-09-26T17:53:16Z"},"message":"The license is in active state","links":[{"rel":"license","href":"https://app.example.com/api/license/refresh/123-456-789","type":"application/vnd.readium.lcp.license.v1.0+json"},{"rel":"self","href":"https://license.example.com/licenses/123-456-789/status","type":"application/vnd.readium.license.status.v1.0+json"},{"rel":"return","href":"https://license.example.com/licenses/123-456-789/return{?id,name}","type":"application/vnd.readium.license.status.v1.0+json","templated":true}],"potential_rights":{"end":"2044-09-21T17:53:02Z"}} diff --git a/tests/files/opds/lcp/status/ul-returned.json b/tests/files/opds/lcp/status/ul-returned.json new file mode 100644 index 0000000000..f0e5c2a76f --- /dev/null +++ b/tests/files/opds/lcp/status/ul-returned.json @@ -0,0 +1 @@ +{"id":"123","status":"returned","updated":{"license":"2024-10-02T13:33:18Z","status":"2024-10-02T13:33:18Z"},"message":"The license is in returned state","links":[{"rel":"license","href":"https://app.unlimitedlistens.com/api/license/refresh/123","type":"application/vnd.readium.lcp.license.v1.0+json"},{"rel":"self","href":"https://license.unlimitedlistens.com/licenses/123/status","type":"application/vnd.readium.license.status.v1.0+json"}],"events":[{"name":"iPhone","timestamp":"2024-10-02T13:33:02Z","type":"register","id":"456"},{"name":"Palace Manager","timestamp":"2024-10-02T13:33:18Z","type":"return","id":""}]} diff --git a/tests/files/opds/odl/info/feedbooks-ab-checked-out.json b/tests/files/opds/odl/info/feedbooks-ab-checked-out.json new file mode 100644 index 0000000000..e470ac4cd2 --- /dev/null +++ b/tests/files/opds/odl/info/feedbooks-ab-checked-out.json @@ -0,0 +1,36 @@ +{ + "identifier": "urn:uuid:123", + "status": "available", + "created": "2022-10-28T18:20:12Z", + "format": "application/audiobook+json; protection=http://www.feedbooks.com/audiobooks/access-restriction", + "price": { + "value": 72, + "currency": "usd" + }, + "terms": { + "expires": "2024-10-27T00:00:00Z", + "concurrency": 1, + "length": 5097600 + }, + "protection": { + "format": [ + "application/vnd.readium.lcp.license.v1.0+json" + ], + "devices": 6, + "copy": false, + "print": false, + "tts": false + }, + "expires": "2024-10-27T00:00:00Z", + "checkouts": { + "available": 0, + "active": [ + { + "id": "5f15025b-747a-4203-b01b-b58170cea359", + "expires": "2024-10-06T13:36:18Z", + "href": "https://license.example.com/loan/status/123", + "patron": "16a32415-a550-43a4-8e87-d72889df5f6e" + } + ] + } +} \ No newline at end of file diff --git a/tests/files/opds/odl/info/feedbooks-ab-loan-limited.json b/tests/files/opds/odl/info/feedbooks-ab-loan-limited.json new file mode 100644 index 0000000000..effc5003eb --- /dev/null +++ b/tests/files/opds/odl/info/feedbooks-ab-loan-limited.json @@ -0,0 +1,36 @@ +{ + "identifier": "urn:uuid:123", + "status": "available", + "created": "2024-08-30T14:22:04Z", + "format": "application/audiobook+json; protection=http://www.feedbooks.com/audiobooks/access-restriction", + "price": { + "value": 49.95, + "currency": "usd" + }, + "terms": { + "checkouts": 40, + "concurrency": 1, + "length": 5097600 + }, + "protection": { + "format": [ + "application/vnd.readium.lcp.license.v1.0+json" + ], + "devices": 6, + "copy": false, + "print": false, + "tts": false + }, + "checkouts": { + "left": 37, + "available": 0, + "active": [ + { + "id": "fb8c09ec-7b60-11ef-8495-0242ac130002", + "expires": "2024-10-16T17:09:54Z", + "href": "https://license.example.com/loan/status/123", + "patron": "fb8d46b8-7b60-11ef-8495-0242ac130002" + } + ] + } +} \ No newline at end of file diff --git a/tests/files/opds/odl/info/feedbooks-ab-not-checked-out.json b/tests/files/opds/odl/info/feedbooks-ab-not-checked-out.json new file mode 100644 index 0000000000..1e73c99e7f --- /dev/null +++ b/tests/files/opds/odl/info/feedbooks-ab-not-checked-out.json @@ -0,0 +1,28 @@ +{ + "identifier": "urn:uuid:123", + "status": "available", + "created": "2024-02-28T00:22:16Z", + "format": "application/audiobook+json; protection=http://www.feedbooks.com/audiobooks/access-restriction", + "price": { + "value": 49.99, + "currency": "usd" + }, + "terms": { + "expires": "2026-02-27T00:00:00Z", + "concurrency": 1, + "length": 5097600 + }, + "protection": { + "format": [ + "application/vnd.readium.lcp.license.v1.0+json" + ], + "devices": 6, + "copy": false, + "print": false, + "tts": false + }, + "expires": "2026-02-27T00:00:00Z", + "checkouts": { + "available": 1 + } +} \ No newline at end of file diff --git a/tests/files/opds/odl/info/feedbooks-book-adept.json b/tests/files/opds/odl/info/feedbooks-book-adept.json new file mode 100644 index 0000000000..8e225f01c4 --- /dev/null +++ b/tests/files/opds/odl/info/feedbooks-book-adept.json @@ -0,0 +1,28 @@ +{ + "identifier": "urn:uuid:123", + "status": "unavailable", + "created": "2021-12-21T19:25:00Z", + "format": "application/epub+zip", + "price": { + "value": 45, + "currency": "usd" + }, + "terms": { + "expires": "2023-12-21T00:00:00Z", + "concurrency": 1, + "length": 5097600 + }, + "protection": { + "format": [ + "application/vnd.adobe.adept+xml" + ], + "devices": 6, + "copy": false, + "print": false, + "tts": false + }, + "expires": "2023-12-21T00:00:00Z", + "checkouts": { + "available": 1 + } +} \ No newline at end of file diff --git a/tests/files/opds/odl/info/feedbooks-book-unavailable.json b/tests/files/opds/odl/info/feedbooks-book-unavailable.json new file mode 100644 index 0000000000..037aa91295 --- /dev/null +++ b/tests/files/opds/odl/info/feedbooks-book-unavailable.json @@ -0,0 +1,28 @@ +{ + "identifier": "urn:uuid:123", + "status": "unavailable", + "created": "2022-05-26T18:20:16Z", + "format": "application/epub+zip", + "price": { + "value": 55, + "currency": "usd" + }, + "terms": { + "expires": "2024-05-25T00:00:00Z", + "concurrency": 1, + "length": 5097600 + }, + "protection": { + "format": [ + "application/vnd.adobe.adept+xml" + ], + "devices": 6, + "copy": false, + "print": false, + "tts": false + }, + "expires": "2024-05-25T00:00:00Z", + "checkouts": { + "available": 1 + } +} \ No newline at end of file diff --git a/tests/files/opds/odl/info/ul-ab.json b/tests/files/opds/odl/info/ul-ab.json new file mode 100644 index 0000000000..5cf75b9207 --- /dev/null +++ b/tests/files/opds/odl/info/ul-ab.json @@ -0,0 +1 @@ +{"identifier":"urn:uuid:123","status":"available","created":"2024-09-13T07:00:00Z","format":"application\/audiobook+lcp","terms":{"concurrency":2},"protection":{"format":"application\/vnd.readium.lcp.license.v1.0+json","copy":true,"print":true},"checkouts":{"available":2}} \ No newline at end of file diff --git a/tests/files/opds/odl/info/ul-book.json b/tests/files/opds/odl/info/ul-book.json new file mode 100644 index 0000000000..f006a78997 --- /dev/null +++ b/tests/files/opds/odl/info/ul-book.json @@ -0,0 +1 @@ +{"identifier":"urn:uuid:123","status":"available","created":"2024-09-13T07:00:00Z","format":"application\/epub+zip","terms":{"concurrency":2},"protection":{"format":"application\/vnd.readium.lcp.license.v1.0+json","copy":true,"print":true},"checkouts":{"available":0,"active":[{"href":"https:\/\/example.com\/licenses\/123\/status","id":"03ab7837-a5bb-4f7b-959d-64d9100064a2","patron_id":"3e8dceae-c177-48e7-a2b2-5effa35eedb1","expires":"2044-09-22T00:53:02Z"},{"href":"https:\/\/example.com\/licenses\/456\/status","id":"fd6b63e5-a42e-4d7f-b465-ce558f34c946","patron_id":"034e5c77-778f-4006-886d-788106af0c03","expires":"2044-09-22T01:39:31Z"}]}} \ No newline at end of file diff --git a/tests/fixtures/odl.py b/tests/fixtures/odl.py index dd860d4ba3..7a124e4559 100644 --- a/tests/fixtures/odl.py +++ b/tests/fixtures/odl.py @@ -3,7 +3,8 @@ import datetime import json import uuid -from typing import Any +from functools import partial +from typing import Any, Literal from unittest.mock import MagicMock import pytest @@ -14,6 +15,8 @@ from palace.manager.api.odl.api import OPDS2WithODLApi from palace.manager.api.odl.importer import OPDS2WithODLImporter from palace.manager.core.coverage import CoverageFailure +from palace.manager.opds.lcp.license import LicenseDocument +from palace.manager.opds.lcp.status import LoanStatus from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.edition import Edition from palace.manager.sqlalchemy.model.library import Library @@ -25,6 +28,7 @@ ) from palace.manager.sqlalchemy.model.patron import Loan, Patron from palace.manager.sqlalchemy.model.work import Work +from palace.manager.util.datetime_helpers import utc_now from tests.fixtures.database import DatabaseTransactionFixture from tests.fixtures.files import FilesFixture, OPDS2WithODLFilesFixture from tests.mocks.mock import MockHTTPClient, MockRequestsResponse @@ -48,6 +52,19 @@ def __init__( self.api = MockOPDS2WithODLApi(self.db.session, self.collection, self.mock_http) self.patron = db.patron() self.pool = self.license.license_pool + self.license_document = partial( + LicenseDocument, + id=str(uuid.uuid4()), + issued=utc_now(), + provider="Tests", + ) + self.api_checkout = partial( + self.api.checkout, + patron=self.patron, + pin="pin", + licensepool=self.pool, + delivery_mechanism=MagicMock(), + ) def create_work(self, collection: Collection) -> Work: return self.db.work(with_license_pool=True, collection=collection) @@ -88,31 +105,70 @@ def setup_license( pool.update_availability_from_licenses() return license_ + @staticmethod + def loan_status_document( + status: str = "ready", + self_link: str | Literal[False] = "http://status", + return_link: str | Literal[False] = "http://return", + license_link: str | Literal[False] = "http://license", + links: list[dict[str, str]] | None = None, + ) -> LoanStatus: + if links is None: + links = [] + + if license_link: + links.append( + { + "rel": "license", + "href": license_link, + "type": LicenseDocument.content_type(), + }, + ) + + if self_link: + links.append( + { + "rel": "self", + "href": self_link, + "type": LoanStatus.content_type(), + } + ) + + if return_link: + links.append( + { + "rel": "return", + "href": return_link, + "type": LoanStatus.content_type(), + } + ) + + return LoanStatus.model_validate( + dict( + id=str(uuid.uuid4()), + status=status, + message="This is a message", + updated={ + "license": utc_now(), + "status": utc_now(), + }, + links=links, + potential_rights={"end": "3017-10-21T11:12:13Z"}, + ) + ) + def checkin( self, patron: Patron | None = None, pool: LicensePool | None = None ) -> None: patron = patron or self.patron pool = pool or self.pool - lsd = json.dumps( - { - "status": "ready", - "links": [ - { - "rel": "return", - "href": "http://return", - } - ], - } + + self.mock_http.queue_response( + 200, content=self.loan_status_document().model_dump_json() ) - returned_lsd = json.dumps( - { - "status": "returned", - } + self.mock_http.queue_response( + 200, content=self.loan_status_document("returned").model_dump_json() ) - - self.mock_http.queue_response(200, content=lsd) - self.mock_http.queue_response(200, content="") - self.mock_http.queue_response(200, content=returned_lsd) self.api.checkin(patron, "pin", pool) def checkout( @@ -125,19 +181,9 @@ def checkout( pool = pool or self.pool loan_url = loan_url or self.db.fresh_url() - lsd = json.dumps( - { - "status": "ready", - "potential_rights": {"end": "3017-10-21T11:12:13Z"}, - "links": [ - { - "rel": "self", - "href": loan_url, - } - ], - } + self.mock_http.queue_response( + 201, content=self.loan_status_document(self_link=loan_url).model_dump_json() ) - self.mock_http.queue_response(200, content=lsd) loan = self.api.checkout(patron, "pin", pool, MagicMock()) loan_db = ( self.db.session.query(Loan) diff --git a/tests/manager/api/controller/test_odl_notify.py b/tests/manager/api/controller/test_odl_notify.py index 04ce869d10..34105df6aa 100644 --- a/tests/manager/api/controller/test_odl_notify.py +++ b/tests/manager/api/controller/test_odl_notify.py @@ -1,46 +1,43 @@ -import json -import types -from unittest.mock import create_autospec +from unittest.mock import MagicMock -import flask import pytest +from flask import Response +from palace.manager.api.controller.odl_notification import ODLNotificationController from palace.manager.api.odl.api import OPDS2WithODLApi from palace.manager.api.problem_details import ( INVALID_LOAN_FOR_ODL_NOTIFICATION, NO_ACTIVE_LOAN, ) -from palace.manager.sqlalchemy.model.collection import Collection -from tests.fixtures.api_controller import ControllerFixture +from palace.manager.core.problem_details import INVALID_INPUT +from palace.manager.util.problem_detail import ProblemDetail from tests.fixtures.database import DatabaseTransactionFixture +from tests.fixtures.flask import FlaskAppFixture +from tests.fixtures.odl import OPDS2WithODLApiFixture +from tests.fixtures.services import ServicesFixture +from tests.mocks.mock import MockHTTPClient +from tests.mocks.odl import MockOPDS2WithODLApi class ODLFixture: - def __init__(self, db: DatabaseTransactionFixture): + def __init__( + self, db: DatabaseTransactionFixture, services_fixture: ServicesFixture + ) -> None: self.db = db self.library = self.db.default_library() - - """Create a mock ODL collection to use in tests.""" - self.collection, _ = Collection.by_name_and_protocol( - self.db.session, "Test ODL Collection", OPDS2WithODLApi.label() + self.registry = ( + services_fixture.services.integration_registry.license_providers() + ) + self.collection = db.collection( + protocol=self.registry.get_protocol(OPDS2WithODLApi), + settings={ + "username": "a", + "password": "b", + "external_account_id": "http://odl", + "data_source": "Feedbooks", + }, ) - self.collection.integration_configuration.settings_dict = { - "username": "a", - "password": "b", - "url": "http://metadata", - "external_integration_id": "http://odl", - Collection.DATA_SOURCE_NAME_SETTING: "Feedbooks", - } - self.collection.libraries.append(self.library) self.work = self.db.work(with_license_pool=True, collection=self.collection) - - def setup(self, available, concurrency, left=None, expires=None): - self.checkouts_available = available - self.checkouts_left = left - self.terms_concurrency = concurrency - self.expires = expires - self.license_pool.update_availability_from_licenses() - self.pool = self.work.license_pools[0] self.license = self.db.license( self.pool, @@ -48,85 +45,104 @@ def setup(self, available, concurrency, left=None, expires=None): checkouts_available=1, terms_concurrency=1, ) - types.MethodType(setup, self.license) self.pool.update_availability_from_licenses() self.patron = self.db.patron() - - @staticmethod - def integration_protocol(): - return OPDS2WithODLApi.label() + self.http_client = MockHTTPClient() + self.api = MockOPDS2WithODLApi(db.session, self.collection, self.http_client) + self.mock_circulation_manager = MagicMock() + self.mock_circulation_manager.circulation_apis[ + self.library.id + ].api_for_license_pool.return_value = self.api + self.controller = ODLNotificationController( + db.session, self.mock_circulation_manager, self.registry + ) + self.loan_status_document = OPDS2WithODLApiFixture.loan_status_document @pytest.fixture(scope="function") -def odl_fixture(db: DatabaseTransactionFixture) -> ODLFixture: - return ODLFixture(db) +def odl_fixture( + db: DatabaseTransactionFixture, services_fixture: ServicesFixture +) -> ODLFixture: + return ODLFixture(db, services_fixture) class TestODLNotificationController: """Test that an ODL distributor can notify the circulation manager when a loan's status changes.""" - @pytest.mark.parametrize( - "api_cls", - [ - pytest.param(OPDS2WithODLApi, id="ODL 2.x collection"), - ], - ) def test_notify_success( self, - api_cls: type[OPDS2WithODLApi], - controller_fixture: ControllerFixture, + db: DatabaseTransactionFixture, + flask_app_fixture: FlaskAppFixture, odl_fixture: ODLFixture, - ): - db = controller_fixture.db - - odl_fixture.collection.integration_configuration.protocol = api_cls.label() - odl_fixture.pool.licenses_owned = 10 - odl_fixture.pool.licenses_available = 5 - loan, ignore = odl_fixture.pool.loan_to(odl_fixture.patron) + ) -> None: + odl_fixture.license.checkout() + loan, ignore = odl_fixture.license.loan_to(odl_fixture.patron) loan.external_identifier = db.fresh_str() - api = controller_fixture.manager.circulation_apis[ - db.default_library().id - ].api_for_license_pool(loan.license_pool) - update_loan_mock = create_autospec(api_cls.update_loan) - api.update_loan = update_loan_mock - - with controller_fixture.request_context_with_library("/", method="POST"): - text = json.dumps( - { - "id": loan.external_identifier, - "status": "revoked", - } - ) - data = bytes(text, "utf-8") - flask.request.data = data - response = controller_fixture.manager.odl_notification_controller.notify( - loan.id - ) - assert 200 == response.status_code - - # Update loan was called with the expected arguments. - update_loan_mock.assert_called_once_with(loan, json.loads(text)) - - def test_notify_errors(self, controller_fixture: ControllerFixture): - db = controller_fixture.db - - # No loan. - with controller_fixture.request_context_with_library("/", method="POST"): - response = controller_fixture.manager.odl_notification_controller.notify( - db.fresh_str() - ) - assert NO_ACTIVE_LOAN.uri == response.uri + assert odl_fixture.license.checkouts_available == 0 - # Loan from a non-ODL collection. + status_doc = odl_fixture.loan_status_document("revoked") + with flask_app_fixture.test_request_context( + "/", + method="POST", + data=status_doc.model_dump_json(), + library=odl_fixture.library, + ): + assert loan.id is not None + response = odl_fixture.controller.notify(loan.id) + assert response.status_code == 204 + + assert odl_fixture.license.checkouts_available == 1 + + def test_notify_errors( + self, + db: DatabaseTransactionFixture, + flask_app_fixture: FlaskAppFixture, + odl_fixture: ODLFixture, + ): + # Bad JSON. patron = db.patron() pool = db.licensepool(None) loan, ignore = pool.loan_to(patron) loan.external_identifier = db.fresh_str() + with flask_app_fixture.test_request_context( + "/", method="POST", library=odl_fixture.library + ): + response = odl_fixture.controller.notify(loan.id) + assert response == INVALID_INPUT - with controller_fixture.request_context_with_library("/", method="POST"): - response = controller_fixture.manager.odl_notification_controller.notify( - loan.id - ) - assert INVALID_LOAN_FOR_ODL_NOTIFICATION == response + # Loan from a non-ODL collection. + with flask_app_fixture.test_request_context( + "/", + method="POST", + library=odl_fixture.library, + data=odl_fixture.loan_status_document("active").model_dump_json(), + ): + response = odl_fixture.controller.notify(loan.id) + assert isinstance(response, ProblemDetail) + assert response == INVALID_LOAN_FOR_ODL_NOTIFICATION + + # No loan, but distributor thinks it isn't active + NON_EXISTENT_LOAN_ID = -55 + with flask_app_fixture.test_request_context( + "/", + method="POST", + library=odl_fixture.library, + data=odl_fixture.loan_status_document("returned").model_dump_json(), + ): + response = odl_fixture.controller.notify(NON_EXISTENT_LOAN_ID) + assert isinstance(response, Response) + assert response.status_code == 204 + + # No loan, but distributor thinks it is active + with flask_app_fixture.test_request_context( + "/", + method="POST", + library=odl_fixture.library, + data=odl_fixture.loan_status_document("active").model_dump_json(), + ): + response = odl_fixture.controller.notify(-55) + assert isinstance(response, ProblemDetail) + assert response.status_code == 404 + assert response.uri == NO_ACTIVE_LOAN.uri diff --git a/tests/manager/api/odl/test_api.py b/tests/manager/api/odl/test_api.py index 117329cf56..058ae542ff 100644 --- a/tests/manager/api/odl/test_api.py +++ b/tests/manager/api/odl/test_api.py @@ -4,8 +4,6 @@ import json import urllib import uuid -from functools import partial -from typing import Any from unittest.mock import MagicMock from urllib.parse import parse_qs, urlparse @@ -25,6 +23,7 @@ AlreadyOnHold, CannotFulfill, CannotLoan, + CannotReturn, CurrentlyAvailable, HoldOnUnlimitedAccess, HoldsNotPermitted, @@ -38,6 +37,7 @@ from palace.manager.api.odl.api import OPDS2WithODLApi from palace.manager.api.odl.constants import FEEDBOOKS_AUDIO from palace.manager.api.odl.settings import OPDS2AuthType, OPDS2WithODLLibrarySettings +from palace.manager.opds.lcp.status import LoanStatus from palace.manager.sqlalchemy.constants import MediaTypes from palace.manager.sqlalchemy.model.licensing import ( DeliveryMechanism, @@ -46,12 +46,13 @@ RightsStatus, ) from palace.manager.sqlalchemy.model.patron import Hold, Loan -from palace.manager.sqlalchemy.model.resource import Hyperlink, Representation +from palace.manager.sqlalchemy.model.resource import Hyperlink from palace.manager.sqlalchemy.model.work import Work from palace.manager.sqlalchemy.util import create from palace.manager.util.datetime_helpers import datetime_utc, utc_now from palace.manager.util.http import BadResponseException, RemoteIntegrationException from tests.fixtures.database import DatabaseTransactionFixture +from tests.fixtures.files import OPDSFilesFixture from tests.fixtures.odl import OPDS2WithODLApiFixture @@ -194,110 +195,87 @@ def test_hold_limit( .count() ) - def test_get_license_status_document_success( - self, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture + @pytest.mark.parametrize( + "status_code", + [pytest.param(200, id="existing loan"), pytest.param(200, id="new loan")], + ) + def test__request_loan_status_success( + self, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, status_code: int ) -> None: - # With a new loan. New loan returns a 201 status. - loan, _ = opds2_with_odl_api_fixture.license.loan_to( - opds2_with_odl_api_fixture.patron - ) - opds2_with_odl_api_fixture.mock_http.queue_response( - 201, content=json.dumps(dict(status="ready")) - ) - opds2_with_odl_api_fixture.api.get_license_status_document(loan) - requested_url = opds2_with_odl_api_fixture.mock_http.requests.pop() - - parsed = urlparse(requested_url) - assert "https" == parsed.scheme - assert "loan.feedbooks.net" == parsed.netloc - params = parse_qs(parsed.query) - - assert ( - opds2_with_odl_api_fixture.api.settings.passphrase_hint == params["hint"][0] - ) - assert ( - opds2_with_odl_api_fixture.api.settings.passphrase_hint_url - == params["hint_url"][0] - ) - - assert opds2_with_odl_api_fixture.license.identifier == params["id"][0] - - # The checkout id and patron id are random UUIDs. - checkout_id = params["checkout_id"][0] - assert uuid.UUID(checkout_id) - patron_id = params["patron_id"][0] - assert uuid.UUID(patron_id) - - # Loans expire in 21 days by default. - now = utc_now() - after_expiration = now + datetime.timedelta(days=23) - expires = urllib.parse.unquote(params["expires"][0]) - - # The expiration time passed to the server is associated with - # the UTC time zone. - assert expires.endswith("+00:00") - expires_t = dateutil.parser.parse(expires) - assert expires_t.tzinfo == dateutil.tz.tz.tzutc() - - # It's a time in the future, but not _too far_ in the future. - assert expires_t > now - assert expires_t < after_expiration - - notification_url = urllib.parse.unquote_plus(params["notification_url"][0]) - assert ( - "http://odl_notify?library_short_name=%s&loan_id=%s" - % (opds2_with_odl_api_fixture.library.short_name, loan.id) - == notification_url - ) - - # With an existing loan. Existing loan returns a 200 status. - loan, _ = opds2_with_odl_api_fixture.license.loan_to( - opds2_with_odl_api_fixture.patron - ) - loan.external_identifier = opds2_with_odl_api_fixture.db.fresh_str() + expected_document = opds2_with_odl_api_fixture.loan_status_document("active") opds2_with_odl_api_fixture.mock_http.queue_response( - 200, content=json.dumps(dict(status="active")) + status_code, content=expected_document.model_dump_json() ) - opds2_with_odl_api_fixture.api.get_license_status_document(loan) - requested_url = opds2_with_odl_api_fixture.mock_http.requests.pop() - assert loan.external_identifier == requested_url + requested_document = opds2_with_odl_api_fixture.api._request_loan_status( + "GET", "http://loan" + ) + assert "GET" == opds2_with_odl_api_fixture.mock_http.requests_methods.pop() + assert "http://loan" == opds2_with_odl_api_fixture.mock_http.requests.pop() + assert requested_document == expected_document - def test_get_license_status_document_errors( + @pytest.mark.parametrize( + "status, headers, content, exception, expected_log_message", + [ + pytest.param( + 200, + {}, + "not json", + RemoteIntegrationException, + "Error validating Loan Status Document. 'http://loan' returned and invalid document.", + id="invalid json", + ), + pytest.param( + 200, + {}, + json.dumps(dict(status="unknown")), + RemoteIntegrationException, + "Error validating Loan Status Document. 'http://loan' returned and invalid document.", + id="invalid document", + ), + pytest.param( + 403, + {"header": "value"}, + "server error", + RemoteIntegrationException, + "Error requesting Loan Status Document. 'http://loan' returned status code 403. " + "Response headers: header: value. Response content: server error.", + id="bad status code", + ), + pytest.param( + 403, + {"Content-Type": "application/api-problem+json"}, + json.dumps( + dict( + type="http://problem-detail-uri", + title="server error", + detail="broken", + ) + ), + RemoteIntegrationException, + "Error requesting Loan Status Document. 'http://loan' returned status code 403. " + "Problem Detail: 'http://problem-detail-uri' - server error - broken", + id="problem detail response", + ), + ], + ) + def test__request_loan_status_errors( self, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, caplog: pytest.LogCaptureFixture, + status: int, + headers: dict[str, str], + content: str, + exception: type[Exception], + expected_log_message: str, ) -> None: - loan, _ = opds2_with_odl_api_fixture.license.loan_to( - opds2_with_odl_api_fixture.patron - ) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content="not json") - pytest.raises( - RemoteIntegrationException, - opds2_with_odl_api_fixture.api.get_license_status_document, - loan, - ) - + # The response can't be parsed as JSON. opds2_with_odl_api_fixture.mock_http.queue_response( - 200, content=json.dumps(dict(status="unknown")) - ) - pytest.raises( - RemoteIntegrationException, - opds2_with_odl_api_fixture.api.get_license_status_document, - loan, + status, other_headers=headers, content=content ) - - opds2_with_odl_api_fixture.mock_http.queue_response( - 403, content="just junk " * 100 - ) - pytest.raises( - RemoteIntegrationException, - opds2_with_odl_api_fixture.api.get_license_status_document, - loan, - ) - assert "returned status code 403. Expected 2XX." in caplog.text - assert "just junk ..." in caplog.text + with pytest.raises(exception): + opds2_with_odl_api_fixture.api._request_loan_status("GET", "http://loan") + assert expected_log_message in caplog.text def test_checkin_success( self, @@ -315,13 +293,11 @@ def test_checkin_success( # The patron returns the book successfully. opds2_with_odl_api_fixture.checkin() - assert 3 == len(opds2_with_odl_api_fixture.mock_http.requests) + assert 2 == len(opds2_with_odl_api_fixture.mock_http.requests) assert "http://loan" in opds2_with_odl_api_fixture.mock_http.requests[0] assert "http://return" == opds2_with_odl_api_fixture.mock_http.requests[1] - assert "http://loan" in opds2_with_odl_api_fixture.mock_http.requests[2] - # The pool's availability has increased, and the local loan has - # been deleted. + # The pool's availability has increased assert 7 == opds2_with_odl_api_fixture.pool.licenses_available assert 0 == db.session.query(Loan).count() @@ -350,10 +326,9 @@ def test_checkin_success_with_holds_queue( # The first patron returns the book successfully. opds2_with_odl_api_fixture.checkin() - assert 3 == len(opds2_with_odl_api_fixture.mock_http.requests) + assert 2 == len(opds2_with_odl_api_fixture.mock_http.requests) assert "http://loan" in opds2_with_odl_api_fixture.mock_http.requests[0] assert "http://return" == opds2_with_odl_api_fixture.mock_http.requests[1] - assert "http://loan" in opds2_with_odl_api_fixture.mock_http.requests[2] # Now the license is reserved for the next patron. assert 0 == opds2_with_odl_api_fixture.pool.licenses_available @@ -362,34 +337,6 @@ def test_checkin_success_with_holds_queue( assert 0 == db.session.query(Loan).count() assert 0 == hold.position - def test_checkin_already_fulfilled( - self, - db: DatabaseTransactionFixture, - opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, - ) -> None: - # The loan is already fulfilled. - opds2_with_odl_api_fixture.setup_license(concurrency=7, available=6) - loan, _ = opds2_with_odl_api_fixture.license.loan_to( - opds2_with_odl_api_fixture.patron - ) - loan.external_identifier = db.fresh_str() - loan.end = utc_now() + datetime.timedelta(days=3) - - lsd = json.dumps( - { - "status": "active", - } - ) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - # Checking in the book silently does nothing. - opds2_with_odl_api_fixture.api.checkin( - opds2_with_odl_api_fixture.patron, "pinn", opds2_with_odl_api_fixture.pool - ) - assert 1 == len(opds2_with_odl_api_fixture.mock_http.requests) - assert 6 == opds2_with_odl_api_fixture.pool.licenses_available - assert 1 == db.session.query(Loan).count() - def test_checkin_not_checked_out( self, db: DatabaseTransactionFixture, @@ -411,16 +358,14 @@ def test_checkin_not_checked_out( loan.external_identifier = db.fresh_str() loan.end = utc_now() + datetime.timedelta(days=3) - lsd = json.dumps( - { - "status": "revoked", - } + opds2_with_odl_api_fixture.mock_http.queue_response( + 200, + content=opds2_with_odl_api_fixture.loan_status_document( + "revoked" + ).model_dump_json(), ) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - pytest.raises( - NotCheckedOut, - opds2_with_odl_api_fixture.api.checkin, + # Checking in silently does nothing. + opds2_with_odl_api_fixture.api.checkin( opds2_with_odl_api_fixture.patron, "pin", opds2_with_odl_api_fixture.pool, @@ -438,38 +383,34 @@ def test_checkin_cannot_return( loan.external_identifier = db.fresh_str() loan.end = utc_now() + datetime.timedelta(days=3) - lsd = json.dumps( - { - "status": "ready", - } - ) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - # Checking in silently does nothing. - opds2_with_odl_api_fixture.api.checkin( - opds2_with_odl_api_fixture.patron, "pin", opds2_with_odl_api_fixture.pool - ) + opds2_with_odl_api_fixture.mock_http.queue_response( + 200, + content=opds2_with_odl_api_fixture.loan_status_document( + "ready", return_link=False + ).model_dump_json(), + ) + # Checking in raises the CannotReturn exception, since the distributor + # does not support returning the book. + with pytest.raises(CannotReturn): + opds2_with_odl_api_fixture.api.checkin( + opds2_with_odl_api_fixture.patron, + "pin", + opds2_with_odl_api_fixture.pool, + ) - # If the return link doesn't change the status, it still - # silently ignores the problem. - lsd = json.dumps( - { - "status": "ready", - "links": [ - { - "rel": "return", - "href": "http://return", - } - ], - } - ) + # If the return link doesn't change the status, we raise the same exception. + lsd = opds2_with_odl_api_fixture.loan_status_document( + "ready", return_link="http://return" + ).model_dump_json() opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - opds2_with_odl_api_fixture.mock_http.queue_response(200, content="Deleted") opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - opds2_with_odl_api_fixture.api.checkin( - opds2_with_odl_api_fixture.patron, "pin", opds2_with_odl_api_fixture.pool - ) + with pytest.raises(CannotReturn): + opds2_with_odl_api_fixture.api.checkin( + opds2_with_odl_api_fixture.patron, + "pin", + opds2_with_odl_api_fixture.pool, + ) def test_checkin_open_access( self, @@ -528,6 +469,52 @@ def test_checkout_success( assert 5 == opds2_with_odl_api_fixture.pool.licenses_available assert 29 == opds2_with_odl_api_fixture.license.checkouts_left + # The parameters that we templated into the checkout URL are correct. + requested_url = opds2_with_odl_api_fixture.mock_http.requests.pop() + + parsed = urlparse(requested_url) + assert "https" == parsed.scheme + assert "loan.feedbooks.net" == parsed.netloc + params = parse_qs(parsed.query) + + assert ( + opds2_with_odl_api_fixture.api.settings.passphrase_hint == params["hint"][0] + ) + assert ( + opds2_with_odl_api_fixture.api.settings.passphrase_hint_url + == params["hint_url"][0] + ) + + assert opds2_with_odl_api_fixture.license.identifier == params["id"][0] + + # The checkout id and patron id are random UUIDs. + checkout_id = params["checkout_id"][0] + assert uuid.UUID(checkout_id) + patron_id = params["patron_id"][0] + assert uuid.UUID(patron_id) + + # Loans expire in 21 days by default. + now = utc_now() + after_expiration = now + datetime.timedelta(days=23) + expires = urllib.parse.unquote(params["expires"][0]) + + # The expiration time passed to the server is associated with + # the UTC time zone. + assert expires.endswith("+00:00") + expires_t = dateutil.parser.parse(expires) + assert expires_t.tzinfo == dateutil.tz.tz.tzutc() + + # It's a time in the future, but not _too far_ in the future. + assert expires_t > now + assert expires_t < after_expiration + + notification_url = urllib.parse.unquote_plus(params["notification_url"][0]) + assert ( + "http://odl_notify?library_short_name=%s&loan_id=%s" + % (opds2_with_odl_api_fixture.library.short_name, db_loan.id) + == notification_url + ) + def test_checkout_open_access( self, db: DatabaseTransactionFixture, @@ -538,11 +525,8 @@ def test_checkout_open_access( with_open_access_download=True, collection=opds2_with_odl_api_fixture.collection, ) - loan = opds2_with_odl_api_fixture.api.checkout( - opds2_with_odl_api_fixture.patron, - "pin", - oa_work.license_pools[0], - MagicMock(), + loan = opds2_with_odl_api_fixture.api_checkout( + licensepool=oa_work.license_pools[0], ) assert loan.collection(db.session) == opds2_with_odl_api_fixture.collection @@ -596,6 +580,34 @@ def test_checkout_success_with_hold( assert 0 == opds2_with_odl_api_fixture.pool.patrons_in_hold_queue assert 0 == db.session.query(Hold).count() + def test_checkout_success_external_identifier_fallback( + self, + db: DatabaseTransactionFixture, + opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, + opds_files_fixture: OPDSFilesFixture, + ) -> None: + # This book is available to check out. + opds2_with_odl_api_fixture.setup_license(concurrency=1, available=1) + + # The server returns a loan status document with no self link, but the license document + # has a link to the loan status document, so we make the extra request to get the external identifier + # from the license document. + opds2_with_odl_api_fixture.mock_http.queue_response( + 201, + content=opds2_with_odl_api_fixture.loan_status_document( + self_link=False, + ).model_dump_json(), + ) + opds2_with_odl_api_fixture.mock_http.queue_response( + 201, content=opds_files_fixture.sample_text("lcp/license/ul.json") + ) + loan = opds2_with_odl_api_fixture.api_checkout() + assert ( + loan.external_identifier + == "https://license.example.com/licenses/123-456/status" + ) + assert len(opds2_with_odl_api_fixture.mock_http.requests) == 2 + def test_checkout_already_checked_out( self, db: DatabaseTransactionFixture, @@ -629,14 +641,8 @@ def test_checkout_expired_hold( ) opds2_with_odl_api_fixture.setup_license(concurrency=2, available=1) - pytest.raises( - NoAvailableCopies, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoAvailableCopies): + opds2_with_odl_api_fixture.api_checkout() def test_checkout_no_available_copies( self, @@ -647,14 +653,8 @@ def test_checkout_no_available_copies( opds2_with_odl_api_fixture.setup_license(concurrency=1, available=0) existing_loan, _ = opds2_with_odl_api_fixture.license.loan_to(db.patron()) - pytest.raises( - NoAvailableCopies, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoAvailableCopies): + opds2_with_odl_api_fixture.api_checkout() assert 1 == db.session.query(Loan).count() @@ -670,14 +670,8 @@ def test_checkout_no_available_copies( ) opds2_with_odl_api_fixture.pool.update_availability_from_licenses() - pytest.raises( - NoAvailableCopies, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoAvailableCopies): + opds2_with_odl_api_fixture.api_checkout() assert 0 == db.session.query(Loan).count() @@ -687,14 +681,8 @@ def test_checkout_no_available_copies( ) opds2_with_odl_api_fixture.pool.update_availability_from_licenses() - pytest.raises( - NoAvailableCopies, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoAvailableCopies): + opds2_with_odl_api_fixture.api_checkout() assert 0 == db.session.query(Loan).count() @@ -703,14 +691,8 @@ def test_checkout_no_available_copies( hold.end = yesterday opds2_with_odl_api_fixture.pool.update_availability_from_licenses() - pytest.raises( - NoAvailableCopies, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoAvailableCopies): + opds2_with_odl_api_fixture.api_checkout() assert 0 == db.session.query(Loan).count() @@ -728,14 +710,6 @@ def test_checkout_no_available_copies_unknown_to_us( The title has no available copies, but we are out of sync with the distributor, so we think there are copies available. """ - checkout = partial( - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - MagicMock(), - ) - # We think there are copies available. license = opds2_with_odl_api_fixture.setup_license(concurrency=1, available=1) @@ -747,7 +721,7 @@ def test_checkout_no_available_copies_unknown_to_us( ) with pytest.raises(NoAvailableCopies): - checkout() + opds2_with_odl_api_fixture.api_checkout() assert db.session.query(Loan).count() == 0 assert license.license_pool.licenses_available == 0 @@ -758,14 +732,6 @@ def test_checkout_failures( db: DatabaseTransactionFixture, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, ) -> None: - checkout = partial( - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - MagicMock(), - ) - # We think there are copies available. opds2_with_odl_api_fixture.setup_license(concurrency=1, available=1) @@ -777,14 +743,14 @@ def test_checkout_failures( ) with pytest.raises(BadResponseException): - checkout() + opds2_with_odl_api_fixture.api_checkout() # Test the case where we just get an unknown bad response. opds2_with_odl_api_fixture.mock_http.queue_response( 500, "text/plain", content="halt and catch fire 🔥" ) with pytest.raises(BadResponseException): - checkout() + opds2_with_odl_api_fixture.api_checkout() def test_checkout_no_licenses( self, @@ -793,14 +759,8 @@ def test_checkout_no_licenses( ) -> None: opds2_with_odl_api_fixture.setup_license(concurrency=1, available=1, left=0) - pytest.raises( - NoLicenses, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoLicenses): + opds2_with_odl_api_fixture.api_checkout() assert 0 == db.session.query(Loan).count() @@ -815,14 +775,8 @@ def test_checkout_when_all_licenses_expired( expires=utc_now() - datetime.timedelta(weeks=1), ) - pytest.raises( - NoLicenses, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoLicenses): + opds2_with_odl_api_fixture.api_checkout() # license expired by no remaining checkouts opds2_with_odl_api_fixture.setup_license( @@ -832,62 +786,39 @@ def test_checkout_when_all_licenses_expired( expires=utc_now() + datetime.timedelta(weeks=1), ) - pytest.raises( - NoLicenses, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, - ) + with pytest.raises(NoLicenses): + opds2_with_odl_api_fixture.api_checkout() def test_checkout_cannot_loan( self, db: DatabaseTransactionFixture, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, ) -> None: - lsd = json.dumps( - { - "status": "revoked", - } - ) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - pytest.raises( - CannotLoan, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, + opds2_with_odl_api_fixture.mock_http.queue_response( + 200, + content=opds2_with_odl_api_fixture.loan_status_document( + "revoked" + ).model_dump_json(), ) - + with pytest.raises(CannotLoan): + opds2_with_odl_api_fixture.api_checkout() assert 0 == db.session.query(Loan).count() # No external identifier. - lsd = json.dumps( - { - "status": "ready", - "potential_rights": {"end": "2017-10-21T11:12:13Z"}, - } - ) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) - pytest.raises( - CannotLoan, - opds2_with_odl_api_fixture.api.checkout, - opds2_with_odl_api_fixture.patron, - "pin", - opds2_with_odl_api_fixture.pool, - Representation.EPUB_MEDIA_TYPE, + opds2_with_odl_api_fixture.mock_http.queue_response( + 200, + content=opds2_with_odl_api_fixture.loan_status_document( + self_link=False, license_link=False + ).model_dump_json(), ) - + with pytest.raises(CannotLoan): + opds2_with_odl_api_fixture.api_checkout() assert 0 == db.session.query(Loan).count() @pytest.mark.parametrize( - "delivery_mechanism, correct_type, correct_link, links", + "drm_scheme, correct_type, correct_link, links", [ - ( + pytest.param( DeliveryMechanism.ADOBE_DRM, DeliveryMechanism.ADOBE_DRM, "http://acsm", @@ -898,20 +829,35 @@ def test_checkout_cannot_loan( "type": DeliveryMechanism.ADOBE_DRM, } ], + id="adobe drm", ), - ( - MediaTypes.AUDIOBOOK_MANIFEST_MEDIA_TYPE, - MediaTypes.AUDIOBOOK_MANIFEST_MEDIA_TYPE, - "http://manifest", + pytest.param( + DeliveryMechanism.LCP_DRM, + DeliveryMechanism.LCP_DRM, + "http://lcp", [ { - "rel": "manifest", - "href": "http://manifest", - "type": MediaTypes.AUDIOBOOK_MANIFEST_MEDIA_TYPE, + "rel": "license", + "href": "http://lcp", + "type": DeliveryMechanism.LCP_DRM, } ], + id="lcp drm", ), - ( + pytest.param( + DeliveryMechanism.NO_DRM, + "application/epub+zip", + "http://publication", + [ + { + "rel": "publication", + "href": "http://publication", + "type": "application/epub+zip", + } + ], + id="no drm", + ), + pytest.param( DeliveryMechanism.FEEDBOOKS_AUDIOBOOK_DRM, FEEDBOOKS_AUDIO, "http://correct", @@ -927,6 +873,7 @@ def test_checkout_cannot_loan( "type": FEEDBOOKS_AUDIO, }, ], + id="feedbooks audio", ), ], ) @@ -934,10 +881,10 @@ def test_fulfill_success( self, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, db: DatabaseTransactionFixture, - delivery_mechanism: str, + drm_scheme: str, correct_type: str, correct_link: str, - links: dict[str, Any], + links: list[dict[str, str]], ) -> None: # Fulfill a loan in a way that gives access to a license file. opds2_with_odl_api_fixture.setup_license(concurrency=1, available=1) @@ -945,28 +892,30 @@ def test_fulfill_success( lpdm = MagicMock(spec=LicensePoolDeliveryMechanism) lpdm.delivery_mechanism = MagicMock(spec=DeliveryMechanism) - lpdm.delivery_mechanism.content_type = "ignored/format" - lpdm.delivery_mechanism.drm_scheme = delivery_mechanism - - lsd = json.dumps( - { - "status": "ready", - "potential_rights": {"end": "2017-10-21T11:12:13Z"}, - "links": links, - } + lpdm.delivery_mechanism.content_type = ( + "ignored/format" if drm_scheme != DeliveryMechanism.NO_DRM else correct_type ) + lpdm.delivery_mechanism.drm_scheme = drm_scheme - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) + lsd = opds2_with_odl_api_fixture.loan_status_document("active", links=links) + opds2_with_odl_api_fixture.mock_http.queue_response( + 200, content=lsd.model_dump_json() + ) fulfillment = opds2_with_odl_api_fixture.api.fulfill( opds2_with_odl_api_fixture.patron, "pin", opds2_with_odl_api_fixture.pool, lpdm, ) - assert isinstance(fulfillment, FetchFulfillment) - assert correct_link == fulfillment.content_link - assert correct_type == fulfillment.content_type - assert ["2xx"] == fulfillment.allowed_response_codes + assert ( + isinstance(fulfillment, FetchFulfillment) + if drm_scheme != DeliveryMechanism.NO_DRM + else isinstance(fulfillment, RedirectFulfillment) + ) + assert correct_link == fulfillment.content_link # type: ignore[attr-defined] + assert correct_type == fulfillment.content_type # type: ignore[attr-defined] + if isinstance(fulfillment, FetchFulfillment): + assert fulfillment.allowed_response_codes == ["2xx"] def test_fulfill_open_access( self, @@ -985,14 +934,10 @@ def test_fulfill_open_access( spec=LicensePoolDeliveryMechanism, delivery_mechanism=MagicMock(drm_scheme=None), ) - pytest.raises( - CannotFulfill, - opds2_with_odl_api_fixture.api.fulfill, - opds2_with_odl_api_fixture.patron, - "pin", - pool, - mock_lpdm, - ) + with pytest.raises(CannotFulfill): + opds2_with_odl_api_fixture.api.fulfill( + opds2_with_odl_api_fixture.patron, "pin", pool, mock_lpdm + ) lpdm = pool.delivery_mechanisms[0] fulfillment = opds2_with_odl_api_fixture.api.fulfill( @@ -1066,15 +1011,20 @@ def test_fulfill_bearer_token( @pytest.mark.parametrize( "status_document, updated_availability", [ - ({"status": "revoked"}, True), - ({"status": "cancelled"}, True), - ( - { - "status": "active", - "potential_rights": {"end": "2017-10-21T11:12:13Z"}, - "links": [], - }, + pytest.param( + OPDS2WithODLApiFixture.loan_status_document("revoked"), + True, + id="revoked", + ), + pytest.param( + OPDS2WithODLApiFixture.loan_status_document("cancelled"), + True, + id="cancelled", + ), + pytest.param( + OPDS2WithODLApiFixture.loan_status_document("active"), False, + id="missing link", ), ], ) @@ -1082,7 +1032,7 @@ def test_fulfill_cannot_fulfill( self, db: DatabaseTransactionFixture, opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, - status_document: dict[str, Any], + status_document: LoanStatus, updated_availability: bool, ) -> None: opds2_with_odl_api_fixture.setup_license(concurrency=7, available=7) @@ -1091,9 +1041,9 @@ def test_fulfill_cannot_fulfill( assert 1 == db.session.query(Loan).count() assert 6 == opds2_with_odl_api_fixture.pool.licenses_available - lsd = json.dumps(status_document) - - opds2_with_odl_api_fixture.mock_http.queue_response(200, content=lsd) + opds2_with_odl_api_fixture.mock_http.queue_response( + 200, content=status_document.model_dump_json() + ) with pytest.raises(CannotFulfill): opds2_with_odl_api_fixture.api.fulfill( opds2_with_odl_api_fixture.patron, @@ -1847,27 +1797,13 @@ def test_update_loan_still_active( opds2_with_odl_api_fixture.patron ) loan.external_identifier = db.fresh_str() - status_doc = { - "status": "active", - } + status_doc = opds2_with_odl_api_fixture.loan_status_document("active") opds2_with_odl_api_fixture.api.update_loan(loan, status_doc) # Availability hasn't changed, and the loan still exists. assert 6 == opds2_with_odl_api_fixture.pool.licenses_available assert 1 == db.session.query(Loan).count() - def test_update_loan_bad_status( - self, - db: DatabaseTransactionFixture, - opds2_with_odl_api_fixture: OPDS2WithODLApiFixture, - ) -> None: - status_doc = { - "status": "foo", - } - - with pytest.raises(RemoteIntegrationException, match="unknown status value"): - opds2_with_odl_api_fixture.api.update_loan(MagicMock(), status_doc) - def test_update_loan_removes_loan( self, db: DatabaseTransactionFixture, @@ -1879,9 +1815,7 @@ def test_update_loan_removes_loan( assert 6 == opds2_with_odl_api_fixture.pool.licenses_available assert 1 == db.session.query(Loan).count() - status_doc = { - "status": "cancelled", - } + status_doc = opds2_with_odl_api_fixture.loan_status_document("cancelled") opds2_with_odl_api_fixture.api.update_loan(loan, status_doc) @@ -1903,9 +1837,7 @@ def test_update_loan_removes_loan_with_hold_queue( assert opds2_with_odl_api_fixture.pool.licenses_reserved == 0 assert opds2_with_odl_api_fixture.pool.patrons_in_hold_queue == 1 - status_doc = { - "status": "cancelled", - } + status_doc = opds2_with_odl_api_fixture.loan_status_document("cancelled") opds2_with_odl_api_fixture.api.update_loan(loan, status_doc) diff --git a/tests/manager/api/odl/test_auth.py b/tests/manager/api/odl/test_auth.py index eb963be574..c545f4d6ae 100644 --- a/tests/manager/api/odl/test_auth.py +++ b/tests/manager/api/odl/test_auth.py @@ -10,7 +10,11 @@ from freezegun import freeze_time from typing_extensions import Self -from palace.manager.api.odl.auth import ODLAuthenticatedGet, TokenTuple +from palace.manager.api.odl.auth import ( + OdlAuthenticatedRequest, + OpdsWithOdlException, + TokenTuple, +) from palace.manager.api.odl.settings import OPDS2AuthType from palace.manager.core.exceptions import IntegrationException, PalaceValueError from palace.manager.util.datetime_helpers import utc_now @@ -18,7 +22,61 @@ from tests.mocks.mock import MockRequestsResponse -class MockODLAuthenticatedGet(ODLAuthenticatedGet): +class TestOpdsWithOdlException: + @pytest.mark.parametrize( + "code,type,data,none_response", + [ + pytest.param(400, None, "Error", True, id="no content type"), + pytest.param( + 500, "application/json", "Error", True, id="unsupported content type" + ), + pytest.param( + 404, + "application/problem+json", + "{}", + True, + id="missing required fields", + ), + pytest.param( + 420, + "application/api-problem+json", + "hot garbage", + True, + id="invalid json", + ), + pytest.param( + 404, + "application/problem+json", + json.dumps( + { + "type": "http://problem-uri", + "title": "Robot overlords on strike", + "status": 404, + } + ), + False, + id="missing required fields", + ), + ], + ) + def test_from_response( + self, code: int, type: str, data: str, none_response: bool + ) -> None: + headers = {} + if type: + headers["Content-Type"] = type + response = MockRequestsResponse(code, headers, data) + exception = OpdsWithOdlException.from_response(response) + + if none_response: + assert exception is None + else: + assert isinstance(exception, OpdsWithOdlException) + assert exception.status == code + assert exception.problem_detail.response[0] == data + + +class MockOdlAuthenticatedRequest(OdlAuthenticatedRequest): def __init__( self, username: str, password: str, auth_type: OPDS2AuthType, feed_url: str ) -> None: @@ -45,7 +103,7 @@ def _feed_url(self) -> str: return self.feed_url -class AuthenticatedGetFixture: +class AuthenticatedRequestFixture: def __init__(self, request_with_timeout: MagicMock) -> None: self.username = "username" self.password = "password" @@ -54,8 +112,8 @@ def __init__(self, request_with_timeout: MagicMock) -> None: self.auth_url = "http://authenticate.example.com" self.request_url = "http://example.com/123" self.headers = {"header": "value"} - self.authenticated_get = partial( - MockODLAuthenticatedGet, + self.authenticated_request = partial( + MockOdlAuthenticatedRequest, username=self.username, password=self.password, feed_url=self.feed_url, @@ -122,21 +180,21 @@ def __init__(self, request_with_timeout: MagicMock) -> None: auth=BearerAuth(self.token), ) - def initialize_authenticated_get( + def initialize_token( self, - authenticated_get: MockODLAuthenticatedGet | None = None, + authenticated_request: MockOdlAuthenticatedRequest | None = None, *, expired: bool = False - ) -> MockODLAuthenticatedGet: - # Set the token url and session token so that the authenticated_get can make requests + ) -> MockOdlAuthenticatedRequest: + # Set the token url and session token so that the authenticated_request can make requests # without first going through the refresh process - if authenticated_get is None: - authenticated_get = self.authenticated_get() - authenticated_get._token_url = self.auth_url - authenticated_get._session_token = ( + if authenticated_request is None: + authenticated_request = self.authenticated_request() + authenticated_request._token_url = self.auth_url + authenticated_request._session_token = ( self.valid_token if not expired else self.expired_token ) - return authenticated_get + return authenticated_request @property def auth_document(self) -> dict[str, Any]: @@ -172,59 +230,67 @@ def fixture(cls) -> Generator[Self, None, None]: @pytest.fixture -def authenticated_get_fixture() -> Generator[AuthenticatedGetFixture, None, None]: - with AuthenticatedGetFixture.fixture() as fixture: +def authenticated_request_fixture() -> ( + Generator[AuthenticatedRequestFixture, None, None] +): + with AuthenticatedRequestFixture.fixture() as fixture: yield fixture -class TestODLAuthenticatedGet: - def test__basic_auth_get( - self, authenticated_get_fixture: AuthenticatedGetFixture +class TestODLAuthenticatedRequest: + def test__basic_auth_request( + self, authenticated_request_fixture: AuthenticatedRequestFixture ) -> None: - mock_request_with_timeout = authenticated_get_fixture.request_with_timeout - authenticated_get = authenticated_get_fixture.authenticated_get( + mock_request_with_timeout = authenticated_request_fixture.request_with_timeout + authenticated_request = authenticated_request_fixture.authenticated_request( auth_type=OPDS2AuthType.BASIC ) - response = authenticated_get._get( - authenticated_get_fixture.request_url, authenticated_get_fixture.headers + response = authenticated_request._request( + "GET", + authenticated_request_fixture.request_url, + authenticated_request_fixture.headers, ) assert response == mock_request_with_timeout.return_value mock_request_with_timeout.assert_called_once_with( "GET", - authenticated_get_fixture.request_url, - headers=authenticated_get_fixture.headers, + authenticated_request_fixture.request_url, + headers=authenticated_request_fixture.headers, auth=( - authenticated_get_fixture.username, - authenticated_get_fixture.password, + authenticated_request_fixture.username, + authenticated_request_fixture.password, ), ) - def test__no_auth_get( - self, authenticated_get_fixture: AuthenticatedGetFixture + def test__no_auth_request( + self, authenticated_request_fixture: AuthenticatedRequestFixture ) -> None: - mock_request_with_timeout = authenticated_get_fixture.request_with_timeout - authenticated_get = authenticated_get_fixture.authenticated_get( + mock_request_with_timeout = authenticated_request_fixture.request_with_timeout + authenticated_request = authenticated_request_fixture.authenticated_request( auth_type=OPDS2AuthType.NONE ) - response = authenticated_get._get( - authenticated_get_fixture.request_url, authenticated_get_fixture.headers + response = authenticated_request._request( + "GET", + authenticated_request_fixture.request_url, + authenticated_request_fixture.headers, ) assert response == mock_request_with_timeout.return_value mock_request_with_timeout.assert_called_once_with( "GET", - authenticated_get_fixture.request_url, - headers=authenticated_get_fixture.headers, + authenticated_request_fixture.request_url, + headers=authenticated_request_fixture.headers, ) def test__unknown_auth_type( - self, authenticated_get_fixture: AuthenticatedGetFixture + self, authenticated_request_fixture: AuthenticatedRequestFixture ) -> None: - authenticated_get = authenticated_get_fixture.authenticated_get( + authenticated_request = authenticated_request_fixture.authenticated_request( auth_type="invalid" # type: ignore[arg-type] ) with pytest.raises(PalaceValueError) as exc_info: - authenticated_get._get( - authenticated_get_fixture.request_url, authenticated_get_fixture.headers + authenticated_request._request( + "GET", + authenticated_request_fixture.request_url, + authenticated_request_fixture.headers, ) assert str(exc_info.value) == "Invalid OPDS2AuthType: 'invalid'" @@ -337,11 +403,11 @@ def test__unknown_auth_type( ) def test__get_oauth_url_from_auth_document( self, - authenticated_get_fixture: AuthenticatedGetFixture, + authenticated_request_fixture: AuthenticatedRequestFixture, authentication: list[dict[str, Any]], expected: type[Exception] | str, ) -> None: - auth_document = authenticated_get_fixture.auth_document + auth_document = authenticated_request_fixture.auth_document auth_document["authentication"] = authentication context = ( nullcontext() if isinstance(expected, str) else pytest.raises(expected) @@ -349,7 +415,7 @@ def test__get_oauth_url_from_auth_document( with context: assert ( - MockODLAuthenticatedGet._get_oauth_url_from_auth_document( + MockOdlAuthenticatedRequest._get_oauth_url_from_auth_document( json.dumps(auth_document) ) == expected @@ -383,11 +449,11 @@ def test__get_oauth_url_from_auth_document( @freeze_time("2021-01-01") def test__oauth_session_token_refresh( self, - authenticated_get_fixture: AuthenticatedGetFixture, + authenticated_request_fixture: AuthenticatedRequestFixture, data: str, expected: TokenTuple | type[Exception], ) -> None: - mock_request_with_timeout = authenticated_get_fixture.request_with_timeout + mock_request_with_timeout = authenticated_request_fixture.request_with_timeout mock_request_with_timeout.return_value = MockRequestsResponse(200, {}, data) context = ( nullcontext() @@ -396,30 +462,32 @@ def test__oauth_session_token_refresh( ) with context: - token = MockODLAuthenticatedGet._oauth_session_token_refresh( - authenticated_get_fixture.auth_url, - authenticated_get_fixture.username, - authenticated_get_fixture.password, + token = MockOdlAuthenticatedRequest._oauth_session_token_refresh( + authenticated_request_fixture.auth_url, + authenticated_request_fixture.username, + authenticated_request_fixture.password, ) assert token == expected assert mock_request_with_timeout.call_count == 1 mock_request_with_timeout.assert_has_calls( - [authenticated_get_fixture.request_with_timeout_calls["token_grant"]()] + [authenticated_request_fixture.request_with_timeout_calls["token_grant"]()] ) def test__oauth_get_failed_auth_document_request( - self, authenticated_get_fixture: AuthenticatedGetFixture + self, authenticated_request_fixture: AuthenticatedRequestFixture ) -> None: """ If the auth document request fails, an exception is raised. """ - mock_request_with_timeout = authenticated_get_fixture.request_with_timeout + mock_request_with_timeout = authenticated_request_fixture.request_with_timeout mock_request_with_timeout.return_value = ( - authenticated_get_fixture.responses.get("other_401") + authenticated_request_fixture.responses.get("other_401") ) with pytest.raises(IntegrationException) as exc_info: - authenticated_get_fixture.authenticated_get()._get( - authenticated_get_fixture.request_url, authenticated_get_fixture.headers + authenticated_request_fixture.authenticated_request()._request( + "GET", + authenticated_request_fixture.request_url, + authenticated_request_fixture.headers, ) assert "Unable to fetch OPDS authentication document" in str(exc_info.value) @@ -477,54 +545,60 @@ def test__oauth_get_failed_auth_document_request( ), ], ) - def test__oauth_get( + def test__oauth_request( self, - authenticated_get_fixture: AuthenticatedGetFixture, + authenticated_request_fixture: AuthenticatedRequestFixture, responses: list[str], calls: list[str], initialized: bool, expired: bool, ) -> None: - mock_request_with_timeout = authenticated_get_fixture.request_with_timeout - authenticated_get = authenticated_get_fixture.authenticated_get() + mock_request_with_timeout = authenticated_request_fixture.request_with_timeout + authenticated_request = authenticated_request_fixture.authenticated_request() if initialized: - authenticated_get = authenticated_get_fixture.initialize_authenticated_get( - authenticated_get, expired=expired + authenticated_request = authenticated_request_fixture.initialize_token( + authenticated_request, expired=expired ) - responses_data = [authenticated_get_fixture.responses[r] for r in responses] + responses_data = [authenticated_request_fixture.responses[r] for r in responses] mock_request_with_timeout.side_effect = responses_data final_response = responses_data[-1] assert ( - authenticated_get._get( - authenticated_get_fixture.request_url, authenticated_get_fixture.headers + authenticated_request._request( + "GET", + authenticated_request_fixture.request_url, + authenticated_request_fixture.headers, ) == final_response ) assert mock_request_with_timeout.call_count == len(calls) mock_request_with_timeout.assert_has_calls( - [authenticated_get_fixture.request_with_timeout_calls[c]() for c in calls] + [ + authenticated_request_fixture.request_with_timeout_calls[c]() + for c in calls + ] ) - def test__oauth_get_allowed_response_codes( - self, authenticated_get_fixture: AuthenticatedGetFixture + def test__oauth_request_allowed_response_codes( + self, authenticated_request_fixture: AuthenticatedRequestFixture ) -> None: """ Calling with allowed_response_codes should still allow a token refresh, but if the refresh fails an exception will be raised. """ - mock_request_with_timeout = authenticated_get_fixture.request_with_timeout - authenticated_get = authenticated_get_fixture.initialize_authenticated_get() + mock_request_with_timeout = authenticated_request_fixture.request_with_timeout + authenticated_request = authenticated_request_fixture.initialize_token() mock_request_with_timeout.side_effect = [ - authenticated_get_fixture.responses.get("auth_document_401"), - authenticated_get_fixture.responses.get("token_grant"), - authenticated_get_fixture.responses.get("other_401"), + authenticated_request_fixture.responses.get("auth_document_401"), + authenticated_request_fixture.responses.get("token_grant"), + authenticated_request_fixture.responses.get("other_401"), ] with pytest.raises(IntegrationException) as exc_info: - authenticated_get._get( - authenticated_get_fixture.request_url, - authenticated_get_fixture.headers, + authenticated_request._request( + "GET", + authenticated_request_fixture.request_url, + authenticated_request_fixture.headers, allowed_response_codes=["2xx"], ) assert ( @@ -532,12 +606,14 @@ def test__oauth_get_allowed_response_codes( in str(exc_info.value) ) assert mock_request_with_timeout.call_count == 3 - token_grant_call = authenticated_get_fixture.request_with_timeout_calls[ + token_grant_call = authenticated_request_fixture.request_with_timeout_calls[ "token_grant" ]() - request_with_token_call = authenticated_get_fixture.request_with_timeout_calls[ - "request_with_token" - ](allowed_response_codes=["2xx", 401]) + request_with_token_call = ( + authenticated_request_fixture.request_with_timeout_calls[ + "request_with_token" + ](allowed_response_codes=["2xx", 401]) + ) mock_request_with_timeout.assert_has_calls( [ diff --git a/tests/manager/api/odl/test_importer.py b/tests/manager/api/odl/test_importer.py index 3bec21ea04..249c1173e0 100644 --- a/tests/manager/api/odl/test_importer.py +++ b/tests/manager/api/odl/test_importer.py @@ -1143,21 +1143,23 @@ def test_get( ): monitor = opds2_with_odl_import_monitor_fixture.monitor - with patch.object(HTTP, "get_with_timeout") as mock_get: + with patch.object(HTTP, "request_with_timeout") as mock_get: monitor._get("/absolute/path", {}) assert mock_get.call_args.args == ( + "GET", "https://opds.import.com:9999/absolute/path", ) - with patch.object(HTTP, "get_with_timeout") as mock_get: + with patch.object(HTTP, "request_with_timeout") as mock_get: monitor._get("relative/path", {}) assert mock_get.call_args.args == ( + "GET", "https://opds.import.com:9999/relative/path", ) - with patch.object(HTTP, "get_with_timeout") as mock_get: + with patch.object(HTTP, "request_with_timeout") as mock_get: monitor._get("http://example.com/full/url") - assert mock_get.call_args.args == ("http://example.com/full/url",) + assert mock_get.call_args.args == ("GET", "http://example.com/full/url") # assert that we set the expected extra args to the HTTP request kwargs = mock_get.call_args.kwargs assert kwargs.get("timeout") == 120 diff --git a/tests/manager/opds/__init__.py b/tests/manager/opds/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/manager/opds/lcp/__init__.py b/tests/manager/opds/lcp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/manager/opds/lcp/test_license.py b/tests/manager/opds/lcp/test_license.py new file mode 100644 index 0000000000..f92233fa16 --- /dev/null +++ b/tests/manager/opds/lcp/test_license.py @@ -0,0 +1,21 @@ +import pytest + +from palace.manager.opds.lcp.license import LicenseDocument +from tests.fixtures.files import OPDSFilesFixture + + +class TestLicenseDocument: + + @pytest.mark.parametrize( + "filename", + [ + "fb.json", + "ul.json", + ], + ) + def test_license_document( + self, filename: str, opds_files_fixture: OPDSFilesFixture + ) -> None: + LicenseDocument.model_validate_json( + opds_files_fixture.sample_data("lcp/license/" + filename) + ) diff --git a/tests/manager/opds/lcp/test_status.py b/tests/manager/opds/lcp/test_status.py new file mode 100644 index 0000000000..bc84265766 --- /dev/null +++ b/tests/manager/opds/lcp/test_status.py @@ -0,0 +1,24 @@ +import pytest + +from palace.manager.opds.lcp.status import LoanStatus +from tests.fixtures.files import OPDSFilesFixture + + +class TestLcpStatus: + + @pytest.mark.parametrize( + "filename", + [ + "fb-active.json", + "fb-book-adobe.json", + "fb-early-return.json", + "ul-active.json", + "ul-returned.json", + ], + ) + def test_lcp_status( + self, filename: str, opds_files_fixture: OPDSFilesFixture + ) -> None: + LoanStatus.model_validate_json( + opds_files_fixture.sample_data("lcp/status/" + filename) + ) diff --git a/tests/manager/opds/odl/__init__.py b/tests/manager/opds/odl/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/manager/opds/odl/test_info.py b/tests/manager/opds/odl/test_info.py new file mode 100644 index 0000000000..a55150ad75 --- /dev/null +++ b/tests/manager/opds/odl/test_info.py @@ -0,0 +1,29 @@ +import pytest + +from palace.manager.opds.odl.info import LicenseInfo +from tests.fixtures.files import OPDSFilesFixture + + +class TestLicenseInfo: + + @pytest.mark.parametrize( + "filename", + [ + "feedbooks-ab-checked-out.json", + "feedbooks-ab-loan-limited.json", + "feedbooks-ab-not-checked-out.json", + "feedbooks-book-adept.json", + "feedbooks-book-unavailable.json", + "ul-ab.json", + "ul-book.json", + ], + ) + def test_license_info( + self, filename: str, opds_files_fixture: OPDSFilesFixture + ) -> None: + info = LicenseInfo.model_validate_json( + opds_files_fixture.sample_data("odl/info/" + filename) + ) + assert info.identifier == "urn:uuid:123" + assert isinstance(info.protection.formats, frozenset) + assert len(info.protection.formats) == 1 diff --git a/tests/manager/opds/test_base.py b/tests/manager/opds/test_base.py new file mode 100644 index 0000000000..b0b54b63da --- /dev/null +++ b/tests/manager/opds/test_base.py @@ -0,0 +1,161 @@ +import json + +import pytest +from pydantic import TypeAdapter, ValidationError + +from palace.manager.core.exceptions import PalaceValueError +from palace.manager.opds.base import BaseLink, ListOfLinks, obj_or_set_to_set + + +def test_obj_or_set_to_set(): + assert obj_or_set_to_set(None) == set() + assert obj_or_set_to_set("foo") == {"foo"} + assert obj_or_set_to_set({"foo"}) == {"foo"} + + +class TestBaseLink: + def test_rels(self): + link = BaseLink(href="http://example.com", rel="foo") + assert link.rels == {"foo"} + link = BaseLink(href="http://example.com", rel={"foo", "bar"}) + assert link.rels == {"foo", "bar"} + + def test_href_templated(self): + link = BaseLink(href="http://example.com", rel="foo") + assert link.href_templated() == "http://example.com" + link = BaseLink(href="http://example.com/{?x,y,z}", rel="foo", templated=True) + assert ( + link.href_templated({"x": 1, "y": "foo"}) == "http://example.com/?x=1&y=foo" + ) + + +class ListOfLinksFixture: + def __init__(self): + self.foo_link = BaseLink( + href="http://example.com/foo", rel="foo", type="application/xyz" + ) + self.bar_link = BaseLink(href="http://example.com/bar", rel="bar") + self.baz_link = BaseLink( + href="http://example.com/baz", rel="bar", type="application/xyz" + ) + self.bam_link = BaseLink( + href="http://example.com/bam", rel="bam", type="application/abc" + ) + self.fizz_link = BaseLink( + href="http://example.com/fizz", rel="foo", type="application/xyz" + ) + + self.list = [ + self.foo_link, + self.bar_link, + self.baz_link, + self.bam_link, + self.fizz_link, + ] + self.links = ListOfLinks(self.list) + self.validator = TypeAdapter(ListOfLinks[BaseLink]) + + +@pytest.fixture +def list_of_links_fixture(): + return ListOfLinksFixture() + + +class TestListOfLinks: + def test_get_list(self, list_of_links_fixture: ListOfLinksFixture) -> None: + links = list_of_links_fixture.links + assert links.get_list() == list_of_links_fixture.list + assert links.get_list(rel="bar") == [ + list_of_links_fixture.bar_link, + list_of_links_fixture.baz_link, + ] + assert links.get_list(type="application/xyz") == [ + list_of_links_fixture.foo_link, + list_of_links_fixture.baz_link, + list_of_links_fixture.fizz_link, + ] + assert links.get_list(rel="bar", type="application/xyz") == [ + list_of_links_fixture.baz_link + ] + + def test_get(self, list_of_links_fixture: ListOfLinksFixture) -> None: + links = list_of_links_fixture.links + assert links.get() == list_of_links_fixture.foo_link + assert links.get(rel="foo") == list_of_links_fixture.foo_link + assert links.get(type="application/xyz") == list_of_links_fixture.foo_link + assert links.get(type="application/abc") == list_of_links_fixture.bam_link + assert ( + links.get(rel="bar", type="application/xyz") + == list_of_links_fixture.baz_link + ) + assert links.get(rel="nonexistent") is None + assert links.get(type="nonexistent") is None + assert links.get(rel="nonexistent", type="nonexistent") is None + + with pytest.raises( + PalaceValueError, match="^No links found with rel='nonexistent'$" + ): + links.get(rel="nonexistent", raising=True) + + with pytest.raises(PalaceValueError, match="^Multiple links found$"): + links.get(raising=True) + with pytest.raises( + PalaceValueError, match="^Multiple links found with type='application/xyz'$" + ): + links.get(type="application/xyz", raising=True) + with pytest.raises( + PalaceValueError, match="^Multiple links found with rel='bar'$" + ): + links.get(rel="bar", raising=True) + with pytest.raises( + PalaceValueError, + match="^Multiple links found with rel='foo' and type='application/xyz'$", + ): + links.get(rel="foo", type="application/xyz", raising=True) + + def test_validate(self, list_of_links_fixture: ListOfLinksFixture) -> None: + validator = list_of_links_fixture.validator + + # The list of links is valid, so it should return the same list. + validated = validator.validate_python(list_of_links_fixture.list) + assert validated == list_of_links_fixture.links + assert isinstance(validated, ListOfLinks) + for link in validated: + assert isinstance(link, BaseLink) + + # The list of links is invalid if there are multiple links with the same relation and type. + invalid_list = list_of_links_fixture.list + [list_of_links_fixture.foo_link] + with pytest.raises( + ValidationError, + match="Duplicate link with relation 'foo', type 'application/xyz' and href 'http://example.com/foo'", + ): + validator.validate_python(invalid_list) + + # Load the list of links from a JSON object. + json_obj = json.dumps( + [ + {"href": "http://example.com/foo", "rel": "foo"}, + { + "href": "http://example.com/bar", + "rel": "bar", + "type": "application/xyz", + }, + ], + separators=(",", ":"), + ) + validated = validator.validate_json(json_obj) + assert len(validated) == 2 + assert isinstance(validated, ListOfLinks) + for link in validated: + assert isinstance(link, BaseLink) + + [first, second] = validated + assert first.href == "http://example.com/foo" + assert first.rel == "foo" + assert first.type is None + + assert second.href == "http://example.com/bar" + assert second.rel == "bar" + assert second.type == "application/xyz" + + assert validator.dump_json(validated, exclude_unset=True) == json_obj.encode() diff --git a/tests/mocks/mock.py b/tests/mocks/mock.py index cdca136a35..73d1932fd8 100644 --- a/tests/mocks/mock.py +++ b/tests/mocks/mock.py @@ -236,6 +236,7 @@ def __init__(self) -> None: self.responses: list[Response] = [] self.requests: list[str] = [] self.requests_args: list[Args] = [] + self.requests_methods: list[str] = [] def queue_response( self, @@ -251,17 +252,21 @@ def queue_response( self.responses.append(MockRequestsResponse(response_code, headers, content)) - def _get(self, *args: Any, **kwargs: Any) -> Response: + def _request(self, *args: Any, **kwargs: Any) -> Response: return self.responses.pop(0) - def do_get(self, url: str, *args: Any, **kwargs: Any) -> Response: + def do_request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Response: self.requests.append(url) + self.requests_methods.append(method) self.requests_args.append(Args(args, kwargs)) - return HTTP._request_with_timeout(url, self._get, *args, **kwargs) + return HTTP._request_with_timeout(url, self._request, *args, **kwargs) + + def do_get(self, url: str, *args: Any, **kwargs: Any) -> Response: + return self.do_request("GET", url, *args, **kwargs) @contextmanager def patch(self) -> Generator[None, None, None]: - with patch.object(HTTP, "get_with_timeout", self.do_get): + with patch.object(HTTP, "request_with_timeout", self.do_request): yield diff --git a/tests/mocks/odl.py b/tests/mocks/odl.py index bc7e7b346e..40ebce5934 100644 --- a/tests/mocks/odl.py +++ b/tests/mocks/odl.py @@ -44,7 +44,29 @@ def _url_for(self, *args: Any, **kwargs: Any) -> str: "&".join([f"{key}={val}" for key, val in list(kwargs.items())]), ) - def _get( - self, url: str, headers: Mapping[str, str] | None = None, **kwargs: Any + def _basic_auth_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, + ) -> Response: + return self.mock_http_client.do_request(method, url, headers=headers, **kwargs) + + def _oauth_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, + ) -> Response: + return self.mock_http_client.do_request(method, url, headers=headers, **kwargs) + + def _no_auth_request( + self, + method: str, + url: str, + headers: Mapping[str, str] | None = None, + **kwargs: Any, ) -> Response: - return self.mock_http_client.do_get(url, headers=headers, **kwargs) + return self.mock_http_client.do_request(method, url, headers=headers, **kwargs)