Skip to content

Commit

Permalink
Add caching
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathangreen
jonathangreen committed Jun 27, 2024
1 parent 258d2bb commit 98a22ab
Show file tree
Hide file tree
Showing 13 changed files with 826 additions and 9 deletions.
30 changes: 21 additions & 9 deletions src/palace/manager/core/opds_schema.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
import re
from collections.abc import Generator
from importlib.abc import Traversable
from typing import Any
from pathlib import Path
from typing import Any, cast
from urllib.parse import urlparse

import requests
Expand All @@ -25,19 +26,30 @@ def opds2_schema_resources() -> Traversable:
@to_cached_resource(loads=json.loads)
def opds2_cached_retrieve(uri: str) -> str:
"""
Fetch file from local filesystem if it has a file:// url, else fetch remotely.
Fetch files from the resources directory or cache them.
We use the to_cached_resource decorator, which caches the results of this function,
so each uri should only get fetched once.
If the uri is a file:// uri, fetch the file from the resources directory. Otherwise,
fetch the file from the local cache in the 'cached' directory falling back to downloading
the file if it is not found and adding it to the cache.
To refresh the cache, delete the 'cached' directory. This will force the function to
re-download the files.
"""
parsed = urlparse(uri)
resources = opds2_schema_resources()
if parsed.scheme == "file":
filename = "/".join([parsed.netloc, parsed.path])
package_file = opds2_schema_resources() / filename
with package_file.open("r") as fp:
return fp.read()
filename = f"{parsed.netloc}{parsed.path}"
package_file = resources / filename
else:
return requests.get(uri).text
netloc_dir = parsed.netloc
filename = parsed.path.removeprefix("/").replace("/", "_")
package_file = resources / "cached" / netloc_dir / filename
if not package_file.is_file():
cached_dir = cast(Path, resources / "cached" / netloc_dir)
cached_dir.mkdir(parents=True, exist_ok=True)
(cached_dir / filename).write_text(requests.get(uri).text)

Check warning on line 50 in src/palace/manager/core/opds_schema.py

View check run for this annotation

Codecov / codecov/patch

src/palace/manager/core/opds_schema.py#L48-L50 

Added lines #L48 - L50 were not covered by tests

return package_file.read_text()


def opds2_pattern_validator(
Expand Down
52 changes: 52 additions & 0 deletions ...ace/manager/resources/opds2_schema/cached/drafts.opds.io/schema_feed-metadata.schema.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://drafts.opds.io/schema/feed-metadata.schema.json",
"title": "OPDS Metadata",
"type": "object",
"properties": {
"identifier": {
"type": "string",
"format": "uri"
},
"@type": {
"type": "string",
"format": "uri"
},
"title": {
"type": [
"string",
"array",
"object"
]
},
"subtitle": {
"type": [
"string",
"array",
"object"
]
},
"modified": {
"type": "string",
"format": "date-time"
},
"description": {
"type": "string"
},
"itemsPerPage": {
"type": "integer",
"exclusiveMinimum": 0
},
"currentPage": {
"type": "integer",
"exclusiveMinimum": 0
},
"numberOfItems": {
"type": "integer",
"minimum": 0
}
},
"required": [
"title"
]
}
155 changes: 155 additions & 0 deletions src/palace/manager/resources/opds2_schema/cached/drafts.opds.io/schema_feed.schema.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://drafts.opds.io/schema/feed.schema.json",
"title": "OPDS Feed",
"type": "object",
"properties": {
"metadata": {
"description": "Contains feed-level metadata such as title or number of items",
"$ref": "feed-metadata.schema.json"
},
"links": {
"description": "Feed-level links such as search or pagination",
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"uniqueItems": true,
"minItems": 1,
"contains": {
"properties": {
"rel": {
"anyOf": [
{
"type": "string",
"const": "self"
},
{
"type": "array",
"contains": {
"const": "self"
}
}
]
}
},
"required": [
"rel"
]
}
},
"publications": {
"description": "A list of publications that can be acquired",
"type": "array",
"items": {
"$ref": "publication.schema.json"
},
"uniqueItems": true,
"minItems": 1
},
"navigation": {
"description": "Navigation for the catalog using links",
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"uniqueItems": true,
"minItems": 1,
"allOf": [
{
"description": "Each Link Object in a navigation collection must contain a title",
"items": {
"required": [
"title"
]
}
}
]
},
"facets": {
"description": "Facets are meant to re-order or obtain a subset for the current list of publications",
"type": "array",
"items": {
"type": "object",
"properties": {
"metadata": {
"$ref": "feed-metadata.schema.json"
},
"links": {
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"uniqueItems": true,
"minItems": 1
}
}
},
"uniqueItems": true,
"minItems": 1
},
"groups": {
"description": "Groups provide a curated experience, grouping publications or navigation links together",
"type": "array",
"items": {
"type": "object",
"properties": {
"metadata": {
"$ref": "feed-metadata.schema.json"
},
"links": {
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"uniqueItems": true,
"minItems": 1
},
"publications": {
"type": "array",
"items": {
"$ref": "publication.schema.json"
},
"uniqueItems": true,
"minItems": 1
},
"navigation": {
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"uniqueItems": true,
"minItems": 1
}
},
"required": [
"metadata"
]
}
}
},
"required": [
"metadata",
"links"
],
"additionalProperties": {
"$ref": "https://readium.org/webpub-manifest/schema/subcollection.schema.json"
},
"anyOf": [
{
"required": [
"publications"
]
},
{
"required": [
"navigation"
]
},
{
"required": [
"groups"
]
}
]
}
82 changes: 82 additions & 0 deletions ...alace/manager/resources/opds2_schema/cached/drafts.opds.io/schema_publication.schema.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://drafts.opds.io/schema/publication.schema.json",
"title": "OPDS Publication",
"type": "object",
"properties": {
"metadata": {
"$ref": "https://readium.org/webpub-manifest/schema/metadata.schema.json"
},
"links": {
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"contains": {
"description": "A publication must contain at least one acquisition link.",
"properties": {
"rel": {
"anyOf": [
{
"type": "string",
"enum": [
"preview",
"http://opds-spec.org/acquisition",
"http://opds-spec.org/acquisition/buy",
"http://opds-spec.org/acquisition/open-access",
"http://opds-spec.org/acquisition/borrow",
"http://opds-spec.org/acquisition/sample",
"http://opds-spec.org/acquisition/subscribe"
]
},
{
"type": "array",
"contains": {
"type": "string",
"enum": [
"preview",
"http://opds-spec.org/acquisition",
"http://opds-spec.org/acquisition/buy",
"http://opds-spec.org/acquisition/open-access",
"http://opds-spec.org/acquisition/borrow",
"http://opds-spec.org/acquisition/sample",
"http://opds-spec.org/acquisition/subscribe"
]
}
}
]
}
}
}
},
"images": {
"description": "Images are meant to be displayed to the user when browsing publications",
"type": "array",
"items": {
"$ref": "https://readium.org/webpub-manifest/schema/link.schema.json"
},
"minItems": 1,
"allOf": [
{
"description": "At least one image resource must use one of the following formats: image/jpeg, image/avif, image/png or image/gif.",
"contains": {
"properties": {
"type": {
"enum": [
"image/jpeg",
"image/avif",
"image/png",
"image/gif"
]
}
}
}
}
]
}
},
"required": [
"metadata",
"links"
]
}
54 changes: 54 additions & 0 deletions ...ces/opds2_schema/cached/readium.org/webpub-manifest_schema_contributor-object.schema.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://readium.org/webpub-manifest/schema/contributor-object.schema.json",
"title": "Contributor Object",
"type": "object",
"properties": {
"name": {
"anyOf": [
{
"type": "string"
},
{
"description": "The language in a language map must be a valid BCP 47 tag.",
"type": "object",
"patternProperties": {
"^((?<grandfathered>(en-GB-oed|i-ami|i-bnn|i-default|i-enochian|i-hak|i-klingon|i-lux|i-mingo|i-navajo|i-pwn|i-tao|i-tay|i-tsu|sgn-BE-FR|sgn-BE-NL|sgn-CH-DE)|(art-lojban|cel-gaulish|no-bok|no-nyn|zh-guoyu|zh-hakka|zh-min|zh-min-nan|zh-xiang))|((?<language>([A-Za-z]{2,3}(-(?<extlang>[A-Za-z]{3}(-[A-Za-z]{3}){0,2}))?)|[A-Za-z]{4}|[A-Za-z]{5,8})(-(?<script>[A-Za-z]{4}))?(-(?<region>[A-Za-z]{2}|[0-9]{3}))?(-(?<variant>[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*(-(?<extension>[0-9A-WY-Za-wy-z](-[A-Za-z0-9]{2,8})+))*(-(?<privateUse>x(-[A-Za-z0-9]{1,8})+))?)|(?<privateUse2>x(-[A-Za-z0-9]{1,8})+))$": {
"type": "string"
}
},
"additionalProperties": false,
"minProperties": 1
}
]
},
"identifier": {
"type": "string",
"format": "uri"
},
"sortAs": {
"type": "string"
},
"role": {
"type": [
"string",
"array"
],
"items": {
"type": "string"
}
},
"position": {
"type": "number"
},
"links": {
"type": "array",
"items": {
"$ref": "link.schema.json"
}
}
},
"required": [
"name"
]
}
Loading

0 comments on commit 98a22ab

Please sign in to comment.