From 410d33f332389a78ec1c58667cd77e6ab4ba5837 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 25 Oct 2024 09:08:51 +0000 Subject: [PATCH 1/4] Warn about problem with dev server and question marks --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96c34db..eb5ebaf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,8 @@ yarn dev Do not forget to cleanup `public/content` folder before building the docker image again, otherwise all assets will be pushed to the ZIM. +Note that some assets (e.g. icomoon fonts on LibreTexts Geoscience) having a question mark in their URL are not properly working in the yarn dev server. This is OK inside the ZIM. See https://github.com/openzim/mindtouch/issues/34. + ``` rm -rf zimui/public/content ``` From 5c3a94a66d761d582b3cc566ed51ad1b0db0ccc5 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 25 Oct 2024 09:11:51 +0000 Subject: [PATCH 2/4] Update page title when navigating --- zimui/src/views/HomeView.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zimui/src/views/HomeView.vue b/zimui/src/views/HomeView.vue index 358864c..bb317de 100644 --- a/zimui/src/views/HomeView.vue +++ b/zimui/src/views/HomeView.vue @@ -37,6 +37,9 @@ watch( () => (main.shared ? route.params.pathMatch : undefined), () => { page.value = getPage() + if (page.value) { + document.title = page.value.title + } } ) From 741bc1be61b71dc4824eb9e6c062e95317f9df04 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 25 Oct 2024 09:21:52 +0000 Subject: [PATCH 3/4] Add navigation back to home when clicking on logo --- zimui/src/components/HeaderBar.vue | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/zimui/src/components/HeaderBar.vue b/zimui/src/components/HeaderBar.vue index c95c482..0ebc46d 100644 --- a/zimui/src/components/HeaderBar.vue +++ b/zimui/src/components/HeaderBar.vue @@ -18,14 +18,16 @@ onMounted(async () => { From deebfa392028e364ba3bd0fdc5bfc5f934d4c0cc Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 25 Oct 2024 11:51:20 +0000 Subject: [PATCH 4/4] Retrieve illustration from online source and resize for ZIM + favicon --- scraper/src/mindtouch2zim/client.py | 13 + scraper/src/mindtouch2zim/entrypoint.py | 7 + scraper/src/mindtouch2zim/processor.py | 107 +++-- .../src/mindtouch2zim/third_party/README.md | 1 - .../third_party/libretexts_newsite/LICENSE | 386 ------------------ .../third_party/libretexts_newsite/README.md | 4 - .../libretexts_newsite/header_logo_mini.png | Bin 6648 -> 0 bytes scraper/tests-integration/conftest.py | 8 + scraper/tests-integration/test_client.py | 5 + scraper/tests-integration/test_zim_content.py | 1 + zimui/public/favicon.ico | Bin 4286 -> 0 bytes 11 files changed, 111 insertions(+), 421 deletions(-) delete mode 100644 scraper/src/mindtouch2zim/third_party/README.md delete mode 100644 scraper/src/mindtouch2zim/third_party/libretexts_newsite/LICENSE delete mode 100644 scraper/src/mindtouch2zim/third_party/libretexts_newsite/README.md delete mode 100644 scraper/src/mindtouch2zim/third_party/libretexts_newsite/header_logo_mini.png delete mode 100644 zimui/public/favicon.ico diff --git a/scraper/src/mindtouch2zim/client.py b/scraper/src/mindtouch2zim/client.py index 8c2ccfc..17d3a53 100644 --- a/scraper/src/mindtouch2zim/client.py +++ b/scraper/src/mindtouch2zim/client.py @@ -24,6 +24,7 @@ class MindtouchHome(BaseModel): screen_css_url: str print_css_url: str inline_css: list[str] + icons_urls: list[str] LibraryPageId = str @@ -181,6 +182,7 @@ def get_home(self) -> MindtouchHome: print_css_url=_get_print_css_url_from_home(soup), inline_css=_get_inline_css_from_home(soup), home_url=f"{self.library_url}/", + icons_urls=_get_icons_urls(soup), ) def get_deki_token(self) -> str: @@ -381,3 +383,14 @@ def _get_inline_css_from_home(soup: BeautifulSoup) -> list[str]: """Returns inline CSS code found on home page""" links = soup.find_all("style", {"type": "text/css"}) return [link.text for link in links if link.text] + + +def _get_icons_urls(soup: BeautifulSoup) -> list[str]: + """Returns list of potential icons""" + # prefer apple-touch-icon since they are usually bigger than the classic 32x32 + # favicon which is ugly once upscaled to 48x48 which is what we need for the ZIM + # illustration + links = soup.find_all("link", {"rel": "apple-touch-icon"}) + soup.find_all( + "link", {"rel": "icon"} + ) + return [link.get("href", None) for link in links if link.get("href", None)] diff --git a/scraper/src/mindtouch2zim/entrypoint.py b/scraper/src/mindtouch2zim/entrypoint.py index 8d8972b..208d203 100644 --- a/scraper/src/mindtouch2zim/entrypoint.py +++ b/scraper/src/mindtouch2zim/entrypoint.py @@ -212,6 +212,12 @@ def main(tmpdir: str) -> None: dest="stats_filename", ) + parser.add_argument( + "--illustration-url", + help="URL to illustration to use for ZIM illustration and favicon", + dest="illustration_url", + ) + args = parser.parse_args() logger.setLevel(level=logging.DEBUG if args.debug else logging.INFO) @@ -246,6 +252,7 @@ def main(tmpdir: str) -> None: content_filter=doc_filter, stats_file=Path(args.stats_filename) if args.stats_filename else None, overwrite_existing_zim=args.overwrite, + illustration_url=args.illustration_url, ).run() except SystemExit: logger.error("Generation failed, exiting") diff --git a/scraper/src/mindtouch2zim/processor.py b/scraper/src/mindtouch2zim/processor.py index a01affc..1502de6 100644 --- a/scraper/src/mindtouch2zim/processor.py +++ b/scraper/src/mindtouch2zim/processor.py @@ -11,7 +11,9 @@ from zimscraperlib.download import ( stream_file, # pyright: ignore[reportUnknownVariableType] ) -from zimscraperlib.image import resize_image +from zimscraperlib.image import convert_image, resize_image +from zimscraperlib.image.conversion import convert_svg2png +from zimscraperlib.image.probing import format_for from zimscraperlib.rewriting.css import CssRewriter from zimscraperlib.rewriting.html import HtmlRewriter from zimscraperlib.rewriting.html import rules as html_rules @@ -30,8 +32,9 @@ LibraryPageId, LibraryTree, MindtouchClient, + MindtouchHome, ) -from mindtouch2zim.constants import LANGUAGE_ISO_639_3, NAME, ROOT_DIR, VERSION, logger +from mindtouch2zim.constants import LANGUAGE_ISO_639_3, NAME, VERSION, logger from mindtouch2zim.ui import ( ConfigModel, PageContentModel, @@ -59,6 +62,12 @@ class UnsupportedTagError(Exception): pass +class NoIllustrationFoundError(Exception): + """An exception raised when no suitable illustration has been found""" + + pass + + class ContentFilter(BaseModel): """Supports filtering documents by user provided attributes.""" @@ -174,6 +183,7 @@ def __init__( output_folder: Path, zimui_dist: Path, stats_file: Path | None, + illustration_url: str | None, *, overwrite_existing_zim: bool, ) -> None: @@ -195,6 +205,7 @@ def __init__( self.zimui_dist = zimui_dist self.stats_file = stats_file self.overwrite_existing_zim = overwrite_existing_zim + self.illustration_url = illustration_url self.stats_items_done = 0 # we add 1 more items to process so that progress is not 100% at the beginning @@ -203,21 +214,6 @@ def __init__( # could happen in the loop in terms of exit conditions self.stats_items_total = 1 - self.zim_illustration_path = self.libretexts_newsite_path( - "header_logo_mini.png" - ) - - @staticmethod - def libretexts_newsite_path(name: str) -> Path: - """Returns the path to name in the third_party/libretexts_newsite folder. - - Raises ValueError if the resource doesn't exist. - """ - path = ROOT_DIR.joinpath("third_party", "libretexts_newsite", name) - if not path.exists(): - raise ValueError(f"File not found at {path}") - return path - def run(self) -> Path: """Generates a zim for a single document. @@ -253,15 +249,11 @@ def run(self) -> Path: creator = Creator(zim_path, "index.html") - logger.debug("Resizing ZIM illustration") - zim_illustration = BytesIO() - resize_image( - src=self.zim_illustration_path, - dst=zim_illustration, - width=48, - height=48, - method="cover", - ) + logger.info(" Fetching and storing home page...") + home = self.mindtouch_client.get_home() + + logger.info(" Fetching ZIM illustration...") + zim_illustration = self._fetch_zim_illustration(home) logger.debug("Configuring metadata") creator.config_metadata( @@ -278,11 +270,19 @@ def run(self) -> Path: Scraper=f"{NAME} v{VERSION}", Illustration_48x48_at_1=zim_illustration.getvalue(), ) - del zim_illustration # Start creator early to detect problems early. with creator as creator: + add_item_for( + creator, + "favicon.ico", + content=self._fetch_favicon_from_illustration( + zim_illustration + ).getvalue(), + ) + del zim_illustration + logger.info(" Storing configuration...") add_item_for( creator, @@ -342,9 +342,6 @@ def run(self) -> Path: is_front=False, ) - logger.info(" Fetching and storing home page...") - home = self.mindtouch_client.get_home() - welcome_image = BytesIO() stream_file(home.welcome_image_url, byte_stream=welcome_image) add_item_for(creator, "content/logo.png", content=welcome_image.getvalue()) @@ -516,6 +513,56 @@ def _report_progress(self): } self.stats_file.write_text(json.dumps(progress, indent=2)) + def _fetch_zim_illustration(self, home: MindtouchHome) -> BytesIO: + """Fetch ZIM illustration, convert/resize and return it""" + for icon_url in ( + [self.illustration_url] if self.illustration_url else home.icons_urls + ): + try: + logger.debug(f"Downloading {icon_url} illustration") + illustration_content = BytesIO() + stream_file(icon_url, byte_stream=illustration_content) + illustration_format = format_for( + illustration_content, from_suffix=False + ) + png_illustration = BytesIO() + if illustration_format == "SVG": + logger.debug("Converting SVG illustration to PNG") + convert_svg2png(illustration_content, png_illustration, 48, 48) + elif illustration_format == "PNG": + png_illustration = illustration_content + else: + logger.debug( + f"Converting {illustration_format} illustration to PNG" + ) + convert_image(illustration_content, png_illustration, fmt="PNG") + logger.debug("Resizing ZIM illustration") + resize_image( + src=png_illustration, + width=48, + height=48, + method="cover", + ) + return png_illustration + except Exception as exc: + logger.warning( + f"Failed to retrieve illustration at {icon_url}", exc_info=exc + ) + raise NoIllustrationFoundError("Failed to find a suitable illustration") + + def _fetch_favicon_from_illustration(self, illustration: BytesIO) -> BytesIO: + """Return a converted version of the illustration into favicon""" + favicon = BytesIO() + convert_image(illustration, favicon, fmt="ICO") + logger.debug("Resizing ZIM illustration") + resize_image( + src=favicon, + width=32, + height=32, + method="cover", + ) + return favicon + # remove all standard rules, they are not adapted to Vue.JS UI html_rules.rewrite_attribute_rules.clear() diff --git a/scraper/src/mindtouch2zim/third_party/README.md b/scraper/src/mindtouch2zim/third_party/README.md deleted file mode 100644 index 6974785..0000000 --- a/scraper/src/mindtouch2zim/third_party/README.md +++ /dev/null @@ -1 +0,0 @@ -This folder contains resources fetched from third parties, with accompaying license information. diff --git a/scraper/src/mindtouch2zim/third_party/libretexts_newsite/LICENSE b/scraper/src/mindtouch2zim/third_party/libretexts_newsite/LICENSE deleted file mode 100644 index 7416b6f..0000000 --- a/scraper/src/mindtouch2zim/third_party/libretexts_newsite/LICENSE +++ /dev/null @@ -1,386 +0,0 @@ -Attribution 4.0 International - -======================================================================= - -Creative Commons Corporation ("Creative Commons") is not a law firm and -does not provide legal services or legal advice. Distribution of -Creative Commons public licenses does not create a lawyer-client or -other relationship. Creative Commons makes its licenses and related -information available on an "as-is" basis. Creative Commons gives no -warranties regarding its licenses, any material licensed under their -terms and conditions, or any related information. Creative Commons -disclaims all liability for damages resulting from their use to the -fullest extent possible. - -Using Creative Commons Public Licenses - -Creative Commons public licenses provide a standard set of terms and -conditions that creators and other rights holders may use to share -original works of authorship and other material subject to copyright -and certain other rights specified in the public license below. The -following considerations are for informational purposes only, are not -exhaustive, and do not form part of our licenses. - - Considerations for licensors: Our public licenses are - intended for use by those authorized to give the public - permission to use material in ways otherwise restricted by - copyright and certain other rights. Our licenses are - irrevocable. Licensors should read and understand the terms - and conditions of the license they choose before applying it. - Licensors should also secure all rights necessary before - applying our licenses so that the public can reuse the - material as expected. Licensors should clearly mark any - material not subject to the license. This includes other CC- - licensed material, or material used under an exception or - limitation to copyright. More considerations for licensors: - wiki.creativecommons.org/Considerations_for_licensors - - Considerations for the public: By using one of our public - licenses, a licensor grants the public permission to use the - licensed material under specified terms and conditions. If - the licensor's permission is not necessary for any reason--for - example, because of any applicable exception or limitation to - copyright--then that use is not regulated by the license. Our - licenses grant only permissions under copyright and certain - other rights that a licensor has authority to grant. Use of - the licensed material may still be restricted for other - reasons, including because others have copyright or other - rights in the material. A licensor may make special requests, - such as asking that all changes be marked or described. - Although not required by our licenses, you are encouraged to - respect those requests where reasonable. More considerations - for the public: - wiki.creativecommons.org/Considerations_for_licensees - -======================================================================= - -Creative Commons Attribution 4.0 International Public License - -By exercising the Licensed Rights (defined below), You accept and agree -to be bound by the terms and conditions of this Creative Commons -Attribution 4.0 International Public License ("Public License"). To the -extent this Public License may be interpreted as a contract, You are -granted the Licensed Rights in consideration of Your acceptance of -these terms and conditions, and the Licensor grants You such rights in -consideration of benefits the Licensor receives from making the -Licensed Material available under these terms and conditions. - -Section 1 -- Definitions. - -a. Adapted Material means material subject to Copyright and Similar -Rights that is derived from or based upon the Licensed Material -and in which the Licensed Material is translated, altered, -arranged, transformed, or otherwise modified in a manner requiring -permission under the Copyright and Similar Rights held by the -Licensor. For purposes of this Public License, where the Licensed -Material is a musical work, performance, or sound recording, -Adapted Material is always produced where the Licensed Material is -synched in timed relation with a moving image. - -b. Adapter's License means the license You apply to Your Copyright -and Similar Rights in Your contributions to Adapted Material in -accordance with the terms and conditions of this Public License. - -c. Copyright and Similar Rights means copyright and/or similar rights -closely related to copyright including, without limitation, -performance, broadcast, sound recording, and Sui Generis Database -Rights, without regard to how the rights are labeled or -categorized. For purposes of this Public License, the rights -specified in Section 2(b)(1)-(2) are not Copyright and Similar -Rights. - -d. Effective Technological Measures means those measures that, in the -absence of proper authority, may not be circumvented under laws -fulfilling obligations under Article 11 of the WIPO Copyright -Treaty adopted on December 20, 1996, and/or similar international -agreements. - -e. Exceptions and Limitations means fair use, fair dealing, and/or -any other exception or limitation to Copyright and Similar Rights -that applies to Your use of the Licensed Material. - -f. Licensed Material means the artistic or literary work, database, -or other material to which the Licensor applied this Public -License. - -g. Licensed Rights means the rights granted to You subject to the -terms and conditions of this Public License, which are limited to -all Copyright and Similar Rights that apply to Your use of the -Licensed Material and that the Licensor has authority to license. - -h. Licensor means the individual(s) or entity(ies) granting rights -under this Public License. - -i. Share means to provide material to the public by any means or -process that requires permission under the Licensed Rights, such -as reproduction, public display, public performance, distribution, -dissemination, communication, or importation, and to make material -available to the public including in ways that members of the -public may access the material from a place and at a time -individually chosen by them. - -j. Sui Generis Database Rights means rights other than copyright -resulting from Directive 96/9/EC of the European Parliament and of -the Council of 11 March 1996 on the legal protection of databases, -as amended and/or succeeded, as well as other essentially -equivalent rights anywhere in the world. - -k. You means the individual or entity exercising the Licensed Rights -under this Public License. Your has a corresponding meaning. - -Section 2 -- Scope. - -a. License grant. - - 1. Subject to the terms and conditions of this Public License, - the Licensor hereby grants You a worldwide, royalty-free, - non-sublicensable, non-exclusive, irrevocable license to - exercise the Licensed Rights in the Licensed Material to: - - a. reproduce and Share the Licensed Material, in whole or - in part; and - - b. produce, reproduce, and Share Adapted Material. - - 2. Exceptions and Limitations. For the avoidance of doubt, where - Exceptions and Limitations apply to Your use, this Public - License does not apply, and You do not need to comply with - its terms and conditions. - - 3. Term. The term of this Public License is specified in Section - 6(a). - - 4. Media and formats; technical modifications allowed. The - Licensor authorizes You to exercise the Licensed Rights in - all media and formats whether now known or hereafter created, - and to make technical modifications necessary to do so. The - Licensor waives and/or agrees not to assert any right or - authority to forbid You from making technical modifications - necessary to exercise the Licensed Rights, including - technical modifications necessary to circumvent Effective - Technological Measures. For purposes of this Public License, - simply making modifications authorized by this Section 2(a) - (4) never produces Adapted Material. - - 5. Downstream recipients. - - a. Offer from the Licensor -- Licensed Material. Every - recipient of the Licensed Material automatically - receives an offer from the Licensor to exercise the - Licensed Rights under the terms and conditions of this - Public License. - - b. No downstream restrictions. You may not offer or impose - any additional or different terms or conditions on, or - apply any Effective Technological Measures to, the - Licensed Material if doing so restricts exercise of the - Licensed Rights by any recipient of the Licensed - Material. - - 6. No endorsement. Nothing in this Public License constitutes or - may be construed as permission to assert or imply that You - are, or that Your use of the Licensed Material is, connected - with, or sponsored, endorsed, or granted official status by, - the Licensor or others designated to receive attribution as - provided in Section 3(a)(1)(A)(i). - -b. Other rights. - - 1. Moral rights, such as the right of integrity, are not - licensed under this Public License, nor are publicity, - privacy, and/or other similar personality rights; however, to - the extent possible, the Licensor waives and/or agrees not to - assert any such rights held by the Licensor to the limited - extent necessary to allow You to exercise the Licensed - Rights, but not otherwise. - - 2. Patent and trademark rights are not licensed under this - Public License. - - 3. To the extent possible, the Licensor waives any right to - collect royalties from You for the exercise of the Licensed - Rights, whether directly or through a collecting society - under any voluntary or waivable statutory or compulsory - licensing scheme. In all other cases the Licensor expressly - reserves any right to collect such royalties. - -Section 3 -- License Conditions. - -Your exercise of the Licensed Rights is expressly made subject to the -following conditions. - -a. Attribution. - - 1. If You Share the Licensed Material (including in modified - form), You must: - - a. retain the following if it is supplied by the Licensor - with the Licensed Material: - - i. identification of the creator(s) of the Licensed - Material and any others designated to receive - attribution, in any reasonable manner requested by - the Licensor (including by pseudonym if - designated); - - ii. a copyright notice; - - iii. a notice that refers to this Public License; - - iv. a notice that refers to the disclaimer of - warranties; - - v. a URI or hyperlink to the Licensed Material to the - extent reasonably practicable; - - b. indicate if You modified the Licensed Material and - retain an indication of any previous modifications; and - - c. indicate the Licensed Material is licensed under this - Public License, and include the text of, or the URI or - hyperlink to, this Public License. - - 2. You may satisfy the conditions in Section 3(a)(1) in any - reasonable manner based on the medium, means, and context in - which You Share the Licensed Material. For example, it may be - reasonable to satisfy the conditions by providing a URI or - hyperlink to a resource that includes the required - information. - - 3. If requested by the Licensor, You must remove any of the - information required by Section 3(a)(1)(A) to the extent - reasonably practicable. - - 4. If You Share Adapted Material You produce, the Adapter's - License You apply must not prevent recipients of the Adapted - Material from complying with this Public License. - -Section 4 -- Sui Generis Database Rights. - -Where the Licensed Rights include Sui Generis Database Rights that -apply to Your use of the Licensed Material: - -a. for the avoidance of doubt, Section 2(a)(1) grants You the right -to extract, reuse, reproduce, and Share all or a substantial -portion of the contents of the database; - -b. if You include all or a substantial portion of the database -contents in a database in which You have Sui Generis Database -Rights, then the database in which You have Sui Generis Database -Rights (but not its individual contents) is Adapted Material; and - -c. You must comply with the conditions in Section 3(a) if You Share -all or a substantial portion of the contents of the database. - -For the avoidance of doubt, this Section 4 supplements and does not -replace Your obligations under this Public License where the Licensed -Rights include other Copyright and Similar Rights. - -Section 5 -- Disclaimer of Warranties and Limitation of Liability. - -a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE -EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS -AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF -ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, -IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, -WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, -ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT -KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT -ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. - -b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE -TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, -NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, -INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, -COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR -USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN -ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR -DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR -IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. - -c. The disclaimer of warranties and limitation of liability provided -above shall be interpreted in a manner that, to the extent -possible, most closely approximates an absolute disclaimer and -waiver of all liability. - -Section 6 -- Term and Termination. - -a. This Public License applies for the term of the Copyright and -Similar Rights licensed here. However, if You fail to comply with -this Public License, then Your rights under this Public License -terminate automatically. - -b. Where Your right to use the Licensed Material has terminated under -Section 6(a), it reinstates: - - 1. automatically as of the date the violation is cured, provided - it is cured within 30 days of Your discovery of the - violation; or - - 2. upon express reinstatement by the Licensor. - - For the avoidance of doubt, this Section 6(b) does not affect any - right the Licensor may have to seek remedies for Your violations - of this Public License. - -c. For the avoidance of doubt, the Licensor may also offer the -Licensed Material under separate terms or conditions or stop -distributing the Licensed Material at any time; however, doing so -will not terminate this Public License. - -d. Sections 1, 5, 6, 7, and 8 survive termination of this Public -License. - -Section 7 -- Other Terms and Conditions. - -a. The Licensor shall not be bound by any additional or different -terms or conditions communicated by You unless expressly agreed. - -b. Any arrangements, understandings, or agreements regarding the -Licensed Material not stated herein are separate from and -independent of the terms and conditions of this Public License. - -Section 8 -- Interpretation. - -a. For the avoidance of doubt, this Public License does not, and -shall not be interpreted to, reduce, limit, restrict, or impose -conditions on any use of the Licensed Material that could lawfully -be made without permission under this Public License. - -b. To the extent possible, if any provision of this Public License is -deemed unenforceable, it shall be automatically reformed to the -minimum extent necessary to make it enforceable. If the provision -cannot be reformed, it shall be severed from this Public License -without affecting the enforceability of the remaining terms and -conditions. - -c. No term or condition of this Public License will be waived and no -failure to comply consented to unless expressly agreed to by the -Licensor. - -d. Nothing in this Public License constitutes or may be interpreted -as a limitation upon, or waiver of, any privileges and immunities -that apply to the Licensor or You, including from the legal -processes of any jurisdiction or authority. - -======================================================================= - -Creative Commons is not a party to its public licenses. -Notwithstanding, Creative Commons may elect to apply one of its public -licenses to material it publishes and in those instances will be -considered the “Licensor.” The text of the Creative Commons public -licenses is dedicated to the public domain under the CC0 Public Domain -Dedication. Except for the limited purpose of indicating that material -is shared under a Creative Commons public license or as otherwise -permitted by the Creative Commons policies published at -creativecommons.org/policies, Creative Commons does not authorize the -use of the trademark "Creative Commons" or any other trademark or logo -of Creative Commons without its prior written consent including, -without limitation, in connection with any unauthorized modifications -to any of its public licenses or any other arrangements, -understandings, or agreements concerning use of licensed material. For -the avoidance of doubt, this paragraph does not form part of the public -licenses. - -Creative Commons may be contacted at creativecommons.org. diff --git a/scraper/src/mindtouch2zim/third_party/libretexts_newsite/README.md b/scraper/src/mindtouch2zim/third_party/libretexts_newsite/README.md deleted file mode 100644 index 655139f..0000000 --- a/scraper/src/mindtouch2zim/third_party/libretexts_newsite/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Resources from this `libretexts_newsite` subfolder have been fetched from https://github.com/LibreTexts/NewSite -where they are listed as licensed under "Creative Commons Attribution 4.0 International", see [LICENSE](LICENSE) file. - -- header_logo_mini.png has been retrieved from https://github.com/LibreTexts/NewSite/blob/master/img/LibreTexts/header_logo_mini.png or more precisely https://github.com/LibreTexts/NewSite/blob/08ae96a370f570a59851be7e8ec3dfed78653933/img/LibreTexts/header_logo_mini.png diff --git a/scraper/src/mindtouch2zim/third_party/libretexts_newsite/header_logo_mini.png b/scraper/src/mindtouch2zim/third_party/libretexts_newsite/header_logo_mini.png deleted file mode 100644 index e985e38fa31345cf8655744fb51aa3c7ba9d5514..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6648 zcmVPx#1ZP1_K>z@;j|==^1poj532;bRa{vGqB>(^xB>_oNB=7(L02y>eSaefwW^{L9 za%BK;VQFr3E^cLXAT%y8E;2FkAZe8V02ydWL_t(|UgceBbR5-{t~n?3BQpu!cVK`J zCV>-THiK?B=jPZ)Wn+0Lx)!o&S&*Yx-`BwHXw7w~D5V^!BIe^u1wU zI*)%~R-|^aw8_}Rw}E{1+F=E|@jAf2yuMTW`^Rb2cIBEui*-BSc9n*2P1PTQ+437N z(Xd06^wW=u=qH*=|D z2B8P5?$f1R?Rdm@1Ejcx<)Olx!|k)+^uA%vgj4K{OzU@4Y08y3b6au9IR|bf1-^?}L4D%xxPB>8>rO#s9_65xMl@ zt`aI0@m;l%U*yvbZy%z8bK;CGL>cAhh-;_qU^lh{o>>vE71gyGx9c$QzwJ~L_1`4I z0tl3Z&q`*==FnqyGWA}XPs+6!6Oc$I6VYL=jP^|9=uc zU`~{lTcc$#pagw9{`R7JJtFVk(~qIA-q zi%6mGln(dv=|amh?Y154#xS7wm;>5$k?5GM=E7AyCilg_C(wI&4)vUuM%@H)EU6ao zUn|ooqq4=gg1dp_=hmj{r2lAq6xB4l!cR>24|fPHMi3HF|BgEnlJ?dU%7d*HuadBLf!rm z$C7CU_b*DJs6ziGflF7vrAL;h>Bn@Ro<7|E;~+=+Co^X1lCO3~&>otLY|P+Br0JPocj-fSE|EfalIVdPXPIicLOH>~?(j z8Hs{?kpn>Ei#8#nogPy|Ekeqw2C4hT$LYk~Z%AM_27zZqnqOf`^U>%{S&ekphJ3fM z0?nNc4_mz(|(e^a=0nwi|2-Wn{xqcqBke`V8kg&47PY7uX ziEFl`l13UZCz|CVAg~)P;AcYb#xQCA8Ik6{dZ5fLj`#d9mAXw0ULJNA4{Pk6B04A1 z`9SUEPjcCFa)5+|l$Eg`68Eh`mAd3L+tj_e)Kf@gjvGs0H~PUVn^QD%xfW^u+=M#1 zb&ZVUr}O~~v6Q<>tN7)LG)gJwHMu|Ada{N+r$0c_LfY!yFJeC=FQl)m@koP2eqn7A z%aLDb!*^T8?^Yyur1@ydu7YOjzd86cpTo^F;%QZErAHmdsukC_(M{8X?`42qeo*2+ zBrv3~*|I8|>G7p;EN5Q9ZZzS652Ce15!=j`UVD+A+M%ZTLLPn~()=kwr1=W|_tRXe zYWA(keHkV;ezPirJ$Imh)P>|l$kgvs^ycSxm+I2kZ1Kg7)PGhK%b^F@jRwB=<{oVn zBM%YtMnzUpzm*Otrl&?44}p6YoTP(!o)0)sn{~L9J%1>G^o0b56vjP68c#e{ zP0glS(6`@xN5ddL89v>>ZWKV*F^82aqT8=ZYoI&UIZCxauO&$HLz9QS{}D?w4wh1z zNka!?@ukhwBg7u(3ugHVsSL?%4vCo+%`|x4ah6*h+{TH!PZ@{1EV`_f27PRAnh*SL zT_)w%S=Z#g49V2J3q`17u45~N%7QdjAHc7-6;V>v1*0V{du>Y^@BP{vd)%-ywaPfe zVKuE+Y1r50jtmckLmq~h<^w;Tok-hKYu!q+q1yb&GWJ^G0wgk|vL62hKm9}q8Y@C@ z^JVv*JL^C`nR#~m;Xrn+v8%(r5hM7WBCP1TR>U?t- z;gp{d!u!Po7_cdyQmWm(f2Nsy`JYL9+-v3YVr!gF5+}7ItN^2S7uo9N(*7raL@8WA z_yJc?cQ=YKv@6fOcsFEKqC4>e<{sCK-|03jszPLNjxH2xF-yg-SEYL-A$Hgo8FlRS zIs&@)FUoNA01+W}=ekr{b@L_aJW6D{lK5Dqg!^@V6ZH@yH?%7dswS5}H8Ho=bMUdl zwx`wVg)@{ExNEmJAgi(Z0TTO`g=ti*X#NR8xah?HaO77`Z-*kff4T5*ArIm4LCtr1 z!SWLo##O>++10J|($-U6E0OI9kPqT9Ccd!M6~?R1wK<4?dS#-M_`@Q3=`!I2brYe* zuJ8gpvmwjpbe!FW+Rh&~WwUK*PeAiS`E;9fLU_HaWZr*vw5EM{sL=S4XzEwR)MebU zkSd90+%_wLD$aSP*X?03#pmdWwV77Z#GESz1j6NXsH;xn6`!sI`M+jCn8dfmJl5;Y#^;(S$1*TloMj7 zXzDg4^h5-ydkeFeC|m}ax^I?=T(>QFk~oK#Ien`cjA=oNyCF7HLt z%@dCsS0*}l@hZ6`PMt;_)prV2Mt*R_wcb3OmRxpT_w27uIi?^HVqV{#uggh1gehN_ z_?<39VGWC%?wAvLDTIaK(n0(&f9_n9b+0ZWdRyX~hMddTd`+Y zzlBif8CV3>*M^E`Sc%6xX}AnaJIvP`C}bf@1gh1xT&DN-o~E9LIQZk{9Lh85LrA73 ziUd)HBRKpg!cpGXQ9!XJ&H6UZF!wt~T;Du!2ZFE>f7gde)F=iFJMCood414>=Yb@| zX}K^7#DZXRO@DXRSv{1fZNt7h(%lz9p%*;5!m*B00=>dlm@I8Sct~} zcdW?Td!$wj9wwCu&JQBxG7k)Z@KWi;cp%W^f7+6xZ^zGgP}1uWnq*jl!8!1MgqaVI zmbt}oTn~eF@d65wMdT4UW=a*QebCWujuLr zhKDgpgc*d3yXPl6#{(o-2qOqdkfzeG(&4kzTAyS|qY z1`mW+4qlc@6ZRMD11204q2*2yS`c0~CRRJWwyx!hz8V2PV!Ntr;DrbWW_ymQR6^~T zi+7Nrkfm~;`xc#~{W+e?6jj56zGbFw1!Oz%a-a&c0U$qp83t|hF)C~&9;zZ(!ytxG z06S99NDnSaah?}mJM8m(k8b|LM>%fS2Obu^1cRPxc9x4o zU72TfVr2MHy%vM+B9UpDy%xvADa`FPmocD-%-)TMgh3kDMuO!n$x;pbKF>IX3R{Wi zO0WKmi?#h1Bzr7qLV@j9D_sMu*=u)ZoYg}Q3=a-r1Fz&DnNmXw=e0<#DvfoS=xypl zflM*MHrWEe*TXH7y+Bi2f;ZdAK?QJ0zlCU z%e66i0q>NSBjUImBMzKuaC=A~_T62DPQq~>JX}tBzyVI2_I#Vgm#I$D?!-g)at1P% z9j~Bo@)|6~M;K}<=4=y0V!95deoIEJyQsinO#u4s@sXn<6bh%7se3^{LYB?4IFFfCDs#=i%vB)M>D?EFV-B&`b z;rKJzKD>X0W1+aDwL9@x5yqLh1Pl19fk;ERfgTvq7q%{-fSB22B*ZpiaCg8?@LD`bD~8^HUZLKDM28aJOAx8XUTKJ#-2M`Q5U?k^UMJgp4rH~ep{QnF6OZf2 z>=)OvN;t|9GCNk!r$Qwjz0gXz*f!z;b1fWpTqp!?Uo)(Bd3Y|0aCp$>W6DH0g2%!3 zi9ysXE8(#q#)`=vV`bJ!B_4*bre@w8Y$YCXm$TubVch;4P~Ec<|50U>DoNGji1D%q zN%EfEm~A215y})k(BL z>U<8i5})qn(^H(G;hzgB$f*=>2qekzW#~cZNmfH2AhTkO8@>$V4yvkLIYNbP#Q#WS znf|(nk!YTspcn($`D#jG75D27nfO|$JZZ==#0&odVJC8bI?Ct1bnyNT% z8h^~P#{(FEyd_e2p%$yVxZ^nZ3kP^GsEGPgF8;z;I7$cDe*0`MZ;}|q-!$I!#{tj= zt=^cehut+1L6>WRKn^oH_h{LzYkCP=bAHXMsodiwdQ$fjgX~1AMvL zYwiHz1*f+PZlyH{&b5L%*G=9xE0#KsKB~Wo%$~#xaMM^i=gr`)xPcoj>TD~a0xKG@ z(hRG5y?wG=8V3Dvl@72x#H+UpfJ?;pduPTPM_@Ak0%?&2t}k~yz5FC~diRL_UM6=E zUqIKLBVxNR??5dZ%k;S8IRsiNFk3iE2h#B1v3|#2mJBYz>5zyN<4AlnqMo#ck)_6u zME}^8Po3U5?1=NsNvr@b?#eYW-po9a?GOIqzuAH*9az1F6|JuVT;3o!owmode0<)K z64L&-G2JOC_h;1Ve@URz$Rqm4?YR?Kg}-k{!i(X}R&M~XBLOnQYOjKTQ(p(N0zJ$K zZwIFbvt9L|A?wmKt>2VrC-G?bt>Yr~9YgVY2@rsKwXaeEyc6@ZR%IO*ek@&UZuo9Jgj zDBiroe%$dKq~eGy44f3JlPp}3%qsK{WAONKylb&-@|k7==Q{CEuSs=!J(8k7txk23 zFP>T;^3ogNsaPfE>+L}F3?w7y{@gi21>(4sX+=3k1oh!iT|PQI9e8DhQJN|NvL@-K+19Y^%?@dB61UfkSg4H;VbvURI1u{mcz;|g>&cLA> zVwOgq(Y@V`_y-pzXvOueB@*!bS^EpA%iD*XC+a*}c>jdZy&u2?xWA9Xp^g&D2Y#%^ zaXCf=GH|-VUh1EEY(1VrD+&QcOT&UdKGwlS$7P*Hra0+f!uC;#`PeH{) zVh1(szz}(Z)%*mvi|f+U zVpY=@PH#&kHI00260tmq;l#+|IlR}>VVreqVsRX;x zjORCHxDC88e4u_yT!o$-sEyIT@WeDaP$2xsFE52+=mA@jLl`&(@Sz{|9+70OJ67h= zyDCLQcyV(k%b^+UMsIlMi#(5uwO+bLlfTTT?n=Dt^zLCj268tMC@=x;Z<&ZUL#z*^ zCS-=1@&jFkS{c3llwQ^~+xWdAd`*aAdE>xt^o0i(iov(U^WCP?braX&t8peT;zO-tRhrm-ylPOn7LbD+hBP4TN`GWR?_acMBldID-t`2@U=PV7F86NqG zS8Wi_y%~yfp(b8tdDvgb!@cRAU!HgY{}Bpf)z~4&zr`&kN$Sa zPs~!`l}@}WiC2Ok4MY8m722k@71DYBH>dSwiK;bw& ze@_Nh-y%}~PjjgA2v_!tlCxd%6A1MWvB()BXZwUyer;2xE}hLb_izc_J~M{pAW*Oy zgT^%nyc4Me` zZbOEaZ%Wru6x)$pO?OUq@6vX~|3CoAtF{4>7}6LLSv5%L=Qm}t_ZTX)AHE+IHVoUI z>rsc%U!~b!sHfQS#*up4B|m*ZPxFP;mDhy?h7^V*Rt;m+D6yTpC%eFIv=a|4O48yP z`B~oBMt|Lq;UodjS2)-pKRuwQ`R?)%(ialgY^bdaT%63_i=ALMh69hUOx4mVJZGVh zazFe*oEs^2&3pl7c?hWs$!j)9uXWJNVup8S8h^6M*KMbk)A0kqmxS&suxE=ks< ztl1!W2QLYK9QOt7hH=pyjv?zaw455x^)g)f0l+|sUI3T}P2m0+F>IQjF9y@08#CE9 zv^Us|9fiM&RJ_sjyBkeI>E`z$Y0#WFnz%EMVhS7Cl?(I4fcl@awm5;-bjGBb6rnhH(rnEBqZodTe7uP zF%^_v(W2{be6b^|szv{+Vt*FD>n!%+j=la6t^FUSexc1c#ACq#0000 list[str]: "revision by students, faculty, and outside experts to supplant conventional " "paper-based books." ] + + +@pytest.fixture(scope="module") +def home_icons_urls() -> list[str]: + return [ + "https://a.mtstatic.com/@public/production/site_4038/1486479235-apple-touch-icon.png", + "https://a.mtstatic.com/@public/production/site_4038/1486479325-favicon.ico", + ] diff --git a/scraper/tests-integration/test_client.py b/scraper/tests-integration/test_client.py index cf9a20b..c5b9cdb 100644 --- a/scraper/tests-integration/test_client.py +++ b/scraper/tests-integration/test_client.py @@ -161,3 +161,8 @@ def test_get_home_inline_css(home: MindtouchHome): def test_get_home_url(home: MindtouchHome, libretexts_url: str): assert home.home_url == f"{libretexts_url}/" + + +def test_get_home_icons_urls(home: MindtouchHome, home_icons_urls: list[str]): + """Ensures proper icons urls are retrieved from home of libretexts""" + assert home.icons_urls == home_icons_urls diff --git a/scraper/tests-integration/test_zim_content.py b/scraper/tests-integration/test_zim_content.py index ecd2cf7..df126a9 100644 --- a/scraper/tests-integration/test_zim_content.py +++ b/scraper/tests-integration/test_zim_content.py @@ -50,6 +50,7 @@ def test_zim_metadata(zim_fh: Archive): "item_path,expected_mimetype", [ pytest.param("content/logo.png", "image/png", id="logo"), + pytest.param("favicon.ico", "image/vnd.microsoft.icon", id="favicon"), pytest.param("content/screen.css", "text/css", id="screen.css"), pytest.param("content/print.css", "text/css", id="print.css"), pytest.param("content/inline.css", "text/css", id="inline.css"), diff --git a/zimui/public/favicon.ico b/zimui/public/favicon.ico deleted file mode 100644 index df36fcfb72584e00488330b560ebcf34a41c64c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4286 zcmds*O-Phc6o&64GDVCEQHxsW(p4>LW*W<827=Unuo8sGpRux(DN@jWP-e29Wl%wj zY84_aq9}^Am9-cWTD5GGEo#+5Fi2wX_P*bo+xO!)p*7B;iKlbFd(U~_d(U?#hLj56 zPhFkj-|A6~Qk#@g^#D^U0XT1cu=c-vu1+SElX9NR;kzAUV(q0|dl0|%h|dI$%VICy zJnu2^L*Te9JrJMGh%-P79CL0}dq92RGU6gI{v2~|)p}sG5x0U*z<8U;Ij*hB9z?ei z@g6Xq-pDoPl=MANPiR7%172VA%r)kevtV-_5H*QJKFmd;8yA$98zCxBZYXTNZ#QFk2(TX0;Y2dt&WitL#$96|gJY=3xX zpCoi|YNzgO3R`f@IiEeSmKrPSf#h#Qd<$%Ej^RIeeYfsxhPMOG`S`Pz8q``=511zm zAm)MX5AV^5xIWPyEu7u>qYs?pn$I4nL9J!=K=SGlKLXpE<5x+2cDTXq?brj?n6sp= zphe9;_JHf40^9~}9i08r{XM$7HB!`{Ys~TK0kx<}ZQng`UPvH*11|q7&l9?@FQz;8 zx!=3<4seY*%=OlbCbcae?5^V_}*K>Uo6ZWV8mTyE^B=DKy7-sdLYkR5Z?paTgK-zyIkKjIcpyO z{+uIt&YSa_$QnN_@t~L014dyK(fOOo+W*MIxbA6Ndgr=Y!f#Tokqv}n<7-9qfHkc3 z=>a|HWqcX8fzQCT=dqVbogRq!-S>H%yA{1w#2Pn;=e>JiEj7Hl;zdt-2f+j2%DeVD zsW0Ab)ZK@0cIW%W7z}H{&~yGhn~D;aiP4=;m-HCo`BEI+Kd6 z={Xwx{TKxD#iCLfl2vQGDitKtN>z|-AdCN|$jTFDg0m3O`WLD4_s#$S