Handle libretexts.org glossary rewriting

openzim · Nov 11, 2024 · 4a9c441 · 4a9c441
1 parent 6ddba30
commit 4a9c441
Show file tree

Hide file tree

Showing 3 changed files with 97 additions and 5 deletions.
diff --git a/scraper/src/mindtouch2zim/libretexts/glossary.py b/scraper/src/mindtouch2zim/libretexts/glossary.py
@@ -0,0 +1,62 @@
+from bs4 import BeautifulSoup
+
+
+class GlossaryRewriteError(Exception):
+    """Exception indicating a problem during glossary rewrite"""
+
+    pass
+
+
+def _get_formatted_glossary_row(row) -> str:
+    """Format one row as HTML"""
+    word = row.find("td", attrs={"data-th": "Word(s)"}).text
+    definition = row.find("td", attrs={"data-th": "Definition"}).text
+    return (
+        '<p class="glossaryElement">\n'
+        f'  <span class="glossaryTerm">{word}</span>\n'
+        "  |\n"
+        f'  <span class="glossaryDefinition">{definition}</span>\n'
+        "</p>\n"
+    )
+
+
+def rewrite_glossary(original_content: str) -> str:
+    """Statically rewrite the glossary of libretexts.org
+
+    Only word and description columns are supported.
+    """
+
+    soup = BeautifulSoup(
+        original_content,
+        "html.parser",  # prefer html.parser to not add <html><body> tags
+    )
+
+    glossary_table = None
+
+    for table in soup.find_all("table"):
+        if not table.caption:
+            continue
+        if table.caption and table.caption.text.strip() == "Example and Directions":
+            continue
+        if glossary_table:
+            raise GlossaryRewriteError("Too many glossary tables")
+        glossary_table = table
+
+    if not glossary_table:
+        raise GlossaryRewriteError("Glossary table not found")
+
+    tbody = glossary_table.find("tbody")
+    if not tbody:
+        raise GlossaryRewriteError("Glossary table body not found")
+
+    glossary_table.insert_after(
+        BeautifulSoup(
+            "".join([_get_formatted_glossary_row(row) for row in tbody.find_all("tr")]),
+            "html.parser",  # prefer html.parser to not add <html><body> tags
+        )
+    )
+
+    # remove all tables and scripts
+    for item in soup.find_all("table") + soup.find_all("script"):
+        item.decompose()
+    return soup.prettify()
diff --git a/scraper/src/mindtouch2zim/processor.py b/scraper/src/mindtouch2zim/processor.py
@@ -46,6 +46,7 @@
 from mindtouch2zim.errors import NoIllustrationFoundError
 from mindtouch2zim.html import get_text
 from mindtouch2zim.html_rewriting import HtmlUrlsRewriter
+from mindtouch2zim.libretexts.glossary import rewrite_glossary
 from mindtouch2zim.ui import (
     ConfigModel,
     PageContentModel,
@@ -474,7 +475,13 @@ def _process_page(
             post_head_insert=None,
             notify_js_module=None,
         )
-        rewriten = rewriter.rewrite(page_content.html_body)
+        if (
+            self.mindtouch_client.library_url.endswith(".libretexts.org")
+            and page.title == "Glossary"
+        ):
+            rewriten = rewrite_glossary(page_content.html_body)
+        else:
+            rewriten = rewriter.rewrite(page_content.html_body).content
         for path, urls in url_rewriter.items_to_download.items():
             if path in self.items_to_download:
                 self.items_to_download[path].urls.update(urls)
@@ -484,14 +491,12 @@ def _process_page(
                 )
         creator.add_item_for(
             f"content/page_content_{page.id}.json",
-            content=PageContentModel(html_body=rewriten.content).model_dump_json(
-                by_alias=True
-            ),
+            content=PageContentModel(html_body=rewriten).model_dump_json(by_alias=True),
         )
         self._add_indexing_item_to_zim(
             creator=creator,
             title=page.title,
-            content=get_text(rewriten.content),
+            content=get_text(rewriten),
             fname=f"page_{page.id}",
             zimui_redirect=page.path,
         )

diff --git a/zimui/public/custom.css b/zimui/public/custom.css
@@ -49,3 +49,28 @@ a[href^="https://"]:after
   height: auto;
   display: block;
 }
+
+/* additional CSS for libretexts.org glossary pages, reproduced here for simplificity
+and maintainability (less sensitive to file move upstream)
+*/
+
+.glossaryTerm {
+  font-weight: bold;
+}
+
+p.glossaryElement {
+  font-size: 1em;
+  margin: 0.5rem 0 1em;
+}
+
+@media print {
+  .glossaryDefinition {
+    font-size: 0.85em;
+  }
+  p.glossaryElement {
+    break-inside: avoid;
+    margin: 0.4rem 0;
+    font-size: 0.9em;
+    line-height: 0.9em;
+  }
+}