implementing the UNIMPLEMENTED_PARSERS

align_data/common/alignment_dataset.py

@@ -4,7 +4,7 @@
 import time
 from dataclasses import dataclass, field, KW_ONLY
 from pathlib import Path
-from typing import List, Optional, Dict, Any, Set, Iterable, Tuple, Generator
+from typing import List, Optional, Set, Iterable, Tuple, Generator
 
 import pytz
 from sqlalchemy import select, Select, JSON
@@ -189,7 +189,7 @@ def unprocessed_items(self, items=None) -> list | filter:
 
         return items_to_process
 
-    def fetch_entries(self) -> Generator[Article, None, None]:
+    def fetch_entries(self):
         """Get all entries to be written to the file."""
         for item in tqdm(self.unprocessed_items(), desc=f"Processing {self.name}"):
             entry = self.process_entry(item)

align_data/common/html_dataset.py

@@ -1,11 +1,11 @@
-import pytz
 import logging
 from datetime import datetime
 from dataclasses import dataclass, field
 from urllib.parse import urljoin
 from typing import List, Dict, Any
 import re
 
+import pytz
 import requests
 import feedparser
 from bs4 import BeautifulSoup
@@ -17,7 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclass()
+@dataclass
 class HTMLDataset(AlignmentDataset):
     """
     Fetches articles from a different blog by collecting links to articles from an index page.
@@ -34,7 +34,7 @@ class HTMLDataset(AlignmentDataset):
     source_type = "blog"
     ignored_selectors = []
 
-    def extract_authors(self, article): #TODO: make this work
+    def extract_authors(self, article):
         return self.authors
 
 

align_data/db/models.py

@@ -113,8 +113,6 @@ def update(self, other: "Article"):
             if field not in ["id", "hash_id", "metadata"] and getattr(other, field):
                 setattr(self, field, getattr(other, field))
         self.meta = dict((self.meta or {}), **{k: v for k, v in other.meta.items() if k and v})
-        # TODO: verify that this actually updates the meta column; 
-        # https://amercader.net/blog/beware-of-json-fields-in-sqlalchemy/
 
         if other._id:
             self._id = other._id
@@ -129,8 +127,6 @@ def add_meta(self, key: str, val):
         if self.meta is None:
             self.meta = {}
         self.meta[key] = val
-        # TODO: verify that this actually updates the meta column; 
-        # https://amercader.net/blog/beware-of-json-fields-in-sqlalchemy/
 
     def append_comment(self, comment: str):
         """Appends a comment to the article.comments field. You must run session.commit() to save the comment to the database."""

align_data/db/session.py

@@ -1,4 +1,4 @@
-from typing import List, Generator
+from typing import List
 import logging
 
 from contextlib import contextmanager

align_data/postprocess/postprocess.py

@@ -3,9 +3,10 @@
 from dataclasses import dataclass, field
 from typing import List, DefaultDict
 import logging
+from pathlib import Path
+
 import jsonlines
 from tqdm import tqdm
-from pathlib import Path
 import pylab as plt
 from nltk.tokenize import sent_tokenize, word_tokenize
 import seaborn as sns #TODO: install seaborn or fix this file