Merge pull request #15 from mcqueary/feature/dependency-changes

fixing dependencies
mcqueary · Dec 15, 2023 · 96c1d06 · 96c1d06
2 parents 78cabe7 + 43d4898
commit 96c1d06
Show file tree

Hide file tree

Showing 46 changed files with 33,870 additions and 34,011 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -60,20 +60,19 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
 beautifulsoup4 = "4.12.2"
-lxml = "4.9.3"
 tqdm = "4.66.1"
 typer = "0.9.0"
 
 
 [tool.poetry.group.test.dependencies]
 coverage = ">=4.1,<6.0.dev0 || >6.1,<6.1.1 || >6.1.1,<7.0"
-faker = "^20.0.3"
+coveralls = "^3.3.1"
 flake8 = "^6.1.0"
 pytest = "^7.4.3"
 pytest-cov = "^4.1.0"
 pytest-helpers-namespace = "^2021.12.29"
-coveralls = "^3.3.1"
 pytest-integration = "^0.2.3"
+vcrpy = "^5.1.0"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -84,7 +83,6 @@ pre-commit = "^3.5.0"
 mypy = "^1.7.1"
 types-beautifulsoup4 = "^4.12.0.7"
 types-tqdm = "^4.66.0.5"
-vcrpy = "^5.1.0"
 
 
 [tool.poetry.scripts]

diff --git a/src/graver/api.py b/src/graver/api.py
@@ -9,7 +9,7 @@
 from re import Match
 from time import sleep
 from typing import List, Optional, cast
-from urllib.parse import parse_qs, parse_qsl, urlencode, urlparse, urlunparse
+from urllib.parse import parse_qsl, urlparse, urlunparse
 
 import requests
 from bs4 import BeautifulSoup, Tag
@@ -99,7 +99,7 @@ def __init__(self, findagrave_url: str, **kwargs) -> None:
 
         if self.get:
             response = self.driver.get(findagrave_url, params=self.params)
-            self.soup = BeautifulSoup(response.content, "lxml")
+            self.soup = BeautifulSoup(response.content, "html.parser")
 
         if self.scrape:
             self.scrape_cemetery_info()
@@ -377,7 +377,7 @@ def __init__(self, findagrave_url: str, **kwargs) -> None:
 
         if self.get:
             response = self.driver.get(self.findagrave_url)
-            self.soup = BeautifulSoup(response.content, "lxml")
+            self.soup = BeautifulSoup(response.content, "html.parser")
 
             if not response.ok:
                 if self.check_removed():
@@ -612,7 +612,7 @@ def scrape_page(self):
         dt_list = vitals.find("dl").find_all("dt")
         for dt in dt_list:
             text = dt.get_text(strip=True)
-            if text == "Original Name":
+            if text.startswith("Original Name"):
                 oname = dt.find_next("dd")
                 self.original_name = oname.get_text()
             # elif dt.find("span", id="birthLabel") is not None:
@@ -844,7 +844,7 @@ def search(self):
         # Load the first page to learn how many results there may be
         log.debug(f"Search params={self.params}")
         response = self.driver.get(self.search_url, params=self.params)
-        soup = BeautifulSoup(response.content, "lxml")
+        soup = BeautifulSoup(response.content, "html.parser")
 
         # If this query isn't for a specific page, calculate how many
         # pages the results will span
@@ -865,9 +865,9 @@ def search(self):
 
         # scrape additional pages, if there are any left to get
         for i in range(2, num_pages + 1):
-            url = self.get_page_url(self.search_url, page=i)
-            response = self.driver.get(url)
-            soup = BeautifulSoup(response.content, "lxml")
+            self.params["page"] = i
+            response = self.driver.get(self.search_url, params=self.params)
+            soup = BeautifulSoup(response.content, "html.parser")
 
             results = self.scrape_results_page(soup, max_results=(count - len(rs)))
             rs.extend(results)
@@ -1008,14 +1008,6 @@ def scrape_results_page(
 
         return results
 
-    @staticmethod
-    def get_page_url(url, page) -> Optional[str]:
-        parsed = urlparse(url)
-        qs: dict = parse_qs(parsed.query)
-        qs["page"] = int(page)
-        new_url = parsed._replace(query=urlencode(qs)).geturl()
-        return new_url
-
 
 class ResultSet(list):
     """A ResultSet is just a list that keeps track of the object

diff --git a/tests/fixtures/vcr_cassettes/404-cemetery.yaml b/tests/fixtures/vcr_cassettes/404-cemetery.yaml
diff --git a/tests/fixtures/vcr_cassettes/404-memorial.yaml b/tests/fixtures/vcr_cassettes/404-memorial.yaml
diff --git a/tests/fixtures/vcr_cassettes/andrew-jackson.yaml b/tests/fixtures/vcr_cassettes/andrew-jackson.yaml
diff --git a/tests/fixtures/vcr_cassettes/arlington-national-cemetery-search-results.yaml b/tests/fixtures/vcr_cassettes/arlington-national-cemetery-search-results.yaml
diff --git a/tests/fixtures/vcr_cassettes/carl-sagan.yaml b/tests/fixtures/vcr_cassettes/carl-sagan.yaml
diff --git a/tests/fixtures/vcr_cassettes/cemetery-monticello-graveyard.yaml b/tests/fixtures/vcr_cassettes/cemetery-monticello-graveyard.yaml
diff --git a/tests/fixtures/vcr_cassettes/crown-hill-memorial-park.yaml b/tests/fixtures/vcr_cassettes/crown-hill-memorial-park.yaml
diff --git a/tests/fixtures/vcr_cassettes/dennis-macalistair-ritchie.yaml b/tests/fixtures/vcr_cassettes/dennis-macalistair-ritchie.yaml
diff --git a/tests/fixtures/vcr_cassettes/george-washington.yaml b/tests/fixtures/vcr_cassettes/george-washington.yaml
diff --git a/tests/fixtures/vcr_cassettes/grace-brewster-hopper.yaml b/tests/fixtures/vcr_cassettes/grace-brewster-hopper.yaml
diff --git a/tests/fixtures/vcr_cassettes/isaac-asimov.yaml b/tests/fixtures/vcr_cassettes/isaac-asimov.yaml
diff --git a/tests/fixtures/vcr_cassettes/jacob-wolf.yaml b/tests/fixtures/vcr_cassettes/jacob-wolf.yaml
diff --git a/tests/fixtures/vcr_cassettes/james-fenimore-cooper-search-results.yaml b/tests/fixtures/vcr_cassettes/james-fenimore-cooper-search-results.yaml
diff --git a/tests/fixtures/vcr_cassettes/john-j-pershing-search-results.yaml b/tests/fixtures/vcr_cassettes/john-j-pershing-search-results.yaml
diff --git a/tests/fixtures/vcr_cassettes/john-j-pershing.yaml b/tests/fixtures/vcr_cassettes/john-j-pershing.yaml
diff --git a/tests/fixtures/vcr_cassettes/john-quincy-adams.yaml b/tests/fixtures/vcr_cassettes/john-quincy-adams.yaml
diff --git a/tests/fixtures/vcr_cassettes/martin-luther-king.yaml b/tests/fixtures/vcr_cassettes/martin-luther-king.yaml
diff --git a/tests/fixtures/vcr_cassettes/rod-serling.yaml b/tests/fixtures/vcr_cassettes/rod-serling.yaml
diff --git a/tests/fixtures/vcr_cassettes/synopsis.yaml b/tests/fixtures/vcr_cassettes/synopsis.yaml
@@ -73,7 +73,7 @@ interactions:
         z80ONvYPfbCYvcXRtAguHy0UXuUTVARRZc14fhKYD/9z/2S3cG8R3rPrira5svpfWcKSb8oqAAA=
     headers:
       Age:
-      - '15289'
+      - '1078'
       Cache-Control:
       - public, max-age=21603
       Connection:
@@ -93,9 +93,9 @@ interactions:
       Content-Type:
       - text/html; charset=UTF-8
       Date:
-      - Thu, 14 Dec 2023 20:03:23 GMT
+      - Fri, 15 Dec 2023 20:58:23 GMT
       Expires:
-      - Thu, 14 Dec 2023 20:11:15 GMT
+      - Fri, 15 Dec 2023 22:44:16 GMT
       Keep-Alive:
       - timeout=2, max=358
       Last-Modified:

diff --git a/tests/fixtures/vcr_cassettes/test-cli-scrape-file.yaml b/tests/fixtures/vcr_cassettes/test-cli-scrape-file.yaml
diff --git a/tests/fixtures/vcr_cassettes/test-search-cemetery-monticello-vets.yaml b/tests/fixtures/vcr_cassettes/test-search-cemetery-monticello-vets.yaml
diff --git a/tests/fixtures/vcr_cassettes/test-search-monticello-cemetery-famous-vets.yaml b/tests/fixtures/vcr_cassettes/test-search-monticello-cemetery-famous-vets.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cemetery_search-cemetery-641519-p3.yaml b/tests/fixtures/vcr_cassettes/test_cemetery_search-cemetery-641519-p3.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cemetery_search_all-rachel-levy-gravesite.yaml b/tests/fixtures/vcr_cassettes/test_cemetery_search_all-rachel-levy-gravesite.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cemetery_search_identifies_memorial_type_cenotaph.yaml b/tests/fixtures/vcr_cassettes/test_cemetery_search_identifies_memorial_type_cenotaph.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cemetery_search_identifies_memorial_type_monument.yaml b/tests/fixtures/vcr_cassettes/test_cemetery_search_identifies_memorial_type_monument.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cemetery_search_max_results.yaml b/tests/fixtures/vcr_cassettes/test_cemetery_search_max_results.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cli_search_in_cemetery.yaml b/tests/fixtures/vcr_cassettes/test_cli_search_in_cemetery.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_cli_search_no_cemetery.yaml b/tests/fixtures/vcr_cassettes/test_cli_search_no_cemetery.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_memorial_parser_merged_raises_exception.yaml b/tests/fixtures/vcr_cassettes/test_memorial_parser_merged_raises_exception.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_memorial_parser_removed_raises_exception.yaml b/tests/fixtures/vcr_cassettes/test_memorial_parser_removed_raises_exception.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_empty.yaml b/tests/fixtures/vcr_cassettes/test_search_empty.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-andrew-jackson.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-andrew-jackson.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-carl-sagan.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-carl-sagan.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-george-washington.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-george-washington.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-grace-brewster-hopper.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-grace-brewster-hopper.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-isaac-asimov.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-isaac-asimov.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-john-j-pershing.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-john-j-pershing.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-martin-luther-king.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-martin-luther-king.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-rod-serling.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-rod-serling.yaml
diff --git a/tests/fixtures/vcr_cassettes/test_search_famous_people-thomas-jefferson.yaml b/tests/fixtures/vcr_cassettes/test_search_famous_people-thomas-jefferson.yaml
diff --git a/tests/fixtures/vcr_cassettes/thomas-jefferson.yaml b/tests/fixtures/vcr_cassettes/thomas-jefferson.yaml
diff --git a/tests/test_api.py b/tests/test_api.py
@@ -257,6 +257,16 @@ def test_search(person: dict, cassette):
             assert result.nickname == person["nickname"]
 
 
+@pytest.mark.parametrize(
+    "args, expected", [({"lastname": "Jackson", "max_results": 37}, 37)]
+)
+def test_search_max_results(args, expected):
+    logging.getLogger().setLevel(logging.DEBUG)
+    rs = Memorial.search(**args)
+    assert 0 < len(rs) <= expected
+    pass
+
+
 @pytest.mark.parametrize("name", famous_memorials)
 def test_search_famous_people(name: str):
     person = pytest.helpers.load_memorial_from_json(name)