Improve scrape performance by using lxml parser (home-assistant#71087)

* Improve scape performance by using lxml parser * load it * tweak * tweak * ensure libxml2 is installed in dev container
peribeir · May 2, 2022 · b770ca3 · b770ca3
1 parent c23866e
commit b770ca3
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 2 deletions.
diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -18,6 +18,7 @@ RUN \
         libavfilter-dev \
         libpcap-dev \
         libturbojpeg0 \
+        libxml2 \
         git \
         cmake \
     && apt-get clean \

diff --git a/homeassistant/components/scrape/manifest.json b/homeassistant/components/scrape/manifest.json
@@ -2,7 +2,7 @@
   "domain": "scrape",
   "name": "Scrape",
   "documentation": "https://www.home-assistant.io/integrations/scrape",
-  "requirements": ["beautifulsoup4==4.11.1"],
+  "requirements": ["beautifulsoup4==4.11.1", "lxml==4.8.0"],
   "after_dependencies": ["rest"],
   "codeowners": ["@fabaff"],
   "iot_class": "cloud_polling"

diff --git a/homeassistant/components/scrape/sensor.py b/homeassistant/components/scrape/sensor.py
@@ -154,7 +154,7 @@ def __init__(
 
     def _extract_value(self) -> Any:
         """Parse the html extraction in the executor."""
-        raw_data = BeautifulSoup(self.rest.data, "html.parser")
+        raw_data = BeautifulSoup(self.rest.data, "lxml")
         _LOGGER.debug(raw_data)
 
         try:

diff --git a/requirements_all.txt b/requirements_all.txt
@@ -974,6 +974,9 @@ lupupy==0.0.24
 # homeassistant.components.lw12wifi
 lw12==0.9.2
 
+# homeassistant.components.scrape
+lxml==4.8.0
+
 # homeassistant.components.nmap_tracker
 mac-vendor-lookup==0.1.11
 

diff --git a/requirements_test_all.txt b/requirements_test_all.txt
@@ -663,6 +663,9 @@ lru-dict==1.1.7
 # homeassistant.components.luftdaten
 luftdaten==0.7.2
 
+# homeassistant.components.scrape
+lxml==4.8.0
+
 # homeassistant.components.nmap_tracker
 mac-vendor-lookup==0.1.11