small workaround to deal with html comments in JSON-LD

pangaea-data-publisher · Oct 16, 2023 · f167b51 · f167b51
1 parent 6a10273
commit f167b51
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/fuji_server/harvester/metadata_harvester.py b/fuji_server/harvester/metadata_harvester.py
@@ -615,12 +615,19 @@ def retrieve_metadata_embedded_extruct(self):
             except Exception:
                 extruct_target = self.landing_html
                 pass
+
             try:
                 self.logger.info(
                     "{} : Trying to identify EMBEDDED  Microdata, OpenGraph or Schema.org -: {}".format(
                         "FsF-F2-01M", self.landing_url
                     )
                 )
+                # remove html comments which sometimes fails in extruct...
+                try:
+                    extruct_target = re.sub("(<!--.*?-->)", "", extruct_target.decode("utf-8"))
+                except Exception:
+                    pass
+
                 extracted = extruct.extract(extruct_target, syntaxes=syntaxes, encoding="utf-8")
             except Exception as e:
                 extracted = {}