introduce 'source' to spiders

Signed-off-by: Szymon Łopaciuk <[email protected]>
szymonlopaciuk · Feb 1, 2018 · 1c82c2a · 1c82c2a
1 parent 5bd3c5f
commit 1c82c2a
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 1 deletion.
diff --git a/hepcrawl/pipelines.py b/hepcrawl/pipelines.py
@@ -97,7 +97,7 @@ def open_spider(self, spider):
         self.results_data = []
 
     def _post_enhance_item(self, item, spider):
-        source = spider.name
+        source = spider.source
 
         enhanced_record = item_to_hep(
             item=item,

diff --git a/hepcrawl/spiders/__init__.py b/hepcrawl/spiders/__init__.py
@@ -16,3 +16,7 @@ class StatefulSpider(Spider):
     def __init__(self, *args, **kwargs):
         self.state = {}
         super(StatefulSpider, self).__init__(*args, **kwargs)
+
+    @property
+    def source(self):
+        return self.name
diff --git a/tests/unit/test_pipelines.py b/tests/unit/test_pipelines.py
@@ -31,6 +31,7 @@
 def spider():
     mock_spider = mock.create_autospec(Spider)
     mock_spider.name = 'arXiv'
+    mock_spider.source = 'arXiv'
     mock_spider.state = {}
     return mock_spider