diff --git a/.travis.yml b/.travis.yml index f05e2d22..91407e6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ env: - SUITE=unit - SUITE=functional_wsp - SUITE=functional_arxiv + - SUITE=functional_desy matrix: fast_finish: true diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 7ffe0122..d14d7c87 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -17,6 +17,7 @@ services: - APP_CELERY_RESULT_BACKEND=amqp://guest:guest@rabbitmq:5672// - APP_CRAWLER_HOST_URL=http://scrapyd:6800 - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results + - APP_FILES_STORE=/tmp/file_urls - COVERAGE_PROCESS_START=/code/.coveragerc - BASE_USER_UID=${BASE_USER_UID:-1000} - BASE_USER_GIT=${BASE_USER_GIT:-1000} @@ -26,6 +27,7 @@ services: - ${PWD}:/code/ - ${PWD}/tests/functional/scrapyd_coverage_runner.conf:/etc/scrapyd/scrapyd.conf - /tmp/WSP:/tmp/WSP + - /tmp/file_urls:/tmp/file_urls functional_wsp: <<: *service_base @@ -34,6 +36,13 @@ services: - scrapyd - ftp_server + functional_desy: + <<: *service_base + command: py.test -vv tests/functional/desy + links: + - scrapyd + - ftp_server + functional_arxiv: <<: *service_base command: py.test -vv tests/functional/arxiv @@ -68,6 +77,8 @@ services: environment: - PUBLICHOST=localhost volumes: + - ${PWD}/tests/functional/desy/fixtures/ftp_server/FFT:/home/ftpusers/bob/FFT + - ${PWD}/tests/functional/desy/fixtures/ftp_server/DESY:/home/ftpusers/bob/DESY - ${PWD}/tests/functional/wsp/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP - ${PWD}/tests/functional/wsp/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd diff --git a/docs/Makefile b/docs/Makefile index 28fb79f1..7d24ea24 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -24,7 +24,7 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -SPHINXAPIDOC = sphinx-apidoc -M -P -f -o $(SOURCEDIR) $(CODEDIR) -E $(CODEDIR)/spiders +SPHINXAPIDOC = sphinx-apidoc --module-first --private --force --separate --output-dir $(SOURCEDIR) $(CODEDIR) $(CODEDIR)/spiders .PHONY: help help: diff --git a/docs/conf.py b/docs/conf.py index 7fb14e5e..2e12401b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,6 +85,13 @@ def _warn_node(self, msg, *args, **kwargs): 'sphinx.ext.todo', ] + +autodoc_default_flags = [ + 'members', + 'private-members', + 'show-inheritance', +] + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/spiders.rst b/docs/spiders.rst index aab52646..d016e521 100644 --- a/docs/spiders.rst +++ b/docs/spiders.rst @@ -50,6 +50,12 @@ Brown .. automodule:: hepcrawl.spiders.brown_spider :members: +Desy +~~~~ + +.. automodule:: hepcrawl.spiders.desy_spider + :members: + DNB ~~~ diff --git a/hepcrawl/crawler2hep.py b/hepcrawl/crawler2hep.py deleted file mode 100644 index d6898022..00000000 --- a/hepcrawl/crawler2hep.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of hepcrawl. -# Copyright (C) 2015, 2016, 2017 CERN. -# -# hepcrawl is a free software; you can redistribute it and/or modify it -# under the terms of the Revised BSD License; see LICENSE file for -# more details. - -"""Convert a crawler record to a valid HEP record. - -Don't forget to add pipelines to the ITEM_PIPELINES setting -See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html -""" - -from __future__ import absolute_import, division, print_function - -from inspire_schemas.api import LiteratureBuilder - - -def crawler2hep(crawler_record): - - def _filter_affiliation(affiliations): - return [ - affilation.get('value') - for affilation in affiliations - if affilation.get('value') - ] - - builder = LiteratureBuilder( - source=crawler_record['acquisition_source']['source'] - ) - - for author in crawler_record.get('authors', []): - builder.add_author(builder.make_author( - author['full_name'], - affiliations=_filter_affiliation(author['affiliations']), - )) - - for title in crawler_record.get('titles', []): - builder.add_title( - title=title.get('title'), - source=title.get('source') - ) - - for abstract in crawler_record.get('abstracts', []): - builder.add_abstract( - abstract=abstract.get('value'), - source=abstract.get('source') - ) - - for arxiv_eprint in crawler_record.get('arxiv_eprints', []): - builder.add_arxiv_eprint( - arxiv_id=arxiv_eprint.get('value'), - arxiv_categories=arxiv_eprint.get('categories') - ) - - for doi in crawler_record.get('dois', []): - builder.add_doi( - doi=doi.get('value'), - material=doi.get('material'), - ) - - for public_note in crawler_record.get('public_notes', []): - builder.add_public_note( - public_note=public_note.get('value'), - source=public_note.get('source') - ) - - for license in crawler_record.get('license', []): - builder.add_license( - url=license.get('url'), - license=license.get('license'), - material=license.get('material'), - ) - - for collaboration in crawler_record.get('collaborations', []): - builder.add_collaboration( - collaboration=collaboration.get('value') - ) - - for imprint in crawler_record.get('imprints', []): - builder.add_imprint_date( - imprint_date=imprint.get('date') - ) - - for copyright in crawler_record.get('copyright', []): - builder.add_copyright( - holder=copyright.get('holder'), - material=copyright.get('material'), - statement=copyright.get('statement') - ) - - builder.add_preprint_date( - preprint_date=crawler_record.get('preprint_date') - ) - - acquisition_source = crawler_record.get('acquisition_source', {}) - builder.add_acquisition_source( - method=acquisition_source['method'], - date=acquisition_source['date'], - source=acquisition_source['source'], - submission_number=acquisition_source['submission_number'], - ) - - try: - builder.add_number_of_pages( - number_of_pages=int(crawler_record.get('page_nr', [])[0]) - ) - except (TypeError, ValueError, IndexError): - pass - - publication_types = [ - 'introductory', - 'lectures', - 'review', - ] - - special_collections = [ - 'cdf-internal-note', - 'cdf-note', - 'cds', - 'd0-internal-note', - 'd0-preliminary-note', - 'h1-internal-note', - 'h1-preliminary-note', - 'halhidden', - 'hephidden', - 'hermes-internal-note', - 'larsoft-internal-note', - 'larsoft-note', - 'zeus-internal-note', - 'zeus-preliminary-note', - ] - - document_types = [ - 'book', - 'note', - 'report', - 'proceedings', - 'thesis', - ] - - added_doc_type = False - - for collection in crawler_record.get('collections', []): - collection = collection['primary'].strip().lower() - - if collection == 'arxiv': - continue # ignored - elif collection == 'citeable': - builder.set_citeable(True) - elif collection == 'core': - builder.set_core(True) - elif collection == 'noncore': - builder.set_core(False) - elif collection == 'published': - builder.set_refereed(True) - elif collection == 'withdrawn': - builder.set_withdrawn(True) - elif collection in publication_types: - builder.add_publication_type(collection) - elif collection in special_collections: - builder.add_special_collection(collection.upper()) - elif collection == 'bookchapter': - added_doc_type = True - builder.add_document_type('book chapter') - elif collection == 'conferencepaper': - added_doc_type = True - builder.add_document_type('conference paper') - elif collection in document_types: - added_doc_type = True - builder.add_document_type(collection) - - if not added_doc_type: - builder.add_document_type('article') - - _pub_info = crawler_record.get('publication_info', [{}])[0] - builder.add_publication_info( - year=_pub_info.get('year'), - artid=_pub_info.get('artid'), - page_end=_pub_info.get('page_end'), - page_start=_pub_info.get('page_start'), - journal_issue=_pub_info.get('journal_issue'), - journal_title=_pub_info.get('journal_title'), - journal_volume=_pub_info.get('journal_volume'), - pubinfo_freetext=_pub_info.get('pubinfo_freetext'), - material=_pub_info.get('pubinfo_material'), - ) - - for report_number in crawler_record.get('report_numbers', []): - builder.add_report_number( - report_number=report_number.get('value'), - source=report_number.get('source') - ) - - builder.validate_record() - - return builder.record diff --git a/hepcrawl/pipelines.py b/hepcrawl/pipelines.py index 62ba867c..8cd31c0e 100644 --- a/hepcrawl/pipelines.py +++ b/hepcrawl/pipelines.py @@ -15,30 +15,69 @@ from __future__ import absolute_import, division, print_function -import datetime import os import requests -from .crawler2hep import crawler2hep +from scrapy import Request +from scrapy.pipelines.files import FilesPipeline +from inspire_schemas.utils import validate -def has_publication_info(item): - """If any publication info.""" - return item.get('pubinfo_freetext') or item.get('journal_volume') or \ - item.get('journal_title') or \ - item.get('journal_year') or \ - item.get('journal_issue') or \ - item.get('journal_fpage') or \ - item.get('journal_lpage') or \ - item.get('journal_artid') or \ - item.get('journal_doctype') +from .tohep import item_to_hep +from .settings import FILES_STORE +from .utils import RecordFile -def filter_fields(item, keys): - """Filter away keys.""" - for key in keys: - item.pop(key, None) +class FftFilesPipeline(FilesPipeline): + """Download all the FFT files provided by record. + + Note: + + This pipeline only runs if the spider returns a + :class:`hepcrawl.utils.ParsedItem` that has a ``file_urls`` property. + See the scrapy docs on it for more details. + https://doc.scrapy.org/en/latest/topics/media-pipeline.html?highlight=file_urls#using-the-files-pipeline + """ + + def __init__(self, store_uri, *args, **kwargs): + store_uri = store_uri or FILES_STORE + super(FftFilesPipeline, self).__init__( + *args, + store_uri=store_uri, + **kwargs + ) + + def get_media_requests(self, item, info): + """Download FFT files using FTP.""" + if item.get('file_urls'): + for fft_url in item.file_urls: + yield Request( + url=fft_url, + meta=item.ftp_params, + ) + + def get_absolute_file_path(self, path): + return os.path.abspath( + os.path.join( + self.store.basedir, + path + ) + ) + + def item_completed(self, results, item, info): + """Create a map that connects file names with downloaded files.""" + record_files = [ + RecordFile( + path=self.get_absolute_file_path(result_data['path']), + name=os.path.basename(result_data['url']), + ) + for ok, result_data in results + if ok + ] + item.record_files = record_files + + return item class InspireAPIPushPipeline(object): @@ -50,74 +89,26 @@ def __init__(self): def open_spider(self, spider): self.results_data = [] + def _post_enhance_item(self, item, spider): + source = spider.name + + return item_to_hep( + item=item, + source=source, + ) + def process_item(self, item, spider): """Convert internal format to INSPIRE data model.""" self.count += 1 - if 'related_article_doi' in item: - item['dois'] += item.pop('related_article_doi', []) - source = spider.name - item['acquisition_source'] = { - 'source': source, - 'method': 'hepcrawl', - 'date': datetime.datetime.now().isoformat(), - 'submission_number': os.environ.get('SCRAPY_JOB', ''), - } - - item['titles'] = [{ - 'title': item.pop('title', ''), - 'subtitle': item.pop('subtitle', ''), - 'source': source, - }] - item['abstracts'] = [{ - 'value': item.pop('abstract', ''), - 'source': source, - }] - item['imprints'] = [{ - 'date': item.pop('date_published', ''), - }] - item['copyright'] = [{ - 'holder': item.pop('copyright_holder', ''), - 'year': item.pop('copyright_year', ''), - 'statement': item.pop('copyright_statement', ''), - 'material': item.pop('copyright_material', ''), - }] - if not item.get('publication_info'): - if has_publication_info(item): - item['publication_info'] = [{ - 'journal_title': item.pop('journal_title', ''), - 'journal_volume': item.pop('journal_volume', ''), - 'journal_issue': item.pop('journal_issue', ''), - 'artid': item.pop('journal_artid', ''), - 'page_start': item.pop('journal_fpage', ''), - 'page_end': item.pop('journal_lpage', ''), - 'note': item.pop('journal_doctype', ''), - 'pubinfo_freetext': item.pop('pubinfo_freetext', ''), - 'pubinfo_material': item.pop('pubinfo_material', ''), - }] - if item.get('journal_year'): - item['publication_info'][0]['year'] = int( - item.pop('journal_year') - ) - - # Remove any fields - filter_fields(item, [ - 'journal_title', - 'journal_volume', - 'journal_year', - 'journal_issue', - 'journal_fpage', - 'journal_lpage', - 'journal_doctype', - 'journal_artid', - 'pubinfo_freetext', - 'pubinfo_material', - ]) - - item = crawler2hep(dict(item)) - spider.logger.debug('Validated item.') - self.results_data.append(item) - return item + hep_record = self._post_enhance_item(item, spider) + + validate(hep_record, 'hep') + spider.logger.debug('Validated item by Inspire Schemas.') + + self.results_data.append(hep_record) + + return hep_record def _prepare_payload(self, spider): """Return payload for push.""" diff --git a/hepcrawl/settings.py b/hepcrawl/settings.py index 71dcfc75..31d608bf 100644 --- a/hepcrawl/settings.py +++ b/hepcrawl/settings.py @@ -85,7 +85,7 @@ # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { - 'scrapy.pipelines.files.FilesPipeline': 1, + 'hepcrawl.pipelines.FftFilesPipeline': 1, 'hepcrawl.pipelines.InspireCeleryPushPipeline': 300, } @@ -123,6 +123,11 @@ # ==== JOBDIR = "jobs" +# Marc to HEP conversion settings (Desy) +MARC_TO_HEP_SETTINGS = { + 'LEGACY_BASE_URL': 'https://inspirehep.net', + 'SERVER_NAME': 'https://labs.inspirehep.net', +} # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html diff --git a/hepcrawl/spiders/alpha_spider.py b/hepcrawl/spiders/alpha_spider.py index 2ab883f3..c791546e 100644 --- a/hepcrawl/spiders/alpha_spider.py +++ b/hepcrawl/spiders/alpha_spider.py @@ -20,7 +20,10 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import has_numbers +from ..utils import ( + has_numbers, + ParsedItem, +) class AlphaSpider(CrawlSpider): @@ -145,4 +148,9 @@ def parse(self, response): record.add_value('source', 'Alpha experiment') record.add_value('collections', ['HEP', 'THESIS']) - yield record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + yield parsed_item diff --git a/hepcrawl/spiders/aps_spider.py b/hepcrawl/spiders/aps_spider.py index 496e2e8e..1cda5bea 100644 --- a/hepcrawl/spiders/aps_spider.py +++ b/hepcrawl/spiders/aps_spider.py @@ -20,7 +20,12 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import get_licenses, get_nested, build_dict +from ..utils import ( + get_licenses, + get_nested, + build_dict, + ParsedItem, +) class APSSpider(Spider): @@ -110,7 +115,13 @@ def parse(self, response): record.add_value('license', license) record.add_value('collections', ['HEP', 'Citeable', 'Published']) - yield record.load_item() + + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + yield parsed_item # Pagination support. Will yield until no more "next" pages are found if 'Link' in response.headers: diff --git a/hepcrawl/spiders/arxiv_spider.py b/hepcrawl/spiders/arxiv_spider.py index d82c8318..59a01295 100644 --- a/hepcrawl/spiders/arxiv_spider.py +++ b/hepcrawl/spiders/arxiv_spider.py @@ -16,10 +16,15 @@ from scrapy import Request, Selector from scrapy.spiders import XMLFeedSpider -from ..mappings import CONFERENCE_WORDS, THESIS_WORDS -from ..utils import coll_cleanforthe, get_licenses, split_fullname from ..items import HEPRecord from ..loaders import HEPLoader +from ..mappings import CONFERENCE_WORDS, THESIS_WORDS +from ..utils import ( + coll_cleanforthe, + get_licenses, + split_fullname, + ParsedItem, +) RE_CONFERENCE = re.compile(r'\b(%s)\b' % '|'.join( [re.escape(word) for word in CONFERENCE_WORDS]), re.I | re.U) @@ -33,7 +38,9 @@ class ArxivSpider(XMLFeedSpider): Example: Using OAI-PMH XML files:: - $ scrapy crawl arXiv -a source_file=file://`pwd`/tests/responses/arxiv/sample_arxiv_record.xml + $ scrapy crawl \\ + arXiv \\ + -a "source_file=file://$PWD/tests/responses/arxiv/sample_arxiv_record.xml" """ @@ -110,8 +117,12 @@ def parse_node(self, response, node): ) record.add_value('license', license) - parsed_record = dict(record.load_item()) - return parsed_record + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item def _get_authors_or_collaboration(self, node): """Parse authors, affiliations; extract collaboration""" diff --git a/hepcrawl/spiders/base_spider.py b/hepcrawl/spiders/base_spider.py index 5eb22eb7..d7c2d06d 100644 --- a/hepcrawl/spiders/base_spider.py +++ b/hepcrawl/spiders/base_spider.py @@ -18,7 +18,12 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import get_mime_type, parse_domain, get_node +from ..utils import ( + get_mime_type, + parse_domain, + get_node, + ParsedItem, +) class BaseSpider(XMLFeedSpider): @@ -192,7 +197,13 @@ def build_item(self, response): record.add_value("authors", self.get_authors(node)) record.add_value('thesis', {'degree_type': 'PhD'}) record.add_value('collections', ['HEP', 'THESIS']) - return record.load_item() + + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item def scrape_for_pdf(self, response): """Scrape splash page for any links to PDFs. diff --git a/hepcrawl/spiders/brown_spider.py b/hepcrawl/spiders/brown_spider.py index 6c881252..f17dd197 100644 --- a/hepcrawl/spiders/brown_spider.py +++ b/hepcrawl/spiders/brown_spider.py @@ -21,7 +21,12 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import split_fullname, parse_domain, get_mime_type +from ..utils import ( + split_fullname, + parse_domain, + get_mime_type, + ParsedItem, +) class BrownSpider(CrawlSpider): @@ -219,4 +224,9 @@ def build_item(self, response): record.add_value('thesis', response.meta.get("thesis")) record.add_value('collections', ['HEP', 'THESIS']) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/desy_spider.py b/hepcrawl/spiders/desy_spider.py new file mode 100644 index 00000000..ec70ec39 --- /dev/null +++ b/hepcrawl/spiders/desy_spider.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- +# +# This file is part of hepcrawl. +# Copyright (C) 2017 CERN. +# +# hepcrawl is a free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for +# more details. + +from __future__ import absolute_import, division, print_function + +import os + +from dojson.contrib.marc21.utils import create_record +from flask.app import Flask +from inspire_dojson.hep import hep +from lxml import etree +from scrapy import Request +from scrapy.spiders import Spider +from six.moves import urllib + +from ..utils import ( + ftp_list_files, + ftp_connection_info, + ParsedItem, +) + + +class DesySpider(Spider): + """This spider parses files in XML MARC format (collections or single + records). + + It can retrieve the files from a remote FTP or from a local directory, they + must have the extension ``.xml``. + + Args: + source_folder(str): Path to the folder with the MARC files to ingest, + might be collections or single records. Will be ignored if + ``ftp_host`` is passed. + + ftp_folder(str): Remote folder where to look for the XML files. + + ftp_host(str): + + ftp_netrc(str): Path to the ``.netrc`` file with the authentication + details for the ftp connection. For more details see: + https://linux.die.net/man/5/netrc + + destination_folder(str): Path to put the crawl results into. Will be + created if it does not exist. + + *args: will be passed to the contstructor of + :class:`scrapy.spiders.Spider`. + + **kwargs: will be passed to the contstructor of + :class:`scrapy.spiders.Spider`. + + Examples: + To run a crawl, you need to pass FTP connection information via + ``ftp_host`` and ``ftp_netrc``, if ``ftp_folder`` is not passed, it + will fallback to ``DESY``:: + + $ scrapy crawl desy \\ + -a 'ftp_host=ftp.example.com' \\ + -a 'ftp_netrc=/path/to/netrc' + + To run a crawl on local folder, you need to pass the absolute + ``source_folder``:: + + $ scrapy crawl desy -a 'source_folder=/path/to/package_dir' + """ + name = 'desy' + + def __init__( + self, + source_folder=None, + ftp_folder='/DESY', + ftp_host=None, + ftp_netrc=None, + destination_folder='/tmp/DESY', + *args, + **kwargs + ): + super(DesySpider, self).__init__(*args, **kwargs) + self.ftp_folder = ftp_folder + self.ftp_host = ftp_host + self.ftp_netrc = ftp_netrc + self.source_folder = source_folder + self.destination_folder = destination_folder + self.ftp_enabled = False if self.source_folder else True + + if self.ftp_enabled and not self.ftp_host: + raise Exception('You need to pass source_folder or ftp_host.') + + if not os.path.exists(self.destination_folder): + os.makedirs(self.destination_folder) + + @staticmethod + def _filter_xml_files(list_files_paths): + return ( + xml_file + for xml_file in list_files_paths + if xml_file.endswith('.xml') + ) + + def crawl_local_directory(self): + file_names = os.listdir(self.source_folder) + xml_file_names = self._filter_xml_files(file_names) + + for file_name in xml_file_names: + file_path = os.path.join(self.source_folder, file_name) + self.log('Local: Try to crawl local file: {0}'.format(file_path)) + yield Request( + 'file://{0}'.format(file_path), + callback=self.parse, + ) + + def crawl_ftp_directory(self): + ftp_host, ftp_params = ftp_connection_info( + self.ftp_host, + self.ftp_netrc, + ) + + remote_files_paths = ftp_list_files( + self.ftp_folder, + destination_folder=self.destination_folder, + ftp_host=ftp_host, + user=ftp_params['ftp_user'], + password=ftp_params['ftp_password'], + only_missing_files=False, + ) + + xml_remote_files_paths = self._filter_xml_files(remote_files_paths) + + for remote_file in xml_remote_files_paths: + self.log( + 'Remote: Try to crawl file from FTP: {0}'.format(remote_file), + ) + remote_file = str(remote_file) + ftp_params['ftp_local_filename'] = os.path.join( + self.destination_folder, + os.path.basename(remote_file), + ) + remote_url = 'ftp://{0}/{1}'.format(ftp_host, remote_file) + yield Request( + str(remote_url), + meta=ftp_params, + callback=self.handle_package_ftp, + ) + + def handle_package_ftp(self, response): + """Yield every XML file found. + + This is an intermediate step before calling :func:`DesySpider.parse` + to handle ftp downloaded "record collections". + + Args: + response(hepcrawl.http.response.Response): response containing the + information about the ftp file download. + """ + self.log('Visited url {}'.format(response.url)) + file_path = response.body + yield Request( + 'file://{0}'.format(file_path), + meta={'source_folder': file_path}, + callback=self.parse, + ) + + def start_requests(self): + """List selected folder on remote FTP and yield files.""" + + if self.ftp_enabled: + requests = self.crawl_ftp_directory() + else: + requests = self.crawl_local_directory() + + for request in requests: + yield request + + @staticmethod + def _get_full_uri(current_path, base_url, schema, hostname=''): + if os.path.isabs(current_path): + full_path = current_path + else: + full_path = os.path.join(base_url, current_path) + + return '{schema}://{hostname}{full_path}'.format(**vars()) + + def parse(self, response): + """Parse a ``Desy`` XML file into a :class:`hepcrawl.utils.ParsedItem`. + """ + + self.log('Got record from url/path: {0}'.format(response.url)) + self.log('FTP enabled: {0}'.format(self.ftp_enabled)) + ftp_params = None + + if self.ftp_enabled: + hostname, ftp_params = ftp_connection_info( + self.ftp_host, + self.ftp_netrc, + ) + base_url = self.ftp_folder + url_schema = 'ftp' + else: + base_url = os.path.dirname( + urllib.parse.urlparse(response.url).path + ) + url_schema = 'file' + hostname = None + + marcxml_records = self._get_marcxml_records(response.body) + hep_records = self._hep_records_from_marcxml(marcxml_records) + + for hep_record in hep_records: + list_file_urls = [ + self._get_full_uri( + current_path=fft_path['path'], + base_url=base_url, + schema=url_schema, + hostname=hostname, + ) + for fft_path in hep_record['_fft'] + ] + + parsed_item = ParsedItem( + record=hep_record, + file_urls=list_file_urls, + ftp_params=ftp_params, + record_format='hep', + ) + + yield parsed_item + + @staticmethod + def _get_marcxml_records(response_body): + root = etree.fromstring(response_body) + list_items = root.findall('.//{http://www.loc.gov/MARC21/slim}record') + if not list_items: + list_items = root.findall('.//record') + + return [etree.tostring(item) for item in list_items] + + def _hep_records_from_marcxml(self, marcxml_records): + def _create_json_record(xml_record): + object_record = create_record(etree.XML(xml_record)) + app = Flask('hepcrawl') + app.config.update( + self.settings.getdict('MARC_TO_HEP_SETTINGS', {}) + ) + with app.app_context(): + dojson_record = hep.do(object_record) + + return dojson_record + + hep_records = [] + for xml_record in marcxml_records: + json_record = _create_json_record(xml_record) + hep_records.append(json_record) + + return hep_records diff --git a/hepcrawl/spiders/dnb_spider.py b/hepcrawl/spiders/dnb_spider.py index 3ac8b901..f350cf8f 100644 --- a/hepcrawl/spiders/dnb_spider.py +++ b/hepcrawl/spiders/dnb_spider.py @@ -16,7 +16,12 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import get_mime_type, parse_domain, get_node +from ..utils import ( + get_mime_type, + parse_domain, + get_node, + ParsedItem, +) class DNBSpider(XMLFeedSpider): @@ -219,4 +224,10 @@ def build_item(self, response): record.add_value('thesis', {'degree_type': 'PhD'}) record.add_value('collections', ['HEP', 'THESIS']) - return record.load_item() + + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/edp_spider.py b/hepcrawl/spiders/edp_spider.py index beea699d..eb6f4cf3 100644 --- a/hepcrawl/spiders/edp_spider.py +++ b/hepcrawl/spiders/edp_spider.py @@ -30,6 +30,7 @@ get_licenses, get_node, parse_domain, + ParsedItem, ) @@ -65,11 +66,11 @@ class EDPSpider(Jats, XMLFeedSpider): To run an ``EDPSpider`` using ``rich`` format:: - $ scrapy crawl EDP -a package_path=file://`pwd`/tests/responses/edp/test_rich.tar.bz2 + $ scrapy crawl EDP -a source_folder=file://`pwd`/tests/responses/edp/test_rich.tar.bz2 To run an ``EDPSpider`` using ``gz`` format:: - $ scrapy crawl EDP -a package_path=file://`pwd`/tests/responses/edp/test_gz.tar.gz + $ scrapy crawl EDP -a source_folder=file://`pwd`/tests/responses/edp/test_gz.tar.gz Todo: @@ -144,9 +145,9 @@ def start_requests(self): ftp_host, ftp_params = ftp_connection_info( self.ftp_host, self.ftp_netrc) _, new_files = ftp_list_files( - self.ftp_folder, - self.target_folder, - server=ftp_host, + server_folder=self.ftp_folder, + destination_folder=self.target_folder, + ftp_host=ftp_host, user=ftp_params['ftp_user'], password=ftp_params['ftp_password'] ) @@ -175,7 +176,7 @@ def handle_package_ftp(self, response): for xml_file in xml_files: yield Request( "file://{0}".format(xml_file), - meta={"package_path": zip_filepath} + meta={"source_folder": zip_filepath} ) def handle_package_file(self, response): @@ -188,7 +189,7 @@ def handle_package_file(self, response): for xml_file in xml_files: request = Request( "file://{0}".format(xml_file), - meta={"package_path": zip_filepath} + meta={"source_folder": zip_filepath} ) if "xml_rich" in xml_file: request.meta["rich"] = True @@ -318,7 +319,12 @@ def build_item_rich(self, response): ) record.add_value("urls", response.meta.get("urls")) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item def build_item_jats(self, response): """Build the final HEPRecord with JATS-format XML ('jp').""" @@ -388,7 +394,12 @@ def build_item_jats(self, response): references = self._get_references(node) record.add_value("references", references) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item def _get_references(self, node): """Get the references.""" diff --git a/hepcrawl/spiders/elsevier_spider.py b/hepcrawl/spiders/elsevier_spider.py index c9aacc00..b91e8372 100644 --- a/hepcrawl/spiders/elsevier_spider.py +++ b/hepcrawl/spiders/elsevier_spider.py @@ -31,6 +31,7 @@ has_numbers, range_as_string, unzip_xml_files, + ParsedItem, ) from ..dateutils import format_year @@ -180,7 +181,7 @@ def handle_package(self, response): xml_url = u"file://{0}".format(os.path.abspath(xml_file)) yield Request( xml_url, - meta={"package_path": zip_filepath, + meta={"source_folder": zip_filepath, "xml_url": xml_url}, ) @@ -1034,4 +1035,9 @@ def build_item(self, response): record.add_value('collections', self.get_collections(doctype)) record.add_value('references', self.get_references(node)) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/hindawi_spider.py b/hepcrawl/spiders/hindawi_spider.py index 941a3674..cce5e8eb 100644 --- a/hepcrawl/spiders/hindawi_spider.py +++ b/hepcrawl/spiders/hindawi_spider.py @@ -16,7 +16,10 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import get_licenses +from ..utils import ( + get_licenses, + ParsedItem, +) class HindawiSpider(XMLFeedSpider): @@ -222,4 +225,9 @@ def parse_node(self, response, node): record.add_xpath('source', "./datafield[@tag='260']/subfield[@code='b']/text()") - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/infn_spider.py b/hepcrawl/spiders/infn_spider.py index 2e970c1c..a3457a21 100644 --- a/hepcrawl/spiders/infn_spider.py +++ b/hepcrawl/spiders/infn_spider.py @@ -21,8 +21,10 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import get_temporary_file - +from ..utils import ( + get_temporary_file, + ParsedItem, +) from ..dateutils import format_date @@ -240,4 +242,9 @@ def build_item(self, response): record.add_value('source', 'INFN') record.add_value('collections', ['HEP', 'THESIS']) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/iop_spider.py b/hepcrawl/spiders/iop_spider.py index 0e3bae65..288bd205 100644 --- a/hepcrawl/spiders/iop_spider.py +++ b/hepcrawl/spiders/iop_spider.py @@ -23,6 +23,7 @@ from ..items import HEPRecord from ..loaders import HEPLoader +from ..utils import ParsedItem class IOPSpider(XMLFeedSpider, NLM): @@ -222,4 +223,9 @@ def parse_node(self, response, node): record.add_value("additional_files", self.add_fft_file(pdf_file_path, file_access, file_type)) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/magic_spider.py b/hepcrawl/spiders/magic_spider.py index 77bf7948..03d54618 100644 --- a/hepcrawl/spiders/magic_spider.py +++ b/hepcrawl/spiders/magic_spider.py @@ -18,7 +18,10 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import split_fullname +from ..utils import ( + split_fullname, + ParsedItem, +) class MagicSpider(XMLFeedSpider): @@ -176,4 +179,9 @@ def build_item(self, response): record.add_value("additional_files", response.meta.get("files")) record.add_value('collections', ['HEP', 'THESIS']) - yield record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + yield parsed_item diff --git a/hepcrawl/spiders/mit_spider.py b/hepcrawl/spiders/mit_spider.py index c71234f9..5387042d 100644 --- a/hepcrawl/spiders/mit_spider.py +++ b/hepcrawl/spiders/mit_spider.py @@ -23,7 +23,11 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import get_temporary_file, split_fullname +from ..utils import ( + get_temporary_file, + split_fullname, + ParsedItem, +) class MITSpider(XMLFeedSpider): @@ -223,4 +227,9 @@ def build_item(self, response): record.add_value('page_nr', self.get_page_nr(node)) record.add_value('collections', ['HEP', 'THESIS']) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/phenix_spider.py b/hepcrawl/spiders/phenix_spider.py index 7200664e..9eaa9da0 100644 --- a/hepcrawl/spiders/phenix_spider.py +++ b/hepcrawl/spiders/phenix_spider.py @@ -18,6 +18,7 @@ from ..items import HEPRecord from ..loaders import HEPLoader +from ..utils import ParsedItem class PhenixSpider(XMLFeedSpider): @@ -128,4 +129,9 @@ def parse_node(self, response, node): record.add_value('source', 'PHENIX') record.add_value('collections', ['HEP', 'THESIS']) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/phil_spider.py b/hepcrawl/spiders/phil_spider.py index 101b1163..d0cddaea 100644 --- a/hepcrawl/spiders/phil_spider.py +++ b/hepcrawl/spiders/phil_spider.py @@ -19,7 +19,11 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import parse_domain, get_mime_type +from ..utils import ( + parse_domain, + get_mime_type, + ParsedItem, +) class PhilSpider(CrawlSpider): @@ -160,4 +164,9 @@ def build_item(self, response): if not jsonrecord.get('year') == "forthcoming": record.add_value('journal_year', int(jsonrecord['year'])) - return record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item diff --git a/hepcrawl/spiders/pos_spider.py b/hepcrawl/spiders/pos_spider.py index 7d3fb87d..152d6688 100644 --- a/hepcrawl/spiders/pos_spider.py +++ b/hepcrawl/spiders/pos_spider.py @@ -13,13 +13,19 @@ import re +from urlparse import urljoin + from scrapy import Request, Selector from scrapy.spiders import Spider -from urlparse import urljoin -from ..utils import get_licenses, get_first + from ..dateutils import create_valid_date from ..items import HEPRecord from ..loaders import HEPLoader +from ..utils import ( + get_licenses, + get_first, + ParsedItem, +) class POSSpider(Spider): @@ -128,7 +134,13 @@ def build_item(self, response): record.add_value('extra_data', extra_data) record.add_value('collections', ['HEP', 'ConferencePaper']) - return record.load_item() + + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + return parsed_item def _get_ext_systems_number(self, node): return [ diff --git a/hepcrawl/spiders/t2k_spider.py b/hepcrawl/spiders/t2k_spider.py index 661f0bec..86076e46 100644 --- a/hepcrawl/spiders/t2k_spider.py +++ b/hepcrawl/spiders/t2k_spider.py @@ -18,7 +18,10 @@ from ..items import HEPRecord from ..loaders import HEPLoader -from ..utils import split_fullname +from ..utils import ( + split_fullname, + ParsedItem, +) class T2kSpider(XMLFeedSpider): @@ -164,4 +167,9 @@ def build_item(self, response): record.add_value("additional_files", response.meta.get("additional_files")) record.add_value('collections', ['HEP', 'THESIS']) - yield record.load_item() + parsed_item = ParsedItem( + record=record.load_item(), + record_format='hepcrawl', + ) + + yield parsed_item diff --git a/hepcrawl/spiders/wsp_spider.py b/hepcrawl/spiders/wsp_spider.py index 3f68131f..058e6cc0 100644 --- a/hepcrawl/spiders/wsp_spider.py +++ b/hepcrawl/spiders/wsp_spider.py @@ -26,6 +26,7 @@ local_list_files, get_licenses, unzip_xml_files, + ParsedItem, ) @@ -71,7 +72,15 @@ class WorldScientificSpider(Jats, XMLFeedSpider): 'rapid-communications' ] - def __init__(self, package_path=None, ftp_folder="WSP", ftp_host=None, ftp_netrc=None, *args, **kwargs): + def __init__( + self, + package_path=None, + ftp_folder="/WSP", + ftp_host=None, + ftp_netrc=None, + *args, + **kwargs + ): """Construct WSP spider.""" super(WorldScientificSpider, self).__init__(*args, **kwargs) self.ftp_folder = ftp_folder @@ -97,8 +106,8 @@ def start_requests(self): new_files_paths = ftp_list_files( self.ftp_folder, - self.target_folder, - server=ftp_host, + destination_folder=self.target_folder, + ftp_host=ftp_host, user=ftp_params['ftp_user'], password=ftp_params['ftp_password'] ) @@ -126,7 +135,7 @@ def handle_package_ftp(self, response): for xml_file in xml_files: yield Request( "file://{0}".format(xml_file), - meta={"package_path": zip_filepath} + meta={"source_folder": zip_filepath} ) def handle_package_file(self, response): @@ -138,7 +147,7 @@ def handle_package_file(self, response): for xml_file in xml_files: yield Request( "file://{0}".format(xml_file), - meta={"package_path": zip_filepath} + meta={"source_folder": zip_filepath} ) def parse_node(self, response, node): @@ -148,7 +157,7 @@ def parse_node(self, response, node): self.log("Got article_type {0}".format(article_type)) if article_type is None or article_type[0] not in self.allowed_article_types: # Filter out non-interesting article types - return None + return record = HEPLoader(item=HEPRecord(), selector=node, response=response) if article_type in ['correction', @@ -203,9 +212,13 @@ def parse_node(self, response, node): record.add_value('license', license) record.add_value('collections', self._get_collections(node, article_type, journal_title)) - parsed_record = dict(record.load_item()) - return parsed_record + parsed_item = ParsedItem( + record=dict(record.load_item()), + record_format='hepcrawl', + ) + + return parsed_item def _get_collections(self, node, article_type, current_journal_title): """Return this articles' collection.""" diff --git a/hepcrawl/testlib/celery_monitor.py b/hepcrawl/testlib/celery_monitor.py index 6c720550..1347ab22 100644 --- a/hepcrawl/testlib/celery_monitor.py +++ b/hepcrawl/testlib/celery_monitor.py @@ -9,7 +9,12 @@ """Celery monitor dealing with celery tasks for functional tests.""" -from __future__ import absolute_import, division, print_function, unicode_literals +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) from itertools import islice @@ -19,13 +24,14 @@ class CeleryMonitor(object): - def __init__(self, app, monitor_timeout=3, monitor_iter_limit=100): + def __init__(self, app, monitor_timeout=3, monitor_iter_limit=100, events_limit=2): self.results = [] self.recv = None self.app = app self.connection = None self.monitor_timeout = monitor_timeout self.monitor_iter_limit = monitor_iter_limit + self.events_limit = events_limit def __enter__(self): state = self.app.events.State() @@ -61,10 +67,16 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.connection.__exit__() def _wait_for_results(self, events_iter): - any(islice( + generator_events = islice( events_iter, # iterable self.monitor_iter_limit # stop - )) + ) + counter = 0 + for dummy in generator_events: + if dummy: + counter += 1 + if counter == self.events_limit: + break @classmethod def do_crawl( @@ -72,6 +84,7 @@ def do_crawl( app, monitor_timeout, monitor_iter_limit, + events_limit, crawler_instance, project='hepcrawl', spider='WSP', @@ -80,7 +93,12 @@ def do_crawl( ): settings = settings or {} - with cls(app, monitor_timeout=monitor_timeout, monitor_iter_limit=monitor_iter_limit) as my_monitor: + with cls( + app, + monitor_timeout=monitor_timeout, + monitor_iter_limit=monitor_iter_limit, + events_limit=events_limit + ) as my_monitor: crawler_instance.schedule( project=project, spider=spider, diff --git a/hepcrawl/testlib/fixtures.py b/hepcrawl/testlib/fixtures.py index 513b0395..73f28f96 100644 --- a/hepcrawl/testlib/fixtures.py +++ b/hepcrawl/testlib/fixtures.py @@ -11,6 +11,7 @@ import os import json +import shutil from scrapy.http import Request, TextResponse from scrapy.selector import Selector @@ -131,3 +132,18 @@ def expected_json_results_from_file(*path_chunks, **kwargs): expected_data = json.load(fd) return expected_data + + +def clean_dir(path): + """ + Deletes all contained files of given target directory path. + + Args: + path: Absolute path of target directory to be cleaned. + + Example: + + >>> clean_dir('/dir_1/dir_11/') + + """ + shutil.rmtree(path, ignore_errors=True) diff --git a/hepcrawl/tohep.py b/hepcrawl/tohep.py new file mode 100644 index 00000000..9dca5433 --- /dev/null +++ b/hepcrawl/tohep.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- +# +# This file is part of hepcrawl. +# Copyright (C) 2015, 2016, 2017 CERN. +# +# hepcrawl is a free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for +# more details. + +"""Functions used to convert records and items from one format to another. + + +Currently there are only two formats for records that we consider: + + * Hepcrawl format: internal format used by the spiders as middle step + before the pipeline, it's a generic wider format that should have at + least the same info as the HEP format used by Inspire. + + * HEP format: Inspire compatible format, it's the fromat that you get as a + result of the crawl. + +""" + +from __future__ import absolute_import, division, print_function + +import os +import datetime + +from inspire_schemas.api import LiteratureBuilder + + +class UnknownItemFormat(Exception): + pass + + +def _get_updated_fft_fields(current_fft_fields, record_files): + """ + + Args: + current_fft_fields(list(dict)): record current fft fields as generated + by ``dojson``. We expect each of then to have, at least, a key + named ``path``. + + record_files(list(RecordFile)): files attached to the record as + populated by :class:`hepcrawl.pipelines.FftFilesPipeline`. + """ + record_files_index = { + record_file.name: record_file.path + for record_file in record_files + } + new_fft_fields = [] + for fft_field in current_fft_fields: + file_name = os.path.basename(fft_field['path']) + if file_name in record_files_index: + fft_field['path'] = record_files_index[file_name] + new_fft_fields.append(fft_field) + + return new_fft_fields + + +def _has_publication_info(item): + return item.get('pubinfo_freetext') or \ + item.get('journal_volume') or \ + item.get('journal_title') or \ + item.get('journal_year') or \ + item.get('journal_issue') or \ + item.get('journal_fpage') or \ + item.get('journal_lpage') or \ + item.get('journal_artid') or \ + item.get('journal_doctype') + + +def _remove_fields(item, keys): + """Remove the given keys from the dict.""" + for key in keys: + # remove the key if there, no error if not there + item.pop(key, None) + + +def _normalize_hepcrawl_record(item, source): + if 'related_article_doi' in item: + item['dois'] += item.pop('related_article_doi', []) + + item['titles'] = [{ + 'title': item.pop('title', ''), + 'subtitle': item.pop('subtitle', ''), + 'source': source, + }] + + item['abstracts'] = [{ + 'value': item.pop('abstract', ''), + 'source': source, + }] + + item['imprints'] = [{ + 'date': item.pop('date_published', ''), + }] + + item['copyright'] = [{ + 'holder': item.pop('copyright_holder', ''), + 'year': item.pop('copyright_year', ''), + 'statement': item.pop('copyright_statement', ''), + 'material': item.pop('copyright_material', ''), + }] + + if _has_publication_info(item): + item['publication_info'] = [{ + 'journal_title': item.pop('journal_title', ''), + 'journal_volume': item.pop('journal_volume', ''), + 'journal_issue': item.pop('journal_issue', ''), + 'artid': item.pop('journal_artid', ''), + 'page_start': item.pop('journal_fpage', ''), + 'page_end': item.pop('journal_lpage', ''), + 'note': item.pop('journal_doctype', ''), + 'pubinfo_freetext': item.pop('pubinfo_freetext', ''), + 'pubinfo_material': item.pop('pubinfo_material', ''), + }] + if item.get('journal_year'): + item['publication_info'][0]['year'] = int( + item.pop('journal_year') + ) + + _remove_fields( + item, + [ + 'journal_title', + 'journal_volume', + 'journal_year', + 'journal_issue', + 'journal_fpage', + 'journal_lpage', + 'journal_doctype', + 'journal_artid', + 'pubinfo_freetext', + 'pubinfo_material', + ] + ) + + return item + + +def _generate_acquisition_source(source): + acquisition_source = { + 'source': source, + 'method': 'hepcrawl', + 'datetime': datetime.datetime.now().isoformat(), + 'submission_number': os.environ.get('SCRAPY_JOB', ''), + } + return acquisition_source + + +def item_to_hep( + item, + source, +): + """Get an output ready hep formatted record from the given + :class:`hepcrawl.utils.ParsedItem`, whatever format it's record might be. + + Args: + item(hepcrawl.utils.ParsedItem): item to convert. + source(str): string identifying the source for this item (ex. 'arXiv'). + + Returns: + hepcrawl.utils.ParsedItem: the new item, with the internal record + formated as hep record. + + Raises: + UnknownItemFormat: if the source item format is unknown. + """ + item.record['acquisition_source'] = _generate_acquisition_source( + source=source + ) + + if item.record_format == 'hep': + return hep_to_hep( + hep_record=item.record, + record_files=item.record_files, + ) + elif item.record_format == 'hepcrawl': + record = _normalize_hepcrawl_record( + item=item.record, + source=source, + ) + return hepcrawl_to_hep(dict(record)) + else: + raise UnknownItemFormat( + 'Unknown ParsedItem::{}'.format(item.record_format) + ) + + +def hep_to_hep(hep_record, record_files): + """This is needed to be able to patch the ``_fft`` field in the record. + + As earlier in the process we don't really have all the files yet. It should + be used by any spiders that generate hep format instead of the internal + hepcrawl one (normally, marc-ingesting spiders). + """ + if record_files: + hep_record['_fft'] = _get_updated_fft_fields( + current_fft_fields=hep_record['_fft'], + record_files=record_files, + ) + + return hep_record + + +def hepcrawl_to_hep(crawler_record): + """ + Args: + crawler_record(dict): dictionary representing the hepcrawl formatted + record. + + + Returns: + dict: The hep formatted (and validated) record. + + Raises: + Exception: if there was a validation error (the exact class depends on + :class:`inspire_schemas.api.validate`). + """ + + def _filter_affiliation(affiliations): + return [ + affilation.get('value') + for affilation in affiliations + if affilation.get('value') + ] + + builder = LiteratureBuilder( + source=crawler_record['acquisition_source']['source'] + ) + + for author in crawler_record.get('authors', []): + builder.add_author(builder.make_author( + author['full_name'], + affiliations=_filter_affiliation(author['affiliations']), + )) + + for title in crawler_record.get('titles', []): + builder.add_title( + title=title.get('title'), + source=title.get('source') + ) + + for abstract in crawler_record.get('abstracts', []): + builder.add_abstract( + abstract=abstract.get('value'), + source=abstract.get('source') + ) + + for arxiv_eprint in crawler_record.get('arxiv_eprints', []): + builder.add_arxiv_eprint( + arxiv_id=arxiv_eprint.get('value'), + arxiv_categories=arxiv_eprint.get('categories') + ) + + for doi in crawler_record.get('dois', []): + builder.add_doi( + doi=doi.get('value'), + material=doi.get('material'), + ) + + for public_note in crawler_record.get('public_notes', []): + builder.add_public_note( + public_note=public_note.get('value'), + source=public_note.get('source') + ) + + for license in crawler_record.get('license', []): + builder.add_license( + url=license.get('url'), + license=license.get('license'), + material=license.get('material'), + ) + + for collaboration in crawler_record.get('collaborations', []): + builder.add_collaboration( + collaboration=collaboration.get('value') + ) + + for imprint in crawler_record.get('imprints', []): + builder.add_imprint_date( + imprint_date=imprint.get('date') + ) + + for copyright in crawler_record.get('copyright', []): + builder.add_copyright( + holder=copyright.get('holder'), + material=copyright.get('material'), + statement=copyright.get('statement') + ) + + builder.add_preprint_date( + preprint_date=crawler_record.get('preprint_date') + ) + + acquisition_source = crawler_record.get('acquisition_source', {}) + builder.add_acquisition_source( + method=acquisition_source['method'], + date=acquisition_source['datetime'], + source=acquisition_source['source'], + submission_number=acquisition_source['submission_number'], + ) + + try: + builder.add_number_of_pages( + number_of_pages=int(crawler_record.get('page_nr', [])[0]) + ) + except (TypeError, ValueError, IndexError): + pass + + publication_types = [ + 'introductory', + 'lectures', + 'review', + ] + + special_collections = [ + 'cdf-internal-note', + 'cdf-note', + 'cds', + 'd0-internal-note', + 'd0-preliminary-note', + 'h1-internal-note', + 'h1-preliminary-note', + 'halhidden', + 'hephidden', + 'hermes-internal-note', + 'larsoft-internal-note', + 'larsoft-note', + 'zeus-internal-note', + 'zeus-preliminary-note', + ] + + document_types = [ + 'book', + 'note', + 'report', + 'proceedings', + 'thesis', + ] + + added_doc_type = False + + for collection in crawler_record.get('collections', []): + collection = collection['primary'].strip().lower() + + if collection == 'arxiv': + continue # ignored + elif collection == 'citeable': + builder.set_citeable(True) + elif collection == 'core': + builder.set_core(True) + elif collection == 'noncore': + builder.set_core(False) + elif collection == 'published': + builder.set_refereed(True) + elif collection == 'withdrawn': + builder.set_withdrawn(True) + elif collection in publication_types: + builder.add_publication_type(collection) + elif collection in special_collections: + builder.add_special_collection(collection.upper()) + elif collection == 'bookchapter': + added_doc_type = True + builder.add_document_type('book chapter') + elif collection == 'conferencepaper': + added_doc_type = True + builder.add_document_type('conference paper') + elif collection in document_types: + added_doc_type = True + builder.add_document_type(collection) + + if not added_doc_type: + builder.add_document_type('article') + + _pub_info = crawler_record.get('publication_info', [{}])[0] + builder.add_publication_info( + year=_pub_info.get('year'), + artid=_pub_info.get('artid'), + page_end=_pub_info.get('page_end'), + page_start=_pub_info.get('page_start'), + journal_issue=_pub_info.get('journal_issue'), + journal_title=_pub_info.get('journal_title'), + journal_volume=_pub_info.get('journal_volume'), + pubinfo_freetext=_pub_info.get('pubinfo_freetext'), + material=_pub_info.get('pubinfo_material'), + ) + + for report_number in crawler_record.get('report_numbers', []): + builder.add_report_number( + report_number=report_number.get('value'), + source=report_number.get('source') + ) + + builder.validate_record() + + return builder.record diff --git a/hepcrawl/utils.py b/hepcrawl/utils.py index 4ad9db3c..256dd508 100644 --- a/hepcrawl/utils.py +++ b/hepcrawl/utils.py @@ -27,10 +27,17 @@ from .mappings import LICENSES, LICENSE_TEXTS -RE_FOR_THE = re.compile(r'\b(?:for|on behalf of|representing)\b', re.IGNORECASE) +RE_FOR_THE = re.compile( + r'\b(?:for|on behalf of|representing)\b', + re.IGNORECASE, +) INST_PHRASES = ['for the development', ] +class PathDoesNotExist(IOError): + pass + + def unzip_xml_files(filename, target_folder): """Unzip files (XML only) into target folder.""" z = ZipFile(filename) @@ -57,17 +64,61 @@ def ftp_connection_info(ftp_host, netrc_file, passive_mode=False): return ftp_host, connection_params -def ftp_list_files(server_folder, target_folder, server, user, password, passive_mode=False): - """List files from given FTP's server folder to target folder.""" +def ftp_list_files( + server_folder, + ftp_host, + user, + password, + destination_folder=None, + passive_mode=False, + only_missing_files=True, +): + """ + + Args: + server_folder(str): remote folder to list. + + ftp_host(str): name of the host. Example: 'ftp.cern.ch' + + user(str): For authentication. + + password(str): For authentication. + + destination_folder(str): local folder to compare with. + + passive_mode(bool): True if it should use firewall friendly ftp passive + mode. + + only_missing_files(bool): If True will only list the files that are not + already in the ``destination_folder``. + """ session_factory = ftputil.session.session_factory( base_class=ftplib.FTP, port=21, use_passive_mode=passive_mode, - encrypt_data_channel=True) - - with ftputil.FTPHost(server, user, password, session_factory=session_factory) as host: - file_names = host.listdir(os.path.join(host.curdir, '/', server_folder)) - return list_missing_files(server_folder, target_folder, file_names) + encrypt_data_channel=True, + ) + + with ftputil.FTPHost( + ftp_host, + user, + password, + session_factory=session_factory, + ) as host: + file_names = host.listdir(os.path.join(host.curdir, server_folder)) + if only_missing_files: + return list_missing_files( + server_folder, + destination_folder, + file_names, + ) + else: + return [ + os.path.join( + server_folder, + file_name + ) for file_name in file_names + ] def local_list_files(local_folder, target_folder): @@ -106,7 +157,8 @@ def split_fullname(author, switch_name_order=False): It accepts author strings with and without comma separation. As default surname is first in case of comma separation, otherwise last. - Multi-part surnames are incorrectly detected in strings without comma separation. + Multi-part surnames are incorrectly detected in strings without comma + separation. """ if not author: return "", "" @@ -166,7 +218,8 @@ def build_dict(seq, key): """ Creates a dictionary from a list, using the specified key. - Used to make searching in a list of objects faster (get operations are O(1)). + Used to make searching in a list of objects faster (get operations are + O(1)). """ return dict((d[key], dict(d, index=i)) for (i, d) in enumerate(seq)) @@ -200,7 +253,10 @@ def range_as_string(data): """ data = [int(i) for i in data] ranges = [] - for key, group in groupby(enumerate(data), lambda (index, item): index - item): + for key, group in groupby( + enumerate(data), + lambda (index, item): index - item + ): group = map(itemgetter(1), group) if len(group) > 1: rangestring = "{}-{}".format(str(group[0]), str(group[-1])) @@ -254,7 +310,12 @@ def get_journal_and_section(publication): if split_pub[-1] in possible_sections: section = split_pub.pop(-1) journal_title = "".join( - [word for word in split_pub if "section" not in word.lower()]).strip(", ") + [ + word + for word in split_pub + if "section" not in word.lower() + ] + ).strip(", ") except IndexError: pass @@ -270,8 +331,10 @@ def get_licenses( Args: license_url(str): Url of the license to generate. - license_text(str): Text with the description of the license (sometimes is - all we got...). + + license_text(str): Text with the description of the license (sometimes + is all we got...). + license_material(str): Material of the license. Returns: @@ -321,3 +384,86 @@ def get_license_by_text(license_text): license = get_license_by_url(license_url=LICENSE_TEXTS[key]) return license + + +class RecordFile(object): + """Metadata of a file needed for a record. + + Args: + path(str): local path to the file. + + name(str): Optional, name of the file, if not passed, will use the name + in the ``path``. + + Rises: + PathDoesNotExist: + """ + def __init__(self, path, name=None): + self.path = path + if not os.path.exists(self.path): + raise PathDoesNotExist( + "The given record file path '%s' does not exist." % self.path + ) + + if name is None: + name = os.path.basename(path) + + self.name = name + + +class ParsedItem(dict): + """Each of the individual items returned by the spider to the pipeline. + + Args: + record(dict): Information about the crawled record, might be in + different formats. + + record_format(str): Format of the above record, for example ``"hep"`` + or ``"hepcrawl"``. + + file_urls(list(str)): URLs to the files to be downloaded by + ``FftFilesPipeline``. + + ftp_params(dict): Parameter for the + :class:`hepcrawl.pipelines.FftFilesPipeline` to be able to connect + to the ftp server, if any. + + record_files(list(RecordFile)): files attached to the record, usually + populated by :class:`hepcrawl.pipelines.FftFilesPipeline` from the + ``file_urls`` parameter. + + Attributes: + *: this class bypasses the regular dict ``__getattr__`` allowing to + access any of it's elements as attributes. + """ + def __init__( + self, + record, + record_format, + file_urls=None, + ftp_params=None, + record_files=None, + **kwargs + ): + super(ParsedItem, self).__init__( + record=record, + record_format=record_format, + file_urls=file_urls, + ftp_params=ftp_params, + record_files=record_files, + **kwargs + ) + + def __getattr__(self, key): + if key not in self: + raise AttributeError( + "'%s' object has no attribute '%s'" % ( + self.__class__.__name__, + key, + ) + ) + + return self[key] + + def __setattr__(self, key, value): + self[key] = value diff --git a/setup.py b/setup.py index b19e5f14..a98aeb88 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ install_requires = [ 'autosemver~=0.2', 'inspire-schemas~=42.0', + 'inspire-dojson~=41.0', 'Scrapy>=1.1.0', # TODO: unpin once they support wheel building again 'scrapyd==1.1.0', diff --git a/tests/functional/arxiv/test_arxiv.py b/tests/functional/arxiv/test_arxiv.py index a9677b89..0f58b17d 100644 --- a/tests/functional/arxiv/test_arxiv.py +++ b/tests/functional/arxiv/test_arxiv.py @@ -72,6 +72,7 @@ def test_arxiv(set_up_local_environment, expected_results): app=celery_app, monitor_timeout=5, monitor_iter_limit=100, + events_limit=1, crawler_instance=crawler, project=set_up_local_environment.get('CRAWLER_PROJECT'), spider='arXiv', diff --git a/tests/functional/desy/fixtures/desy_ftp_records.json b/tests/functional/desy/fixtures/desy_ftp_records.json new file mode 100644 index 00000000..8f7302c4 --- /dev/null +++ b/tests/functional/desy/fixtures/desy_ftp_records.json @@ -0,0 +1,308 @@ +[{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/809d9d2bebcea6eee5e400e3c49b31795a3acc3d.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/1d4b0a4eebdd03b95f882fa7feb9d3f06681ec50.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 111111, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/111111" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}, +{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/809d9d2bebcea6eee5e400e3c49b31795a3acc3d.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/1d4b0a4eebdd03b95f882fa7feb9d3f06681ec50.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 222222, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/222222" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}, +{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/809d9d2bebcea6eee5e400e3c49b31795a3acc3d.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/1d4b0a4eebdd03b95f882fa7feb9d3f06681ec50.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 333333, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/333333" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}, +{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/809d9d2bebcea6eee5e400e3c49b31795a3acc3d.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/1d4b0a4eebdd03b95f882fa7feb9d3f06681ec50.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 444444, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/444444" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}] diff --git a/tests/functional/desy/fixtures/desy_local_records.json b/tests/functional/desy/fixtures/desy_local_records.json new file mode 100644 index 00000000..02f8fad2 --- /dev/null +++ b/tests/functional/desy/fixtures/desy_local_records.json @@ -0,0 +1,308 @@ +[{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 111111, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/111111" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}, +{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 222222, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/222222" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}, +{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 333333, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/333333" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}, +{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-04-03T10:26:40.365216" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.", + "format": ".txt", + "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt", + "type": "Main", + "filename": "cNFW_rogue_curves" + }, + { + "version": 1, + "creation_datetime": "2017-06-27T09:43:16", + "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.", + "format": ".txt", + "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt", + "type": "Main", + "filename": "scalingRelations_DutBeh_DC14_all_Oh" + } + ], + "control_number": 444444, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/444444" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": "Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser\n Acceleration (DLA) From the Source to Relativistic Electrons\n " + } + ], + "urls": [ + { + "description": "Fulltext", + "value": "http://inspirehep.net/record/1608652/files/Towards a fully\n integrated acc on a chip.pdf\n " + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": "Dielectric laser acceleration of electrons has recently been\n demonstrated with significantly higher accelerating gradients than other\n structure-based linear accelerators. Towards the development of an integrated 1 MeV\n electron accelerator based on dielectric laser accelerator technologies,\n development in several relevant technologies is needed. In this work, recent\n developments on electron sources, bunching, accelerating, focussing, deflecting and\n laser coupling structures are reported. With an eye to the near future, components\n required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond\n electron bunches are outlined.\n " + } + ] +}] diff --git a/tests/functional/desy/fixtures/ftp_server/.netrc b/tests/functional/desy/fixtures/ftp_server/.netrc new file mode 100644 index 00000000..59a152f7 --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/.netrc @@ -0,0 +1,3 @@ +machine ftp_server +login bob +password bob diff --git a/tests/functional/desy/fixtures/ftp_server/DESY/FFT/test_fft_1.txt b/tests/functional/desy/fixtures/ftp_server/DESY/FFT/test_fft_1.txt new file mode 100644 index 00000000..bb8e8348 --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/DESY/FFT/test_fft_1.txt @@ -0,0 +1 @@ +sample file fft 1. \ No newline at end of file diff --git a/tests/functional/desy/fixtures/ftp_server/DESY/FFT/test_fft_2.txt b/tests/functional/desy/fixtures/ftp_server/DESY/FFT/test_fft_2.txt new file mode 100644 index 00000000..e1b54448 --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/DESY/FFT/test_fft_2.txt @@ -0,0 +1 @@ +sample file fft 2. \ No newline at end of file diff --git a/tests/functional/desy/fixtures/ftp_server/DESY/desy_collection_records.xml b/tests/functional/desy/fixtures/ftp_server/DESY/desy_collection_records.xml new file mode 100644 index 00000000..359bb570 --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/DESY/desy_collection_records.xml @@ -0,0 +1,149 @@ + + + + 111111 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + + 222222 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + diff --git a/tests/functional/desy/fixtures/ftp_server/DESY/desy_no_namespace_collection_records.xml b/tests/functional/desy/fixtures/ftp_server/DESY/desy_no_namespace_collection_records.xml new file mode 100644 index 00000000..1f9c57a9 --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/DESY/desy_no_namespace_collection_records.xml @@ -0,0 +1,149 @@ + + + + 333333 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + + 444444 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + diff --git a/tests/functional/desy/fixtures/ftp_server/DESY/file_not_for_download.txt b/tests/functional/desy/fixtures/ftp_server/DESY/file_not_for_download.txt new file mode 100644 index 00000000..5254be38 --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/DESY/file_not_for_download.txt @@ -0,0 +1 @@ +This is a file not to download the Desy spider! \ No newline at end of file diff --git a/tests/functional/desy/fixtures/ftp_server/pureftpd.passwd b/tests/functional/desy/fixtures/ftp_server/pureftpd.passwd new file mode 100644 index 00000000..275a727c --- /dev/null +++ b/tests/functional/desy/fixtures/ftp_server/pureftpd.passwd @@ -0,0 +1 @@ +bob:$1$3ccy4I60$nSpFtRN8U6/BgmmPaxrYR/:1000:1000::/home/ftpusers/bob/./:::::::::::: diff --git a/tests/functional/desy/test_desy.py b/tests/functional/desy/test_desy.py new file mode 100644 index 00000000..464907de --- /dev/null +++ b/tests/functional/desy/test_desy.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +# +# This file is part of hepcrawl. +# Copyright (C) 2017 CERN. +# +# hepcrawl is a free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for +# more details. + +"""Functional tests for Desy spider""" + +from __future__ import absolute_import, division, print_function + +import pytest + +from time import sleep +import hashlib + +from hepcrawl.testlib.celery_monitor import CeleryMonitor +from hepcrawl.testlib.fixtures import ( + get_test_suite_path, + expected_json_results_from_file, + clean_dir, +) +from hepcrawl.testlib.tasks import app as celery_app +from hepcrawl.testlib.utils import get_crawler_instance + + +def override_generated_fields(record): + record['acquisition_source']['datetime'] = u'2017-04-03T10:26:40.365216' + record['acquisition_source']['submission_number'] = ( + u'5652c7f6190f11e79e8000224dabeaad' + ) + + return record + + +def assert_files_equal(file_1, file_2): + """Compares two files calculating the md5 hash.""" + def _generate_md5_hash(file_path): + hasher = hashlib.md5() + with open(str(file_path), 'rb') as fd: + buf = fd.read() + hasher.update(buf) + return hasher.hexdigest() + + file_1_hash = _generate_md5_hash(file_1) + file_2_hash = _generate_md5_hash(file_2) + assert file_1_hash == file_2_hash + + +@pytest.fixture(scope="function") +def fft_1_path(): + return get_test_suite_path( + 'desy', + 'fixtures', + 'ftp_server', + 'DESY', + 'FFT', + 'test_fft_1.txt', + test_suite='functional', + ) + + +@pytest.fixture(scope="function") +def fft_2_path(): + return get_test_suite_path( + 'desy', + 'fixtures', + 'ftp_server', + 'DESY', + 'FFT', + 'test_fft_2.txt', + test_suite='functional', + ) + + +@pytest.fixture(scope="function") +def ftp_environment(): + netrc_location = get_test_suite_path( + 'desy', + 'fixtures', + 'ftp_server', + '.netrc', + test_suite='functional', + ) + + # The test must wait until the docker environment is up (takes about 10 + # seconds). + sleep(10) + + yield { + 'CRAWLER_HOST_URL': 'http://scrapyd:6800', + 'CRAWLER_PROJECT': 'hepcrawl', + 'CRAWLER_ARGUMENTS': { + 'ftp_host': 'ftp_server', + 'ftp_netrc': netrc_location, + } + } + + clean_dir('/tmp/file_urls') + clean_dir('/tmp/DESY') + + +@pytest.fixture(scope="function") +def local_environment(): + package_location = get_test_suite_path( + 'desy', + 'fixtures', + 'ftp_server', + 'DESY', + test_suite='functional', + ) + + yield { + 'CRAWLER_HOST_URL': 'http://scrapyd:6800', + 'CRAWLER_PROJECT': 'hepcrawl', + 'CRAWLER_ARGUMENTS': { + 'source_folder': package_location, + } + } + + clean_dir('/tmp/file_urls') + clean_dir('/tmp/DESY') + + +@pytest.mark.parametrize( + 'expected_results', + [ + expected_json_results_from_file( + 'desy', + 'fixtures', + 'desy_ftp_records.json', + ), + ], + ids=[ + 'smoke', + ] +) +def test_desy_ftp( + ftp_environment, + expected_results, + fft_1_path, + fft_2_path, +): + crawler = get_crawler_instance( + ftp_environment.get('CRAWLER_HOST_URL') + ) + + results = CeleryMonitor.do_crawl( + app=celery_app, + monitor_timeout=5, + monitor_iter_limit=100, + events_limit=2, + crawler_instance=crawler, + project=ftp_environment.get('CRAWLER_PROJECT'), + spider='desy', + settings={}, + **ftp_environment.get('CRAWLER_ARGUMENTS') + ) + + gotten_results = [override_generated_fields(result) for result in results] + expected_results = [ + override_generated_fields(expected) + for expected in expected_results + ] + + assert sorted(gotten_results) == expected_results + + # Check using MD5 Hash if downloaded files are there. + for record in expected_results: + fft_file_paths = sorted(record['_fft']) + + assert_files_equal(fft_file_paths[0]['path'], fft_2_path) + assert_files_equal(fft_file_paths[1]['path'], fft_1_path) + + +@pytest.mark.parametrize( + 'expected_results', + [ + expected_json_results_from_file( + 'desy', + 'fixtures', + 'desy_local_records.json', + ), + ], + ids=[ + 'smoke', + ] +) +def test_desy_local_package_path( + local_environment, + expected_results, + fft_1_path, + fft_2_path, +): + crawler = get_crawler_instance(local_environment.get('CRAWLER_HOST_URL')) + + results = CeleryMonitor.do_crawl( + app=celery_app, + monitor_timeout=5, + monitor_iter_limit=100, + events_limit=2, + crawler_instance=crawler, + project=local_environment.get('CRAWLER_PROJECT'), + spider='desy', + settings={}, + **local_environment.get('CRAWLER_ARGUMENTS') + ) + + gotten_results = [override_generated_fields(result) for result in results] + expected_results = [ + override_generated_fields(expected) + for expected in expected_results + ] + + assert sorted(gotten_results) == expected_results + + # Check using MD5 Hash if downloaded files are there. + for record in expected_results: + fft_file_paths = sorted(record['_fft']) + + assert_files_equal(fft_file_paths[0]['path'], fft_2_path) + assert_files_equal(fft_file_paths[1]['path'], fft_1_path) diff --git a/tests/functional/wsp/test_wsp.py b/tests/functional/wsp/test_wsp.py index 70996466..42f691c9 100644 --- a/tests/functional/wsp/test_wsp.py +++ b/tests/functional/wsp/test_wsp.py @@ -13,7 +13,6 @@ import pytest import os -import shutil from time import sleep @@ -21,6 +20,7 @@ from hepcrawl.testlib.fixtures import ( get_test_suite_path, expected_json_results_from_file, + clean_dir, ) from hepcrawl.testlib.tasks import app as celery_app from hepcrawl.testlib.utils import get_crawler_instance @@ -55,7 +55,7 @@ def set_up_ftp_environment(): } } - clean_dir() + clean_dir(path='/tmp/WSP/') @pytest.fixture(scope="function") @@ -80,7 +80,7 @@ def set_up_local_environment(): def remove_generated_files(package_location): - clean_dir() + clean_dir(path='/tmp/WSP/') _, dirs, files = next(os.walk(package_location)) for dir_name in dirs: @@ -90,10 +90,6 @@ def remove_generated_files(package_location): os.unlink(os.path.join(package_location, file_name)) -def clean_dir(path='/tmp/WSP/'): - shutil.rmtree(path, ignore_errors=True) - - @pytest.mark.parametrize( 'expected_results', [ @@ -114,6 +110,7 @@ def test_wsp_ftp(set_up_ftp_environment, expected_results): app=celery_app, monitor_timeout=5, monitor_iter_limit=100, + events_limit=1, crawler_instance=crawler, project=set_up_ftp_environment.get('CRAWLER_PROJECT'), spider='WSP', @@ -147,6 +144,7 @@ def test_wsp_local_package_path(set_up_local_environment, expected_results): app=celery_app, monitor_timeout=5, monitor_iter_limit=100, + events_limit=1, crawler_instance=crawler, project=set_up_local_environment.get('CRAWLER_PROJECT'), spider='WSP', diff --git a/tests/unit/responses/desy/desy_collection_records.xml b/tests/unit/responses/desy/desy_collection_records.xml new file mode 100644 index 00000000..93ede820 --- /dev/null +++ b/tests/unit/responses/desy/desy_collection_records.xml @@ -0,0 +1,149 @@ + + + + 111111 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + + 222222 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + diff --git a/tests/unit/responses/desy/desy_record.xml b/tests/unit/responses/desy/desy_record.xml new file mode 100644 index 00000000..9e20e8d0 --- /dev/null +++ b/tests/unit/responses/desy/desy_record.xml @@ -0,0 +1,76 @@ + + + + 111111 + 20170705125610.0 + + DOI + 10.18429/JACoW-IPAC2017-WEYB1 + + + 9783954501823 + + + Towards a Fully Integrated Accelerator on a Chip: Dielectric Laser + Acceleration (DLA) From the Source to Relativistic Electrons + + JACoW + + + Dielectric laser acceleration of electrons has recently been + demonstrated with significantly higher accelerating gradients than other + structure-based linear accelerators. Towards the development of an integrated 1 MeV + electron accelerator based on dielectric laser accelerator technologies, + development in several relevant technologies is needed. In this work, recent + developments on electron sources, bunching, accelerating, focussing, deflecting and + laser coupling structures are reported. With an eye to the near future, components + required for a 1 MeV kinetic energy tabletop accelerator producing sub-femtosecond + electron bunches are outlined. + + Deutsches Elektronen-Synchrotron + + + 6 + + + *Brief entry* + + + 2017 + 2520-2525 + + + 100176 + http://inspirehep.net/record/1608652/files/Towards a fully + integrated acc on a chip.pdf + + Fulltext + + + oai:inspirehep.net:1608652 + INSPIRE:HEP + + + FFT/test_fft_1.txt + 00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}. + .txt + cNFW_rogue_curves + + 2017-06-27 09:43:17 + Main + 1 + + + + FFT/test_fft_2.txt + 00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines. + .txt + scalingRelations_DutBeh_DC14_all_Oh + + 2017-06-27 09:43:16 + Main + 1 + + + + diff --git a/tests/unit/responses/crawler2hep/in_generic_crawler_record.yaml b/tests/unit/responses/tohep/in_generic_crawler_record.yaml similarity index 98% rename from tests/unit/responses/crawler2hep/in_generic_crawler_record.yaml rename to tests/unit/responses/tohep/in_generic_crawler_record.yaml index 4e80ba6b..1ade2b4b 100644 --- a/tests/unit/responses/crawler2hep/in_generic_crawler_record.yaml +++ b/tests/unit/responses/tohep/in_generic_crawler_record.yaml @@ -3,7 +3,7 @@ "11" ], "acquisition_source": { - "date": "2017-02-21T18:03:40.858985", + "datetime": "2017-02-21T18:03:40.858985", "source": "arXiv", "method": "hepcrawl", "submission_number": "scrapy_job" diff --git a/tests/unit/responses/crawler2hep/in_no_document_type.yaml b/tests/unit/responses/tohep/in_no_document_type.yaml similarity index 98% rename from tests/unit/responses/crawler2hep/in_no_document_type.yaml rename to tests/unit/responses/tohep/in_no_document_type.yaml index 22b93fd0..21543c36 100644 --- a/tests/unit/responses/crawler2hep/in_no_document_type.yaml +++ b/tests/unit/responses/tohep/in_no_document_type.yaml @@ -5,7 +5,7 @@ "11" ], "acquisition_source": { - "date": "2017-02-21T18:03:40.858985", + "datetime": "2017-02-21T18:03:40.858985", "source": "arXiv", "method": "hepcrawl", "submission_number": "scrapy_job" diff --git a/tests/unit/responses/crawler2hep/out_generic_crawler_record.yaml b/tests/unit/responses/tohep/out_generic_crawler_record.yaml similarity index 100% rename from tests/unit/responses/crawler2hep/out_generic_crawler_record.yaml rename to tests/unit/responses/tohep/out_generic_crawler_record.yaml diff --git a/tests/unit/responses/crawler2hep/out_no_document_type.yaml b/tests/unit/responses/tohep/out_no_document_type.yaml similarity index 100% rename from tests/unit/responses/crawler2hep/out_no_document_type.yaml rename to tests/unit/responses/tohep/out_no_document_type.yaml diff --git a/tests/unit/test_alpha.py b/tests/unit/test_alpha.py index eef140b1..96bf9af1 100644 --- a/tests/unit/test_alpha.py +++ b/tests/unit/test_alpha.py @@ -20,13 +20,15 @@ def results(): """Return results generator from the Alpha spider.""" spider = alpha_spider.AlphaSpider() - records = list( + parsed_items = list( spider.parse( fake_response_from_file('alpha/test_1.htm') ) ) + records = [parsed_item.record for parsed_item in parsed_items] assert records + return records diff --git a/tests/unit/test_aps.py b/tests/unit/test_aps.py index eb53269d..3bb3698c 100644 --- a/tests/unit/test_aps.py +++ b/tests/unit/test_aps.py @@ -21,7 +21,7 @@ def results(): from scrapy.http import TextResponse spider = aps_spider.APSSpider() - records = list( + parsed_items = list( spider.parse( fake_response_from_file( 'aps/aps_single_response.json', @@ -30,6 +30,8 @@ def results(): ) ) + records = [parsed_item.record for parsed_item in parsed_items] + assert records return records diff --git a/tests/unit/test_arxiv_all.py b/tests/unit/test_arxiv_all.py index bd75e5a4..1f4155c9 100644 --- a/tests/unit/test_arxiv_all.py +++ b/tests/unit/test_arxiv_all.py @@ -11,7 +11,8 @@ import pytest -from scrapy.crawler import Crawler +from scrapy.crawler import Crawler +from scrapy.http import TextResponse from hepcrawl.pipelines import InspireCeleryPushPipeline from hepcrawl.spiders import arxiv_spider @@ -25,36 +26,16 @@ def spider(): return spider -@pytest.fixture -def one_result(spider): - """Return results generator from the arxiv spider. Tricky fields, one - record. - """ - from scrapy.http import TextResponse - - records = list( - spider.parse( - fake_response_from_file( - 'arxiv/sample_arxiv_record0.xml', - response_type=TextResponse, - ) - ) - ) - - assert records - pipeline = InspireCeleryPushPipeline() - pipeline.open_spider(spider) - return [pipeline.process_item(record, spider) for record in records] - - @pytest.fixture def many_results(spider): """Return results generator from the arxiv spider. Tricky fields, many records. """ - from scrapy.http import TextResponse + def _get_processed_record(item, spider): + record = pipeline.process_item(item, spider) + return record - records = list( + parsed_items = list( spider.parse( fake_response_from_file( 'arxiv/sample_arxiv_record.xml', @@ -63,10 +44,10 @@ def many_results(spider): ) ) - assert records pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - return [pipeline.process_item(record, spider) for record in records] + + return [_get_processed_record(parsed_item, spider) for parsed_item in parsed_items] def test_page_nr(many_results): diff --git a/tests/unit/test_arxiv_single.py b/tests/unit/test_arxiv_single.py index a6ed66d6..329a2a49 100644 --- a/tests/unit/test_arxiv_single.py +++ b/tests/unit/test_arxiv_single.py @@ -24,10 +24,15 @@ def results(): """Return results generator from the arxiv spider. All fields, one record. """ + def _get_processed_item(item, spider): + record = pipeline.process_item(item, spider) + validate(record, 'hep') + assert record + return record crawler = Crawler(spidercls=arxiv_spider.ArxivSpider) spider = arxiv_spider.ArxivSpider.from_crawler(crawler) - records = list( + parsed_items = list( spider.parse( fake_response_from_file( 'arxiv/sample_arxiv_record0.xml', @@ -36,16 +41,10 @@ def results(): ) ) - assert records pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - processed_records = [] - for record in records: - processed_record = pipeline.process_item(record, spider) - validate(processed_record, 'hep') - processed_records.append(processed_record) - return processed_records + return [_get_processed_item(parsed_item, spider) for parsed_item in parsed_items] diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py index cc6ef093..b8ec5b8a 100644 --- a/tests/unit/test_base.py +++ b/tests/unit/test_base.py @@ -38,9 +38,12 @@ def record(): nodes = selector.xpath('.//%s' % spider.itertag) response.meta["record"] = nodes[0].extract() response.meta["urls"] = ["http://hdl.handle.net/1885/10005"] - parsed_record = spider.build_item(response) - assert parsed_record - return parsed_record + + parsed_item = spider.build_item(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture @@ -169,7 +172,12 @@ def splash(): 'Content-Type': 'text/html', }, ) - return spider.scrape_for_pdf(splash_response) + + parsed_item = spider.scrape_for_pdf(splash_response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_splash(splash): @@ -201,7 +209,12 @@ def parsed_node(): response = fake_response_from_string(text=body) node = get_node(spider, 'OAI-PMH:record', text=body) response.meta["record"] = node[0].extract() - return spider.parse_node(response, node[0]) + + parsed_item = spider.parse_node(response, node[0]) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_parsed_node(parsed_node): diff --git a/tests/unit/test_brown.py b/tests/unit/test_brown.py index 0b42b4df..8d0f20de 100644 --- a/tests/unit/test_brown.py +++ b/tests/unit/test_brown.py @@ -41,10 +41,12 @@ def record(): splash_response = fake_response_from_file('brown/test_splash.html') splash_response.meta["jsonrecord"] = jsonrecord - parsed_record = spider.scrape_splash(splash_response) - assert parsed_record - return parsed_record + parsed_item = spider.scrape_splash(splash_response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture @@ -200,7 +202,11 @@ def parsed_node_no_splash(): jsonrecord = jsonresponse["items"]["docs"][0] response.meta["jsonrecord"] = jsonrecord - return spider.parse(response).next() + parsed_item = spider.parse(response).next() + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_no_splash(parsed_node_no_splash): diff --git a/tests/unit/test_desy.py b/tests/unit/test_desy.py new file mode 100644 index 00000000..2b567191 --- /dev/null +++ b/tests/unit/test_desy.py @@ -0,0 +1,482 @@ +# -*- coding: utf-8 -*- +# +# This file is part of hepcrawl. +# Copyright (C) 2017 CERN. +# +# hepcrawl is a free software; you can redistribute it and/or modify it +# under the terms of the Revised BSD License; see LICENSE file for +# more details. + +from __future__ import absolute_import, division, print_function + +import os + +import pytest +from scrapy.crawler import Crawler +from scrapy.http import TextResponse +from scrapy.settings import Settings + +from hepcrawl import settings +from hepcrawl.pipelines import InspireCeleryPushPipeline +from hepcrawl.spiders import desy_spider +from hepcrawl.testlib.fixtures import fake_response_from_file + + +def create_spider(): + custom_settings = Settings() + custom_settings.setmodule(settings) + crawler = Crawler( + spidercls=desy_spider.DesySpider, + settings=custom_settings, + ) + return desy_spider.DesySpider.from_crawler( + crawler, + source_folder='idontexist_but_it_does_not_matter', + ) + + +def get_records(response_file_name): + """Return all results generator from the ``Desy`` spider via pipelines.""" + # environmental variables needed for the pipelines payload + os.environ['SCRAPY_JOB'] = 'scrapy_job' + os.environ['SCRAPY_FEED_URI'] = 'scrapy_feed_uri' + os.environ['SCRAPY_LOG_FILE'] = 'scrapy_log_file' + + spider = create_spider() + records = spider.parse( + fake_response_from_file( + file_name=response_file_name, + response_type=TextResponse + ) + ) + + pipeline = InspireCeleryPushPipeline() + pipeline.open_spider(spider) + + return ( + pipeline.process_item( + record, + spider + ) for record in records + ) + + +def get_one_record(response_file_name): + parsed_items = get_records(response_file_name) + record = parsed_items.next() + return record + + +def override_generated_fields(record): + record['acquisition_source']['datetime'] = '2017-05-04T17:49:07.975168' + record['acquisition_source']['submission_number'] = ( + '5652c7f6190f11e79e8000224dabeaad' + ) + + return record + + +@pytest.mark.parametrize( + 'generated_record', + [ + get_one_record('desy/desy_record.xml'), + ], + ids=[ + 'smoke', + ] +) +def test_pipeline_record(generated_record): + expected = { + '$schema': 'hep.json', + '_collections': [ + 'Literature' + ], + '_fft': [ + { + 'creation_datetime': '2017-06-27T09:43:17', + 'description': ( + '00013 Decomposition of the problematic rotation curves ' + 'in our sample according to the best-fit ' + '\\textsc{core}NFW models. Colors and symbols are as in ' + 'Figure \\ref{fig:dc14_fits}.' + ), + 'filename': 'cNFW_rogue_curves', + 'format': '.txt', + 'path': 'FFT/test_fft_1.txt', + 'type': 'Main', + 'version': 1, + }, + { + 'creation_datetime': '2017-06-27T09:43:16', + 'description': ( + '00005 Comparison of the parameters of the best-fit DC14 ' + 'models to the cosmological halo mass-concentration ' + 'relation from \\cite{dutton14} (left) and the stellar ' + 'mass-halo mass relation from \\cite{behroozi13} (right). ' + 'The error bars correspond to the extremal values of the ' + 'multidimensional 68\\% confidence region for each fit. ' + 'The theoretical relations are shown as red lines and ' + 'their 1$\\sigma$ and 2$\\sigma$ scatter are represented ' + 'by the dark and light grey bands, respectively. The ' + 'mass-concentration relation from \\cite{maccio08} and ' + 'the stellar mass-halo mass relation from ' + '\\cite{behroozi13} are also shown as the black dashed ' + 'lines.' + ), + 'filename': 'scalingRelations_DutBeh_DC14_all_Oh', + 'format': '.txt', + 'path': 'FFT/test_fft_2.txt', + 'type': 'Main', + 'version': 1 + } + ], + 'abstracts': [ + { + 'source': 'Deutsches Elektronen-Synchrotron', + 'value': ( + 'Dielectric laser acceleration of electrons has recently ' + 'been\n demonstrated with significantly ' + 'higher accelerating gradients than other\n ' + ' structure-based linear accelerators. Towards the ' + 'development of an integrated 1 MeV\n ' + 'electron accelerator based on dielectric laser ' + 'accelerator technologies,\n development ' + 'in several relevant technologies is needed. In this ' + 'work, recent\n developments on electron ' + 'sources, bunching, accelerating, focussing, deflecting ' + 'and\n laser coupling structures are ' + 'reported. With an eye to the near future, ' + 'components\n required for a 1 MeV kinetic ' + 'energy tabletop accelerator producing ' + 'sub-femtosecond\n electron bunches are ' + 'outlined.\n ' + ) + } + ], + 'acquisition_source': { + 'datetime': '2017-05-04T17:49:07.975168', + 'method': 'hepcrawl', + 'source': 'desy', + 'submission_number': '5652c7f6190f11e79e8000224dabeaad' + }, + 'control_number': 111111, + 'document_type': [ + 'article' + ], + 'dois': [ + { + 'value': '10.18429/JACoW-IPAC2017-WEYB1' + } + ], + 'number_of_pages': 6, + 'public_notes': [ + { + 'value': '*Brief entry*' + } + ], + 'publication_info': [ + { + 'parent_isbn': '9783954501823' + }, + { + 'page_end': '2525', + 'page_start': '2520', + 'year': 2017 + } + ], + 'self': { + '$ref': 'https://labs.inspirehep.net/api/literature/111111' + }, + 'titles': [ + { + 'source': 'JACoW', + 'title': ( + 'Towards a Fully Integrated Accelerator on a Chip: ' + 'Dielectric Laser\n Acceleration (DLA) ' + 'From the Source to Relativistic Electrons\n ' + ) + } + ], + 'urls': [ + { + 'description': 'Fulltext', + 'value': ( + 'http://inspirehep.net/record/1608652/files/Towards ' + 'a fully\n integrated acc on a chip.pdf' + '\n ' + ) + } + ] + } + + assert override_generated_fields(generated_record) == expected + + +@pytest.mark.parametrize( + 'generated_records', + [ + get_records('desy/desy_collection_records.xml'), + ], + ids=[ + 'smoke', + ] +) +def test_pipeline_collection_records(generated_records): + expected = [{ + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-05-04T17:49:07.975168" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + 'creation_datetime': '2017-06-27T09:43:17', + 'description': ( + '00013 Decomposition of the problematic rotation ' + 'curves in our sample according to the best-fit ' + '\\textsc{core}NFW models. Colors and symbols are as ' + 'in Figure \\ref{fig:dc14_fits}.' + ), + 'filename': 'cNFW_rogue_curves', + 'format': '.txt', + 'path': 'FFT/test_fft_1.txt', + 'type': 'Main', + 'version': 1, + }, + { + 'creation_datetime': '2017-06-27T09:43:16', + 'description': ( + '00005 Comparison of the parameters of the best-fit ' + 'DC14 models to the cosmological halo ' + 'mass-concentration relation from \\cite{dutton14} ' + '(left) and the stellar mass-halo mass relation from ' + '\\cite{behroozi13} (right). The error bars correspond' + ' to the extremal values of the multidimensional 68\\%' + ' confidence region for each fit. The theoretical ' + 'relations are shown as red lines and their ' + '1$\\sigma$ and 2$\\sigma$ scatter are represented ' + 'by the dark and light grey bands, respectively. The ' + 'mass-concentration relation from \\cite{maccio08} ' + 'and the stellar mass-halo mass relation from ' + '\\cite{behroozi13} are also shown as the black ' + 'dashed lines.' + ), + 'filename': 'scalingRelations_DutBeh_DC14_all_Oh', + 'format': '.txt', + 'path': 'FFT/test_fft_2.txt', + 'type': 'Main', + 'version': 1 + } + ], + "control_number": 111111, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/111111" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": ( + 'Towards a Fully Integrated Accelerator on a Chip: ' + 'Dielectric Laser\n Acceleration (DLA) ' + 'From the Source to Relativistic Electrons' + '\n ' + ) + } + ], + "urls": [ + { + "description": "Fulltext", + "value": ( + 'http://inspirehep.net/record/1608652/files/' + 'Towards a fully\n integrated acc on a ' + 'chip.pdf\n ' + ) + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": ( + "Dielectric laser acceleration of electrons has " + "recently been\n demonstrated with " + "significantly higher accelerating gradients than " + "other\n structure-based linear " + "accelerators. Towards the development of an " + "integrated 1 MeV\n electron " + "accelerator based on dielectric laser accelerator " + "technologies,\n development in " + "several relevant technologies is needed. In this work" + ", recent\n developments on electron " + "sources, bunching, accelerating, focussing, " + "deflecting and\n laser coupling " + "structures are reported. With an eye to the near " + "future, components\n required for a 1 " + "MeV kinetic energy tabletop accelerator producing sub" + "-femtosecond\n electron bunches are " + "outlined.\n " + ) + } + ] + }, + { + "acquisition_source": { + "source": "desy", + "method": "hepcrawl", + "submission_number": "5652c7f6190f11e79e8000224dabeaad", + "datetime": "2017-05-04T17:49:07.975168" + }, + "_collections": [ + "Literature" + ], + "_fft": [ + { + 'creation_datetime': '2017-06-27T09:43:17', + 'description': ( + "00013 Decomposition of the problematic rotation " + "curves in our sample according to the best-fit " + "\\textsc{core}NFW models. Colors and symbols are as " + "in Figure \\ref{fig:dc14_fits}." + ), + 'filename': 'cNFW_rogue_curves', + 'format': '.txt', + 'path': 'FFT/test_fft_1.txt', + 'type': 'Main', + 'version': 1, + }, + { + 'creation_datetime': '2017-06-27T09:43:16', + 'description': ( + '00005 Comparison of the parameters of the best-fit ' + 'DC14 models to the cosmological halo ' + 'mass-concentration relation from \\cite{dutton14} ' + '(left) and the stellar mass-halo mass relation ' + 'from \\cite{behroozi13} (right). The error bars ' + 'correspond to the extremal values of the ' + 'multidimensional 68\\% confidence region for each ' + 'fit. The theoretical relations are shown as red ' + 'lines and their 1$\\sigma$ and 2$\\sigma$ scatter ' + 'are represented by the dark and light grey bands, ' + 'respectively. The mass-concentration relation ' + 'from \\cite{maccio08} and the stellar mass-halo ' + 'mass relation from \\cite{behroozi13} are also ' + 'shown as the black dashed lines.' + ), + 'filename': 'scalingRelations_DutBeh_DC14_all_Oh', + 'format': '.txt', + 'path': 'FFT/test_fft_2.txt', + 'type': 'Main', + 'version': 1 + } + ], + "control_number": 222222, + "public_notes": [ + { + "value": "*Brief entry*" + } + ], + "self": { + "$ref": "https://labs.inspirehep.net/api/literature/222222" + }, + "number_of_pages": 6, + "titles": [ + { + "source": "JACoW", + "title": ( + "Towards a Fully Integrated Accelerator on a Chip: " + "Dielectric Laser\n Acceleration " + "(DLA) From the Source to Relativistic Electrons" + "\n " + ) + } + ], + "urls": [ + { + "description": "Fulltext", + "value": ( + "http://inspirehep.net/record/1608652/files/" + "Towards a fully\n integrated acc on a " + "chip.pdf\n " + ) + } + ], + "dois": [ + { + "value": "10.18429/JACoW-IPAC2017-WEYB1" + } + ], + "publication_info": [ + { + "parent_isbn": "9783954501823" + }, + { + "page_start": "2520", + "page_end": "2525", + "year": 2017 + } + ], + "$schema": "hep.json", + "document_type": [ + "article" + ], + "abstracts": [ + { + "source": "Deutsches Elektronen-Synchrotron", + "value": ( + "Dielectric laser acceleration of electrons has " + "recently been\n demonstrated with " + "significantly higher accelerating gradients than " + "other\n structure-based linear " + "accelerators. Towards the development of an " + "integrated 1 MeV\n electron " + "accelerator based on dielectric laser accelerator " + "technologies,\n development in " + "several relevant technologies is needed. In this " + "work, recent\n developments on " + "electron sources, bunching, accelerating, " + "focussing, deflecting and\n laser " + "coupling structures are reported. With an eye to " + "the near future, components\n " + "required for a 1 MeV kinetic energy tabletop " + "accelerator producing sub-femtosecond" + "\n electron bunches are outlined." + "\n " + ) + } + ] + } + ] + + generated_results = [ + override_generated_fields(rec) for rec in generated_records + ] + + assert generated_results == expected diff --git a/tests/unit/test_dnb.py b/tests/unit/test_dnb.py index b00aff3d..5aa05a64 100644 --- a/tests/unit/test_dnb.py +++ b/tests/unit/test_dnb.py @@ -72,7 +72,12 @@ def record(scrape_pos_page_body): body=scrape_pos_page_body, **{'encoding': 'utf-8'} ) - return request.callback(response) + + parsed_item = request.callback(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_title(record): @@ -241,7 +246,12 @@ def parse_without_splash(): 'Content-Type': 'application/pdf;charset=base64', } ) - return spider.parse_node(response, nodes[0]) + + parsed_item = spider.parse_node(response, nodes[0]) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_parse_without_splash(parse_without_splash): diff --git a/tests/unit/test_edp.py b/tests/unit/test_edp.py index cc7885bd..115abda6 100644 --- a/tests/unit/test_edp.py +++ b/tests/unit/test_edp.py @@ -40,6 +40,7 @@ def scrape_pos_page_body(): ) ) + @pytest.fixture def targzfile(): """Path to test tar.gz file with JATS XML file.""" @@ -50,6 +51,7 @@ def targzfile(): 'test_gz.tar.gz' ) + @pytest.fixture def package_jats(targzfile): """Extract tar.gz package with JATS XML file.""" @@ -75,7 +77,12 @@ def record_jats(package_jats, scrape_pos_page_body): body=scrape_pos_page_body, **{'encoding': 'utf-8'} ) - return request.callback(response) + + parsed_item = request.callback(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture @@ -107,7 +114,11 @@ def record_rich(package_rich): fake_resp.meta["rich"] = True node = get_node(spider, "//EDPSArticle", fake_resp)[0] - return spider.parse_node(fake_resp, node) + parsed_item = spider.parse_node(fake_resp, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_title(record_jats): @@ -145,6 +156,7 @@ def test_abstract(record_jats): assert 'abstract' in record_jats assert record_jats['abstract'] == abstract + def test_date_published(record_jats): """Test extracting date_published.""" date_published = "2015-01-01" @@ -179,6 +191,7 @@ def test_doi(record_jats): assert 'dois' in record_jats assert record_jats['dois'][0]['value'] == doi + def test_publication_info(record_jats): """Test extracting publication info.""" assert 'journal_title' in record_jats @@ -206,7 +219,6 @@ def test_keywords(record_jats): assert keyw["value"] in keywords - def test_authors(record_jats): """Test authors.""" authors = ["Arasoglu, Ali", "Ozdemir, Omer Faruk"] @@ -326,7 +338,6 @@ def test_authors_rich(record_rich): assert astr[index]["affiliations"][0]["value"] == affiliations[index] - def test_tarfile(tarbzfile, tmpdir): """Test untarring a tar.bz package with a test XML file. @@ -343,7 +354,6 @@ def test_tarfile(tarbzfile, tmpdir): assert "aas/xml_rich/2000/01" not in xml_files_flat[0] - def test_handle_package_ftp(tarbzfile): """Test getting the target folder name for xml files.""" spider = edp_spider.EDPSpider() @@ -351,7 +361,8 @@ def test_handle_package_ftp(tarbzfile): request = spider.handle_package_ftp(response).next() assert isinstance(request, Request) - assert request.meta["package_path"] == tarbzfile + assert request.meta["source_folder"] == tarbzfile + def test_no_dois_jats(): """Test parsing when no DOI in record. JATS format.""" @@ -370,7 +381,11 @@ def test_no_dois_jats(): """ response = fake_response_from_string(body) node = get_node(spider, "//article", response)[0] - record = spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert "dois" not in record assert "additional_files" not in record @@ -390,7 +405,11 @@ def test_no_dois_rich(): response = fake_response_from_string(body) response.meta["rich"] = True node = get_node(spider, "//EDPSArticle", response)[0] - record = spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert "dois" not in record assert "additional_files" not in record @@ -416,7 +435,11 @@ def test_addendum_jats(): """ response = fake_response_from_string(body) node = get_node(spider, "//article", response)[0] - record = spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert "related_article_doi" in record assert record["related_article_doi"][0][ @@ -439,7 +462,11 @@ def test_author_with_email(): """ response = fake_response_from_string(body) node = get_node(spider, "//article", response)[0] - record = spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert 'email' in record['authors'][0] assert record['authors'][0]['email'] == "Fname.Sname@university.org" @@ -472,7 +499,11 @@ def test_aff_with_email(): """ response = fake_response_from_string(body) node = get_node(spider, "//article", response)[0] - record = spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + record = parsed_item.record affiliation = "Department of Physics, Western Michigan University, Kalamazoo, MI 49008, USA" assert 'affiliations' in record['authors'][0] @@ -481,8 +512,6 @@ def test_aff_with_email(): assert record['authors'][0]['email'] is None - - def test_no_valid_article(): """Test parsing when filtering out non-interesting article types.""" spider = edp_spider.EDPSpider() @@ -506,7 +535,11 @@ def test_collections_review(): """ response = fake_response_from_string(body) node = get_node(spider, "//article", response)[0] - record = spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert "collections" in record assert record["collections"] == [{'primary': 'HEP'}, {'primary': 'Review'}] @@ -533,7 +566,12 @@ def record_references_only(): """ response = fake_response_from_string(body) node = get_node(spider, "//article", response)[0] - return spider.parse_node(response, node) + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_references(record_references_only): diff --git a/tests/unit/test_elsevier.py b/tests/unit/test_elsevier.py index ca023122..3d5fb3f5 100644 --- a/tests/unit/test_elsevier.py +++ b/tests/unit/test_elsevier.py @@ -41,9 +41,12 @@ def record(): response.meta["xml_url"] = 'elsevier/sample_consyn_record.xml' tag = '//%s' % spider.itertag nodes = get_node(spider, tag, response) - parsed_record = spider.parse_node(response, nodes) - assert parsed_record - return parsed_record + + parsed_item = spider.parse_node(response, nodes) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture(scope="module") @@ -97,7 +100,12 @@ def parsed_node(): response.meta["xml_url"] = 'elsevier/sample_consyn_record.xml' parse_response = spider.parse_node(response, node) parse_response.status = 404 - return spider.scrape_sciencedirect(parse_response) + + parsed_item = spider.scrape_sciencedirect(parse_response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_collection(parsed_node): @@ -164,7 +172,11 @@ def cover_display_date(): node = get_node(spider, '/doc', text=body) response = fake_response_from_string(body) - return spider.parse_node(response, node) + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_cover_display_date(cover_display_date): @@ -187,7 +199,11 @@ def cover_display_date_y_m(): """ node = get_node(spider, '/doc', text=body) response = fake_response_from_string(body) - return spider.parse_node(response, node) + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_cover_display_date_y_m(cover_display_date_y_m): @@ -210,7 +226,11 @@ def cover_display_date_y(): """ node = get_node(spider, '/doc', text=body) response = fake_response_from_string(body) - return spider.parse_node(response, node) + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_cover_display_date_y(cover_display_date_y): @@ -1579,11 +1599,11 @@ def test_handle_package(handled_package): for astro, nima in zip(astropart, nima): assert nima assert astro - assert astro.meta["package_path"] == "tests/unit/responses/elsevier/fake_astropart.zip" + assert astro.meta["source_folder"] == "tests/unit/responses/elsevier/fake_astropart.zip" url_to_match = u'file:///tmp/elsevier_fake_astropart_*/0927-6505/aip/S0927650515001656/S0927650515001656.xml' assert astro.meta["xml_url"] == fnmatch.filter([astro.meta["xml_url"]], url_to_match)[0] - assert nima.meta["package_path"] == "tests/unit/responses/elsevier/fake_nima.zip" + assert nima.meta["source_folder"] == "tests/unit/responses/elsevier/fake_nima.zip" url_to_match = u'file:///tmp/elsevier_fake_nima_*/0168-9002/S0168900215X00398/S0168900215015636/S0168900215015636.xml' assert nima.meta["xml_url"] == fnmatch.filter([nima.meta["xml_url"]], url_to_match)[0] @@ -1644,7 +1664,12 @@ def sciencedirect(): ]) response.meta["info"] = {} response.meta["node"] = get_node(spider, '/head', text=body) - return spider.scrape_sciencedirect(response) + + parsed_item = spider.scrape_sciencedirect(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_sciencedirect(sciencedirect): diff --git a/tests/unit/test_hindawi.py b/tests/unit/test_hindawi.py index 37e5e183..3af8ba3a 100644 --- a/tests/unit/test_hindawi.py +++ b/tests/unit/test_hindawi.py @@ -26,9 +26,11 @@ def record(): response = fake_response_from_file("hindawi/test_1.xml") nodes = get_node(spider, "//marc:record", response) - parsed_record = spider.parse_node(response, nodes[0]) - assert parsed_record - return parsed_record + parsed_item = spider.parse_node(response, nodes[0]) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_title(record): diff --git a/tests/unit/test_infn.py b/tests/unit/test_infn.py index 0c60799a..c15ef727 100644 --- a/tests/unit/test_infn.py +++ b/tests/unit/test_infn.py @@ -28,9 +28,12 @@ def record(): """Return scraping results from the INFN spider.""" spider = infn_spider.InfnSpider() response = fake_response_from_file('infn/test_splash.html') - parsed_record = spider.scrape_splash(response) - assert parsed_record - return parsed_record + + parsed_item = spider.scrape_splash(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_title(record): @@ -121,6 +124,7 @@ def test_non_thesis(): assert record is None + def test_parse_node(): """Test parse_node function. This should be a scrapy Request object. @@ -148,6 +152,8 @@ def test_parse_node_nolink(): response = fake_response_from_file('infn/test_1_nolink.html') selector = Selector(response, type='html') node = selector.xpath('//%s' % spider.itertag)[0] - record = spider.parse_node(response, node).next() + parsed_item = spider.parse_node(response, node).next() + assert parsed_item + assert parsed_item.record - assert isinstance(record, hepcrawl.items.HEPRecord) + assert isinstance(parsed_item.record, hepcrawl.items.HEPRecord) diff --git a/tests/unit/test_iop.py b/tests/unit/test_iop.py index b776adfa..bb01766c 100644 --- a/tests/unit/test_iop.py +++ b/tests/unit/test_iop.py @@ -38,9 +38,12 @@ def record(): response = fake_response_from_file('iop/xml/test_standard.xml') node = get_node(spider, "Article", response) spider.pdf_files = TEST_PDF_DIR - parsed_record = spider.parse_node(response, node) - assert parsed_record - return parsed_record + + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_abstract(record): @@ -182,10 +185,12 @@ def erratum_open_access_record(): 'iop', 'pdf', ) - parsed_record = spider.parse_node(response, node) - assert parsed_record - return parsed_record + parsed_item = spider.parse_node(response, node) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_files_erratum_open_access_record(erratum_open_access_record): diff --git a/tests/unit/test_magic.py b/tests/unit/test_magic.py index eeb574fe..f3c0f355 100644 --- a/tests/unit/test_magic.py +++ b/tests/unit/test_magic.py @@ -23,6 +23,7 @@ get_node, ) + @pytest.fixture def record(): """Return results from the MAGIC spider. First parse node, then scrape, @@ -39,9 +40,11 @@ def record(): splash_response.meta["date"] = parsed_node.meta["date"] splash_response.meta["urls"] = parsed_node.meta["urls"] - parsed_record = spider.scrape_for_pdf(splash_response).next() - assert parsed_record - return parsed_record + parsed_item = spider.scrape_for_pdf(splash_response).next() + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_abstract(record): @@ -102,7 +105,6 @@ def test_abstract(record): assert record["abstract"] == abstract - def test_title(record): """Test extracting title.""" title = "Limits to the violation of Lorentz invariance using the emission of the CRAB pulsar at TeV energies, discovered with archival data from the MAGIC telescopes" @@ -139,6 +141,7 @@ def test_url(record): assert 'urls' in record assert record['urls'][0]['value'] == url + def test_pdf_link(record): """Test pdf link(s)""" files = "http://stlab.adobe.com/wiki/images/d/d3/Test.pdf" @@ -164,7 +167,10 @@ def test_no_author_no_date_no_url(): """ response = fake_response_from_string(body) node = get_node(spider, spider.itertag, text=body) - record = spider.parse_node(response, node).next() + parsed_item = spider.parse_node(response, node).next() + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert isinstance(record, hepcrawl.items.HEPRecord) assert "date" not in record @@ -184,7 +190,10 @@ def test_no_aff(): """ response = fake_response_from_string(body) - record = spider.scrape_for_pdf(response).next() + parsed_item = spider.scrape_for_pdf(response).next() + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert isinstance(record, hepcrawl.items.HEPRecord) assert "date" not in record @@ -216,7 +225,10 @@ def test_no_spash_page(): response.status = 404 response.meta["title"] = parsed_node.meta["title"] response.meta["urls"] = parsed_node.meta["urls"] - record = spider.scrape_for_pdf(response).next() + parsed_item = spider.scrape_for_pdf(response).next() + assert parsed_item + assert parsed_item.record + record = parsed_item.record assert isinstance(record, hepcrawl.items.HEPRecord) assert "urls" in record diff --git a/tests/unit/test_mit.py b/tests/unit/test_mit.py index 0253d91f..8a185cef 100644 --- a/tests/unit/test_mit.py +++ b/tests/unit/test_mit.py @@ -25,9 +25,12 @@ def record(): """Return scraping results from the MIT spider.""" spider = mit_spider.MITSpider() response = fake_response_from_file('mit/test_splash.html') - parsed_record = spider.build_item(response) - assert parsed_record - return parsed_record + + parsed_item = spider.build_item(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture @@ -37,7 +40,11 @@ def parsed_node(): response = fake_response_from_file('mit/test_list.html') tag = spider.itertag node = get_node(spider, tag, response, rtype="html") - return spider.parse_node(response, node).next() + + parsed_item = spider.parse_node(response, node).next() + assert parsed_item + + return parsed_item def test_url(parsed_node): @@ -159,7 +166,12 @@ def supervisors(): """ response = fake_response_from_string(body) - return spider.build_item(response) + + parsed_item = spider.build_item(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_two_supervisors(supervisors): diff --git a/tests/unit/test_phenix.py b/tests/unit/test_phenix.py index 75384350..c272683f 100644 --- a/tests/unit/test_phenix.py +++ b/tests/unit/test_phenix.py @@ -29,9 +29,13 @@ def record(): response = fake_response_from_file('phenix/test_1.html') selector = Selector(response, type='html') nodes = selector.xpath('//%s' % spider.itertag) - parsed_record = spider.parse_node(response, nodes[0]) - assert parsed_record - return parsed_record + + parsed_item = spider.parse_node(response, nodes[0]) + assert parsed_item + assert parsed_item.record + + return parsed_item.record + @pytest.fixture def non_thesis(): @@ -49,10 +53,12 @@ def non_thesis(): node = get_node(spider, '//li', text=body) return spider.parse_node(response, node) + def test_non_thesis(non_thesis): """Test MSc thesis skipping.""" assert non_thesis is None + def test_title(record): """Test extracting title.""" title = "MEASUREMENT OF THE DOUBLE HELICITY ASYMMETRY IN INCLUSIVE $\pi^{0}$ PRODUCTION IN POLARIZED PROTON-PROTON COLLISIONS AT $\sqrt{s}$ = 510 GeV" @@ -82,6 +88,7 @@ def test_authors(record): aff['value'] for aff in record['authors'][index]['affiliations'] ] + def test_pdf_link(record): """Test pdf link(s)""" files = "http://www.phenix.bnl.gov/phenix/WWW/talk/archive/theses/2015/Guragain_Hari-DISSERTATION.pdf" diff --git a/tests/unit/test_phil.py b/tests/unit/test_phil.py index e99064b2..6db536ef 100644 --- a/tests/unit/test_phil.py +++ b/tests/unit/test_phil.py @@ -33,9 +33,12 @@ def record(): "http://philpapers.org/go.pl?id=BROBB&proxyId=none&u=http%3A%2F%2Fanalysis.oxfordjournals.org%2Fcontent%2F66%2F3%2F194.full.pdf%2Bhtml%3Fframe%3Dsidebar", "http://philpapers.org/go.pl?id=BROBB&proxyId=none&u=http%3A%2F%2Fbrogaardb.googlepages.com%2Ftensedrelationsoffprint.pdf" ] - parsed_record = spider.build_item(response) - assert parsed_record - return parsed_record + + parsed_item = spider.build_item(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture @@ -48,7 +51,12 @@ def journal(): response = fake_response_from_file('phil/test_journal.json') jsonrecord = json.loads(response.body_as_unicode()) response.meta["jsonrecord"] = jsonrecord[0] - return spider.build_item(response) + + parsed_item = spider.build_item(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record @pytest.fixture @@ -223,7 +231,11 @@ def splash(): ] } - return spider.scrape_for_pdf(response) + parsed_item = spider.scrape_for_pdf(response) + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_scrape(splash): diff --git a/tests/unit/test_pos.py b/tests/unit/test_pos.py index 20c872f4..bea29b34 100644 --- a/tests/unit/test_pos.py +++ b/tests/unit/test_pos.py @@ -51,8 +51,11 @@ def record(scrape_pos_page_body): assert response pipeline = InspireCeleryPushPipeline() pipeline.open_spider(spider) - record = request.callback(response) - return pipeline.process_item(record, spider) + parsed_item = request.callback(response) + parsed_record = pipeline.process_item(parsed_item, spider) + assert parsed_record + + return parsed_record def test_titles(record): diff --git a/tests/unit/test_t2k.py b/tests/unit/test_t2k.py index 283a02e5..d9395aa2 100644 --- a/tests/unit/test_t2k.py +++ b/tests/unit/test_t2k.py @@ -36,9 +36,11 @@ def record(): splash_response.meta["urls"] = parsed_node.meta["urls"] splash_response.meta["authors"] = parsed_node.meta["authors"] - parsed_record = spider.scrape_for_pdf(splash_response).next() - assert parsed_record - return parsed_record + parsed_item = spider.scrape_for_pdf(splash_response).next() + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_abstact(record): @@ -125,9 +127,11 @@ def non_url(): selector = Selector(response, type='html') nodes = selector.xpath('//%s' % spider.itertag) - parsed_record = spider.parse_node(response, nodes[0]).next() - assert parsed_record - return parsed_record + parsed_item = spider.parse_node(response, nodes[0]).next() + assert parsed_item + assert parsed_item.record + + return parsed_item.record def test_non_url(non_url): diff --git a/tests/unit/test_crawler2hep.py b/tests/unit/test_tohep.py similarity index 87% rename from tests/unit/test_crawler2hep.py rename to tests/unit/test_tohep.py index 95375ebf..547bc67e 100644 --- a/tests/unit/test_crawler2hep.py +++ b/tests/unit/test_tohep.py @@ -12,14 +12,14 @@ import pytest import yaml -from hepcrawl.crawler2hep import crawler2hep +from hepcrawl.tohep import hepcrawl_to_hep from hepcrawl.testlib.fixtures import get_test_suite_path def load_file(file_name): path = get_test_suite_path( 'responses', - 'crawler2hep', + 'tohep', file_name, ) with open(path) as input_data: @@ -52,7 +52,7 @@ def test_generic_crawler_record( input_generic_crawler_record, expected_generic_crawler_record ): - produced_record = crawler2hep(input_generic_crawler_record) + produced_record = hepcrawl_to_hep(input_generic_crawler_record) assert produced_record == expected_generic_crawler_record @@ -60,5 +60,5 @@ def test_no_document_type( input_no_document_type_record, expected_no_document_type_record ): - produced_record = crawler2hep(input_no_document_type_record) + produced_record = hepcrawl_to_hep(input_no_document_type_record) assert produced_record == expected_no_document_type_record diff --git a/tests/unit/test_world_scientific.py b/tests/unit/test_world_scientific.py index 36438ab4..291d00d0 100644 --- a/tests/unit/test_world_scientific.py +++ b/tests/unit/test_world_scientific.py @@ -48,8 +48,11 @@ def get_records(response_file_name): def get_one_record(response_file_name): - results = get_records(response_file_name) - return results.next() + records = get_records(response_file_name) + record = records.next() + assert record + + return record def override_generated_fields(record):