Merge pull request #77 from inspirehep/add-file-name

add filename to workflow extradata
inspirehep · Sep 12, 2023 · 459a3be · 459a3be
2 parents dcbe9e0 + 03ea18a
commit 459a3be
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 6 deletions.
diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml
@@ -41,12 +41,6 @@ jobs:
       matrix:
         python-version: [2.7.17]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          fetch-depth: 0
-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
@@ -55,6 +49,7 @@ jobs:
 
       - name: Install python dependencies
         run: |
+          python -m pip install --upgrade pip
           pip install --upgrade pip setuptools
           pip install twine wheel coveralls
           pip install -r requirements.txt

diff --git a/inspire_crawler/tasks.py b/inspire_crawler/tasks.py
@@ -145,6 +145,7 @@ def submit_results(job_id, errors, log_file, results_uri, spider_name, results_d
 
         record = crawl_result.pop('record')
         crawl_errors = crawl_result['errors']
+        file = crawl_result['file_name']
 
         current_app.logger.debug('Parsing record: {}'.format(record))
         engine = WorkflowEngine.with_name(job.workflow)
@@ -159,6 +160,7 @@ def submit_results(job_id, errors, log_file, results_uri, spider_name, results_d
             extra_data = {
                 'crawler_job_id': job_id,
                 'crawler_results_path': results_path,
+                'source_file': file
             }
             record_extra = record.pop('extra_data', {})
             if record_extra: