Improve dataset handling in harvesters by adding checks for dataset existence and logging warnings for missing datasets #136
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| env: | |
| COLUMNS: 120 | |
| on: [pull_request] | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install requirements | |
| run: pip install flake8 pycodestyle | |
| - name: Check syntax | |
| run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan | |
| test: | |
| needs: lint | |
| strategy: | |
| matrix: | |
| include: | |
| - ckan-version: "2.11" | |
| ckan-image: "ckan/ckan-dev:2.11-py3.10" | |
| - ckan-version: "2.10" | |
| ckan-image: "ckan/ckan-dev:2.10-py3.10" | |
| - ckan-version: "2.9" | |
| ckan-image: "ckan/ckan-dev:2.9-py3.9" | |
| fail-fast: false | |
| name: CKAN ${{ matrix.ckan-version }} | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ matrix.ckan-image }} | |
| services: | |
| solr: | |
| # Custom image with spatial support and advanced schema | |
| image: ghcr.io/mjanez/ckan-solr:2.10-solr9-spatial | |
| postgres: | |
| image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| redis: | |
| image: redis:7-alpine | |
| env: | |
| CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test | |
| CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test | |
| CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test | |
| CKAN_SOLR_URL: http://solr:8983/solr/ckan | |
| CKAN_REDIS_URL: redis://redis:6379/1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install requirements (common) | |
| run: | | |
| pip install -r requirements.txt | |
| pip install -r dev-requirements.txt | |
| pip install -e . | |
| # Replace default path to CKAN core config file with the one on the container | |
| sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini | |
| - name: Install requirements (2.9) | |
| run: | | |
| pip install -U pytest-rerunfailures | |
| if: ${{ matrix.ckan-version == '2.9' }} | |
| - name: Setup other extensions | |
| run: | | |
| # Install ckanext-dcat | |
| git clone https://github.com/mjanez/ckanext-dcat | |
| pip install -e ckanext-dcat | |
| pip install -r ckanext-dcat/requirements.txt | |
| # Install ckanext-harvest | |
| git clone https://github.com/ckan/ckanext-harvest | |
| pip install -e ckanext-harvest | |
| pip install -r ckanext-harvest/requirements.txt | |
| # Install ckanext-scheming | |
| git clone https://github.com/ckan/ckanext-scheming | |
| pip install -e ckanext-scheming | |
| # Install ckanext-fluent | |
| git clone https://github.com/mjanez/ckanext-fluent | |
| pip install -e ckanext-fluent | |
| - name: Setup extension | |
| run: | | |
| ckan -c test.ini db init | |
| ckan -c test.ini db pending-migrations --apply | |
| - name: Run tests | |
| run: pytest --log-cli-level=INFO --ckan-ini=test.ini --cov=ckanext.schemingdcat --cov-report=term-missing --cov-append --disable-warnings ckanext/schemingdcat/tests |