From 707d8f91b186c35c8955e44a550a9aac152a9157 Mon Sep 17 00:00:00 2001 From: PascalEgn Date: Wed, 10 Jul 2024 15:34:19 +0200 Subject: [PATCH] only run py3 unit tests --- .github/workflows/build-and-test-py3.yml | 73 -------- .github/workflows/test-py3.yml | 34 ++++ docker-compose.deps.py3.yml | 20 --- docker-compose.test.py3.yml | 204 ----------------------- tests/Dockerfile.hepcrawl_py3 | 14 -- 5 files changed, 34 insertions(+), 311 deletions(-) delete mode 100644 .github/workflows/build-and-test-py3.yml create mode 100644 .github/workflows/test-py3.yml delete mode 100644 docker-compose.deps.py3.yml delete mode 100644 docker-compose.test.py3.yml delete mode 100644 tests/Dockerfile.hepcrawl_py3 diff --git a/.github/workflows/build-and-test-py3.yml b/.github/workflows/build-and-test-py3.yml deleted file mode 100644 index 455c6aec..00000000 --- a/.github/workflows/build-and-test-py3.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Build and release python 3 - -on: - push: - branches: [celery-5] - pull_request: - branches: [celery-5] - - -jobs: - Test: - runs-on: ubuntu-latest - strategy: - matrix: - suite: - [ - unit, - functional_wsp, - functional_arxiv, - functional_desy, - functional_cds, - functional_pos, - functional_elsevier, - functional_aps - ] - python-version: ["3.9", "3.11", "3.12"] - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel coveralls check-manifest - pip install -e .[tests] - - - name: Show python dependencies - run: | - pip freeze - - - name: Install dependencies - run: | - docker-compose -f docker-compose.deps.py3.yml build - docker-compose -f docker-compose.test.py3.yml run --rm scrapyd-deploy - - - name: Print logs if failure - if: ${{ failure() }} - run: | - docker-compose -f docker-compose.test.py3.yml logs --tail=200 - - - name: Run tests - run: | - docker-compose -f docker-compose.test.py3.yml run --rm ${{ matrix.suite }} - - - name: Coveralls - uses: AndreMiras/coveralls-python-action@v20201129 - with: - parallel: true - github-token: ${{ secrets.github_token }} - flag-name: run-${{ matrix.test_number }} - - Coveralls: - needs: Test - runs-on: ubuntu-latest - steps: - - name: Coveralls Finished - uses: AndreMiras/coveralls-python-action@v20201129 - with: - github-token: ${{ secrets.github_token }} - parallel-finished: true diff --git a/.github/workflows/test-py3.yml b/.github/workflows/test-py3.yml new file mode 100644 index 00000000..370c3fc2 --- /dev/null +++ b/.github/workflows/test-py3.yml @@ -0,0 +1,34 @@ +name: Test python 3 + +on: + push: + branches: [celery-5] + pull_request: + branches: [celery-5] + + +jobs: + Test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.11", "3.12"] + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install -e .[all] + + - name: Show python dependencies + run: | + pip freeze + + - name: Run tests + run: | + py.test -vv tests/unit diff --git a/docker-compose.deps.py3.yml b/docker-compose.deps.py3.yml deleted file mode 100644 index 42741a19..00000000 --- a/docker-compose.deps.py3.yml +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of hepcrawl. -# Copyright (C) 2017 CERN. -# -# hepcrawl is a free software; you can redistribute it and/or modify it -# under the terms of the Revised BSD License; see LICENSE file for -# more details. - -version: '2.1' - -services: - hepcrawl_testing_image: - build: - context: ./ - dockerfile: tests/Dockerfile.hepcrawl_py3 - image: hepcrawl_py3 - command: "true" - volumes: - - .:/code/ diff --git a/docker-compose.test.py3.yml b/docker-compose.test.py3.yml deleted file mode 100644 index 77924785..00000000 --- a/docker-compose.test.py3.yml +++ /dev/null @@ -1,204 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of hepcrawl. -# Copyright (C) 2017 CERN. -# -# hepcrawl is a free software; you can redistribute it and/or modify it -# under the terms of the Revised BSD License; see LICENSE file for -# more details. - -version: '2.1' - -services: - service_base: &service_base - image: hepcrawl_py3 - environment: - - APP_BROKER_URL=pyamqp://guest:guest@rabbitmq:5672// - - APP_CELERY_RESULT_BACKEND=redis://redis:6379/1 - - APP_CRAWLER_HOST_URL=http://scrapyd:6800 - - APP_API_PIPELINE_TASK_ENDPOINT_DEFAULT=hepcrawl.testlib.tasks.submit_results - - APP_FILES_STORE=/tmp/file_urls - - APP_LAST_RUNS_PATH=/code/.scrapy/last_runs - - APP_CRAWL_ONCE_PATH=/code/.scrapy - - COVERAGE_PROCESS_START=/code/.coveragerc - tty: true - volumes: - - .:/code/ - user: "${UID:-1000}:${GID:-1000}" - - functional_wsp: - <<: *service_base - command: py.test -vv tests/functional/wsp - depends_on: - scrapyd: - condition: service_healthy - ftp_server: - condition: service_healthy - - functional_desy: - <<: *service_base - command: py.test -vv tests/functional/desy - depends_on: - scrapyd: - condition: service_healthy - ftp_server: - condition: service_healthy - - functional_arxiv: - <<: *service_base - command: py.test -vv tests/functional/arxiv - depends_on: - scrapyd: - condition: service_healthy - arxiv-http-server.local: - condition: service_healthy - - functional_pos: - <<: *service_base - command: py.test -vv tests/functional/pos - depends_on: - scrapyd: - condition: service_healthy - http-server.local: - condition: service_healthy - - unit: - <<: *service_base - command: py.test -vv tests/unit - links: [] - - celery: - <<: *service_base - command: celery worker --events --app hepcrawl.testlib.tasks --loglevel=debug - depends_on: - rabbitmq: - condition: service_healthy - redis: - condition: service_healthy - - scrapyd: - <<: *service_base - command: scrapyd --pidfile=/tmp/scrapyd.pid - networks: - default: - ftp: - depends_on: - celery: - condition: service_started - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "curl -k http://localhost:6800/listprojects.json" - - scrapyd-deploy: - <<: *service_base - command: "true" # do nothing, as the real thing currently fails on Python 3, and not needed for unit - # command: bash -c "scrapyd-deploy" - # depends_on: - # scrapyd: - # condition: service_healthy - - ftp_server: - image: stilliard/pure-ftpd:hardened - environment: - - PUBLICHOST=1.2.3.4 - networks: - ftp: - ipv4_address: 1.2.3.4 - volumes: - - ${PWD}/tests/functional/desy/fixtures/ftp_server/FFT:/home/ftpusers/bob/FFT - - ${PWD}/tests/functional/desy/fixtures/ftp_server/DESY:/home/ftpusers/bob/DESY - - ${PWD}/tests/functional/wsp/fixtures/ftp_server/WSP:/home/ftpusers/bob/WSP - - ${PWD}/tests/functional/wsp/fixtures/ftp_server/pureftpd.passwd:/etc/pure-ftpd/passwd/pureftpd.passwd - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "ls -l /var/run/pure-ftpd.pid" - - http-server.local: - image: nginx:stable-alpine - volumes: - - ${PWD}/tests/functional/pos/fixtures/https_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf - - ${PWD}/tests/functional/pos/fixtures/https_server/conf/ssl:/etc/nginx/ssl - - ${PWD}/tests/functional/pos/fixtures/https_server/records:/etc/nginx/html/ - ports: - - 443:443 - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "curl -k https://localhost:443/187.html" - - functional_cds: - <<: *service_base - command: py.test -vv tests/functional/cds - depends_on: - scrapyd: - condition: service_healthy - cds-http-server.local: - condition: service_healthy - - arxiv-http-server.local: - image: nginx:stable-alpine - volumes: - - ${PWD}/tests/functional/arxiv/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf - - ${PWD}/tests/functional/arxiv/fixtures/http_server/records:/etc/nginx/html/ - ports: - - 80:80 - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "curl -k http://localhost:80/arxiv-physics-hep-th.xml" - - cds-http-server.local: - image: nginx:stable-alpine - volumes: - - ${PWD}/tests/functional/cds/fixtures/http_server/conf/proxy.conf:/etc/nginx/conf.d/default.conf - - ${PWD}/tests/functional/cds/fixtures/http_server/records:/etc/nginx/html/ - ports: - - 80:80 - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "curl -k http://localhost:80/cds-single.xml" - - rabbitmq: - image: rabbitmq - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "rabbitmqctl status" - - redis: - image: redis:3.2.3 - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "redis-cli -h 127.0.0.1 ping| grep PONG" - - localstack: - image: localstack/localstack:latest - ports: - - '4572:4572' - - '4566:4566' - environment: - - SERVICES=s3 - - DEBUG=1 - - DATA_DIR=/home/localstack/data - healthcheck: - timeout: 5s - interval: 5s - retries: 5 - test: "curl -k localhost:4566|grep running" - -networks: - ftp: - ipam: - config: - - subnet: 1.0.0.0/8 diff --git a/tests/Dockerfile.hepcrawl_py3 b/tests/Dockerfile.hepcrawl_py3 deleted file mode 100644 index 5dc73059..00000000 --- a/tests/Dockerfile.hepcrawl_py3 +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3 - -WORKDIR /usr/src/app - -ENV PYTHONDONTWRITEBYTECODE=1 -RUN mkdir /code /var/lib/scrapy /venv - -copy . /code - -ENV PATH="/home/test/.local/bin:${PATH}" - -WORKDIR /code -RUN pip install --upgrade wheel setuptools idutils rfc3987 bleach jsonschema inspire-utils -RUN pip install --no-cache-dir -e .[all]