Fix/esa data ecosystem

.github/workflows/deploy-on-release.yml

@@ -5,87 +5,91 @@ on:
     branches: [ main ]
     types: [ published ]
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   unit-tests:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: Set up Python 3.8.6
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.8.18
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8.6
+          python-version: 3.8.18
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
 
       - name: Get pipenv venv hashes
         id: hashes
         run: |
-          echo "##[set-output name=root;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/Pipfile)"
-          echo "##[set-output name=alembic;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/alembic_migration/Pipfile)"
-          echo "##[set-output name=dategenerator;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/date_generator/Pipfile)"
-          echo "##[set-output name=linkfetcher;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/link_fetcher/Pipfile)"
-          echo "##[set-output name=downloader;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/downloader/Pipfile)"
-          echo "##[set-output name=mockscihubproductapi;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_product_api/Pipfile)"
-          echo "##[set-output name=mockscihubsearchapi;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_search_api/Pipfile)"
-          echo "##[set-output name=db;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/layers/db/Pipfile)"
+          echo "root=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/Pipfile)" >> $GITHUB_OUTPUT
+          echo "alembic=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/alembic_migration/Pipfile)" >> $GITHUB_OUTPUT
+          echo "dategenerator=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/date_generator/Pipfile)" >> $GITHUB_OUTPUT
+          echo "linkfetcher=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/link_fetcher/Pipfile)" >> $GITHUB_OUTPUT
+          echo "downloader=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/downloader/Pipfile)" >> $GITHUB_OUTPUT
+          echo "mockscihubproductapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_product_api/Pipfile)" >> $GITHUB_OUTPUT
+          echo "mockscihubsearchapi=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/lambdas/mock_scihub_search_api/Pipfile)" >> $GITHUB_OUTPUT
+          # echo "db=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/layers/db/Pipfile)" >> $GITHUB_OUTPUT
 
       - name: Setup root cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: root-cache
         with:
           path: /home/runner/.local/share/virtualenvs/hls-sentinel2-downloader-serverless-${{ steps.hashes.outputs.root }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/Pipfile.lock') }}-2
 
       - name: Setup alembic_migration cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: alembic-cache
         with:
           path: /home/runner/.local/share/virtualenvs/alembic_migration-${{ steps.hashes.outputs.alembic }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/alembic_migration/Pipfile.lock') }}
 
       - name: Setup date_generator cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: date-generator-cache
         with:
           path: /home/runner/.local/share/virtualenvs/date_generator-${{ steps.hashes.outputs.dategenerator }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/date_generator/Pipfile.lock') }}
 
       - name: Setup link_fetcher cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: link-fetcher-cache
         with:
           path: /home/runner/.local/share/virtualenvs/link_fetcher-${{ steps.hashes.outputs.linkfetcher }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/link_fetcher/Pipfile.lock') }}
 
       - name: Setup downloader cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: downloader-cache
         with:
           path: /home/runner/.local/share/virtualenvs/downloader-${{ steps.hashes.outputs.downloader }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/downloader/Pipfile.lock') }}
 
       - name: Setup mock_scihub_product_api cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: mock-scihub-product-api-cache
         with:
           path: /home/runner/.local/share/virtualenvs/mock_scihub_product_api-${{ steps.hashes.outputs.mockscihubproductapi }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/mock_scihub_product_api/Pipfile.lock') }}
 
       - name: Setup mock_scihub_search_api cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: mock-scihub-search-api-cache
         with:
           path: /home/runner/.local/share/virtualenvs/mock_scihub_search_api-${{ steps.hashes.outputs.mockscihubsearchapi }}
           key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/lambdas/mock_scihub_search_api/Pipfile.lock') }}
 
-      - name: Setup db cache
-        uses: actions/cache@v2
-        id: db-cache
-        with:
-          path: /home/runner/.local/share/virtualenvs/db-${{ steps.hashes.outputs.db }}
-          key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/layers/db/Pipfile.lock') }}
+      # - name: Setup db cache
+      #   uses: actions/cache@v3
+      #   id: db-cache
+      #   with:
+      #     path: /home/runner/.local/share/virtualenvs/db-${{ steps.hashes.outputs.db }}
+      #     key: ${{ hashFiles('/home/runner/work/hls-sentinel2-downloader-serverless/hls-sentinel2-downloader-serverless/layers/db/Pipfile.lock') }}
 
       - name: Install root dependencies
         if: steps.root-cache.outputs.cache-hit != 'true'
@@ -123,7 +127,7 @@ jobs:
           make -C lambdas/mock_scihub_search_api install
 
       - name: Install db dependencies
-        if: steps.db-cache.outputs.cache-hit != 'true'
+        # if: steps.db-cache.outputs.cache-hit != 'true'
         run: |
           make -C layers/db install
 
@@ -154,13 +158,14 @@ jobs:
       - name: Run unit tests
         run: |
           make unit-tests
+
   deploy:
     runs-on: ubuntu-20.04
     needs: [unit-tests]
     environment:
       name: prod
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Create .env
         run: |
@@ -174,30 +179,30 @@ jobs:
           UPLOAD_BUCKET="${{ secrets.UPLOAD_BUCKET }}"
           EOF
 
-      - name: Set up Python 3.8.6
-        uses: actions/setup-python@v2
+      - name: Set up Python 3.8.18
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8.6
+          python-version: 3.8.18
 
       - name: Setup up Node
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4
         with:
-          node-version: 12
+          node-version-file: ".nvmrc"
 
       - name: Install AWS CDK
         run: |
           npm install -g aws-cdk
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
 
       - name: Get pipenv venv hashes
         id: hashes
         run: |
-          echo "##[set-output name=root;]$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/Pipfile)"
+          echo "root=$(python -c 'import sys; import base64; import hashlib; print(base64.urlsafe_b64encode(hashlib.sha256(sys.argv[-1].encode()).digest()[:6]).decode()[:8])' $(pwd)/Pipfile)" >> $GITHUB_OUTPUT
 
       - name: Setup root cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         id: root-cache
         with:
           path: /home/runner/.local/share/virtualenvs/hls-sentinel2-downloader-serverless-${{ steps.hashes.outputs.root }}
@@ -209,7 +214,7 @@ jobs:
           pipenv install --dev
 
       - name: Configure awscli
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.gitignore

@@ -91,6 +91,10 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
+# Ignoring until migration away from pipenv is complete because layers/db/Makefile
+# still runs `pipenv` for the `test` rule, so it ends up generating this Pipfile.
+layers/db/Pipfile
+
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 

.nvmrc

@@ -1 +1 @@
-12
+18.17.1

Makefile

@@ -1,4 +1,28 @@
-.PHONEY: install lint format diff deploy destroy diff-integration deploy-integration destroy-integration unit-tests integration-tests
+SHELL := $(shell which bash)
+
+.PHONEY:
+	clean
+	deploy
+	deploy-integration
+	destroy
+	destroy-integration
+	diff
+	diff-integration
+	format
+	install
+	integration-tests
+	lint
+	unit-tests
+
+clean:
+	pipenv --rm || true
+	$(MAKE) -C layers/db clean
+	$(MAKE) -C lambdas/link_fetcher clean
+	$(MAKE) -C lambdas/date_generator clean
+	$(MAKE) -C lambdas/downloader clean
+	$(MAKE) -C lambdas/mock_scihub_search_api clean
+	$(MAKE) -C lambdas/mock_scihub_product_api clean
+	$(MAKE) -C alembic_migration clean
 
 install:
 	$(MAKE) -C layers/db install
@@ -34,22 +58,22 @@ format:
 	$(MAKE) -C alembic_migration format
 
 diff:
-	pipenv run npx cdk diff --app cdk/app.py || true
+	tox -e dev -- diff --app cdk/app.py || true
 
 deploy:
-	pipenv run npx cdk deploy --app cdk/app.py --require-approval never
+	tox -e dev -- deploy --app cdk/app.py --require-approval never
 
 destroy:
-	pipenv run npx cdk destroy --app cdk/app.py --force
+	tox -e dev -- destroy --app cdk/app.py --force
 
 diff-integration:
-	pipenv run npx cdk diff '*' --app cdk/app_integration.py || true
+	tox -e dev -- diff '*' --app cdk/app_integration.py || true
 
 deploy-integration:
-	pipenv run npx cdk deploy '*' --app cdk/app_integration.py --require-approval never
+	tox -e dev -- deploy '*' --app cdk/app_integration.py --require-approval never
 
 destroy-integration:
-	pipenv run npx cdk destroy '*' --app cdk/app_integration.py --force
+	tox -e dev -- destroy '*' --app cdk/app_integration.py --force
 
 unit-tests:
 	$(MAKE) -C lambdas/link_fetcher test
@@ -61,4 +85,29 @@ unit-tests:
 	$(MAKE) -C alembic_migration test
 
 integration-tests:
-	pipenv run pytest -s integration_tests
+	tox -e integration
+
+#-------------------------------------------------------------------------------
+# For invocation via tox only (i.e., in tox.ini commands entries)
+#-------------------------------------------------------------------------------
+
+tox:
+	@if [[ -z $${TOX_ENV_DIR+x} ]]; then echo "ERROR: For tox.ini use only" >&2; exit 1; fi
+
+# Install node in the virtualenv, if it's not installed or it's the wrong version.
+# Pull node version from .nvmrc file.
+install-node: tox
+	@NODE_VERSION=$$(<.nvmrc); \
+	if [[ ! $$(type node 2>/dev/null) =~ $${VIRTUAL_ENV} || ! $$(node -v) =~ $${NODE_VERSION} ]]; then \
+	    nodeenv --node $${NODE_VERSION} --python-virtualenv; \
+	fi
+
+# Install cdk in the virtualenv, if it's not installed or it's the wrong version.
+# Pull CDK version from value of aws_cdk_version variable in setup.py.
+install-cdk: install-node
+	@CDK_VERSION=$$(grep "aws_cdk_version = " setup.py | sed -Ee 's/aws_cdk_version = "(.+)"/\1/'); \
+	if [[ ! $$(type cdk 2>/dev/null) =~ $${VIRTUAL_ENV} || ! $$(cdk --version) =~ $${CDK_VERSION} ]]; then \
+	    npm install --location global "aws-cdk@$${CDK_VERSION}"; \
+	fi
+	@# Acknowledge CDK notice regarding CDK v1 being in maintenance mode.
+	@grep -q 19836 cdk.context.json 2>/dev/null || cdk acknowledge 19836

Pipfile

@@ -6,34 +6,9 @@ name = "pypi"
 [packages]
 
 [dev-packages]
-flake8 = "==3.8.4"
-isort = "==5.6.4"
-black = "==20.8b1"
-"aws-cdk.core" = "==1.105.0"
-"aws-cdk.aws-lambda" = "==1.105.0"
-"aws-cdk.aws-lambda-python" = "==1.105.0"
-"aws-cdk.aws-secretsmanager" = "==1.105.0"
-"aws-cdk.aws-sqs" = "==1.105.0"
-"aws-cdk.aws-ec2" = "==1.105.0"
-"aws-cdk.aws-rds" = "==1.105.0"
-"aws-cdk.aws-stepfunctions" = "==1.105.0"
-"aws-cdk.aws-stepfunctions-tasks" = "==1.105.0"
-"aws-cdk.aws-events" = "==1.105.0"
-"aws-cdk.aws-events-targets" = "==1.105.0"
-"aws-cdk.aws-apigateway" = "==1.105.0"
-"aws-cdk.aws-ssm" = "==1.105.0"
-"aws-cdk.aws-s3" = "==1.105.0"
-"aws-cdk.aws-lambda-event-sources" = "==1.105.0"
-"aws-cdk.aws-logs" = "==1.105.0"
-db = {editable = true, path = "./layers/db"}
-psycopg2 = "==2.8.6"
-boto3 = "==1.16.60"
-pytest = "==6.2.1"
-polling2 = "==0.4.6"
-assertpy = "==1.1"
+flake8 = "==6.1.0"
+isort = "==5.12.0"
+black = "==23.11.0"
 
 [requires]
 python_version = "3.8"
-
-[pipenv]
-allow_prereleases = true