From e4b625abe08a45095b98caa24cb9c33ae7676fbc Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 May 2024 21:21:32 -0400 Subject: [PATCH 1/7] add arm64 platform flag --- .github/workflows/aws-ecs-deploy-backend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/aws-ecs-deploy-backend.yml b/.github/workflows/aws-ecs-deploy-backend.yml index 6557f1b2..cb8abcaf 100644 --- a/.github/workflows/aws-ecs-deploy-backend.yml +++ b/.github/workflows/aws-ecs-deploy-backend.yml @@ -80,6 +80,7 @@ jobs: tags: latest,${{ needs.setup_env.outputs.IMAGE_TAG }} dockerfile: ${{ env.DOCKERFILE }} path: ${{ env.DOCKER_PATH }} + extra_build_args: "--platform=linux/arm64" deploy: name: Deploy to AWS ECS runs-on: ubuntu-latest From 4b90de525e201038a71bd2c5376dced9ea50235c Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 May 2024 21:49:42 -0400 Subject: [PATCH 2/7] use custom action --- .github/workflows/aws-ecs-deploy-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-ecs-deploy-backend.yml b/.github/workflows/aws-ecs-deploy-backend.yml index cb8abcaf..7800d378 100644 --- a/.github/workflows/aws-ecs-deploy-backend.yml +++ b/.github/workflows/aws-ecs-deploy-backend.yml @@ -70,7 +70,7 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - name: Build & Push Image to ECR - uses: kciter/aws-ecr-action@v5 + uses: mikeyavorsky/aws-ecr-action@v5 with: access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 736a591d64686ed7f6f755c6b43afabec51084b2 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 14 May 2024 21:51:02 -0400 Subject: [PATCH 3/7] update ref --- .github/workflows/aws-ecs-deploy-backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws-ecs-deploy-backend.yml b/.github/workflows/aws-ecs-deploy-backend.yml index 7800d378..c726b773 100644 --- a/.github/workflows/aws-ecs-deploy-backend.yml +++ b/.github/workflows/aws-ecs-deploy-backend.yml @@ -70,7 +70,7 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - name: Build & Push Image to ECR - uses: mikeyavorsky/aws-ecr-action@v5 + uses: mikeyavorsky/aws-ecr-action@master with: access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 6ac637f428d1827e3d2c5d5928cc76b1a8144465 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Thu, 30 May 2024 23:47:47 -0400 Subject: [PATCH 4/7] use buildx to create both amd64 and arm64 images --- .github/workflows/aws-ecs-deploy-backend.yml | 30 +++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/aws-ecs-deploy-backend.yml b/.github/workflows/aws-ecs-deploy-backend.yml index c726b773..3da1ac3f 100644 --- a/.github/workflows/aws-ecs-deploy-backend.yml +++ b/.github/workflows/aws-ecs-deploy-backend.yml @@ -20,6 +20,7 @@ env: AWS_REGION: us-east-1 DOCKERFILE: ./backend/Dockerfile.cloud DOCKER_PATH: ./ + PLATFORMS: linux/amd64,linux/arm64 jobs: setup_env: name: Set-up environment @@ -66,21 +67,22 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ env.AWS_REGION }} - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Build & Push Image to ECR - uses: mikeyavorsky/aws-ecr-action@master + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and push + uses: docker/build-push-action@v5 with: - access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - account_id: ${{ secrets.AWS_ACCOUNT_ID }} - repo: ${{ needs.setup_env.outputs.AWS_APPENV }} - region: ${{ env.AWS_REGION }} - tags: latest,${{ needs.setup_env.outputs.IMAGE_TAG }} - dockerfile: ${{ env.DOCKERFILE }} - path: ${{ env.DOCKER_PATH }} - extra_build_args: "--platform=linux/arm64" + context: ${{ env.DOCKER_PATH }} + file: ${{ env.DOCKERFILE }} + platforms: ${{ env.PLATFORMS }} + push: true + tags: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ needs.setup_env.outputs.AWS_APPENV }}:${{ needs.setup_env.outputs.IMAGE_TAG }}, ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com/${{ needs.setup_env.outputs.AWS_APPENV }}:latest deploy: name: Deploy to AWS ECS runs-on: ubuntu-latest From 1e998bcbb18d49058ac7e7ec20ae90bea32d8e4e Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Fri, 31 May 2024 01:18:04 -0400 Subject: [PATCH 5/7] use wheel for pandas --- backend/Dockerfile.cloud | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile.cloud b/backend/Dockerfile.cloud index 4732b603..a391bd16 100644 --- a/backend/Dockerfile.cloud +++ b/backend/Dockerfile.cloud @@ -3,14 +3,20 @@ FROM python:3-slim-bullseye # required for pandas -RUN apt-get update -y && apt-get install gcc g++ python3-dev -y +RUN apt-get update -y && apt-get install gcc g++ python3-dev wget -y WORKDIR /app/ ARG PDT_API_PORT=5000 COPY requirements/prod.txt . -RUN pip install -r prod.txt +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN arch=$(arch) && \ + file=pandas-2.2.2-cp312-cp312-manylinux_2_17_${arch}.manylinux2014_${arch}.whl && \ + url="https://files.pythonhosted.org/packages/b0/85/f95b5f322e1ae13b7ed7e97bd999160fa003424711ab4dc8344b8772c270/${file}" && \ + wget ${url} && \ + sed -i "s/pandas==1.5.3/${file}/" prod.txt +RUN pip install --no-cache-dir -r prod.txt COPY . . From 983d077d97902a53bdeb66039851152fa83e80d8 Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Fri, 31 May 2024 01:29:20 -0400 Subject: [PATCH 6/7] use debian mirror --- backend/Dockerfile.cloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Dockerfile.cloud b/backend/Dockerfile.cloud index a391bd16..4e444ec9 100644 --- a/backend/Dockerfile.cloud +++ b/backend/Dockerfile.cloud @@ -13,7 +13,7 @@ COPY requirements/prod.txt . SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN arch=$(arch) && \ file=pandas-2.2.2-cp312-cp312-manylinux_2_17_${arch}.manylinux2014_${arch}.whl && \ - url="https://files.pythonhosted.org/packages/b0/85/f95b5f322e1ae13b7ed7e97bd999160fa003424711ab4dc8344b8772c270/${file}" && \ + url="https://pypi.debian.net/pandas/${file}" && \ wget ${url} && \ sed -i "s/pandas==1.5.3/${file}/" prod.txt RUN pip install --no-cache-dir -r prod.txt From 91635c5981fccb1eaec62636eb042cd03c4f7ecf Mon Sep 17 00:00:00 2001 From: Mike Yavorsky Date: Tue, 4 Jun 2024 19:27:10 -0400 Subject: [PATCH 7/7] update pandas to 2.2.2 in requirements, compile --- requirements/_core.in | 2 +- requirements/dev_unix.txt | 8 +++++--- requirements/dev_windows.txt | 8 +++++--- requirements/docs.txt | 2 +- requirements/prod.txt | 8 +++++--- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/requirements/_core.in b/requirements/_core.in index 599ac202..3a078d02 100644 --- a/requirements/_core.in +++ b/requirements/_core.in @@ -5,7 +5,7 @@ flake8 flask flask-sqlalchemy gunicorn -pandas +pandas==2.2.2 pip-tools pydantic pydantic-sqlalchemy diff --git a/requirements/dev_unix.txt b/requirements/dev_unix.txt index e988198d..df3a1923 100644 --- a/requirements/dev_unix.txt +++ b/requirements/dev_unix.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements/dev_unix.in @@ -308,7 +308,7 @@ packaging==24.0 # pytest-flask-sqlalchemy # qtconsole # qtpy -pandas==1.5.3 +pandas==2.2.2 # via -r requirements/_core.in pandocfilters==1.5.1 # via nbconvert @@ -523,7 +523,9 @@ typing-extensions==4.10.0 # psycopg # pydantic tzdata==2024.1 - # via celery + # via + # celery + # pandas ua-parser==0.18.0 # via -r requirements/_core.in uri-template==1.3.0 diff --git a/requirements/dev_windows.txt b/requirements/dev_windows.txt index 7fbbd1d5..d24228c7 100644 --- a/requirements/dev_windows.txt +++ b/requirements/dev_windows.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements/dev_windows.in @@ -308,7 +308,7 @@ packaging==24.0 # pytest-flask-sqlalchemy # qtconsole # qtpy -pandas==1.5.3 +pandas==2.2.2 # via -r requirements/_core.in pandocfilters==1.5.1 # via nbconvert @@ -523,7 +523,9 @@ typing-extensions==4.10.0 # psycopg # pydantic tzdata==2024.1 - # via celery + # via + # celery + # pandas ua-parser==0.18.0 # via -r requirements/_core.in uri-template==1.3.0 diff --git a/requirements/docs.txt b/requirements/docs.txt index fbb19a5a..34bd163a 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements/docs.in diff --git a/requirements/prod.txt b/requirements/prod.txt index 032bac02..15ecca6f 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements/prod.in @@ -308,7 +308,7 @@ packaging==24.0 # pytest-flask-sqlalchemy # qtconsole # qtpy -pandas==1.5.3 +pandas==2.2.2 # via -r requirements/_core.in pandocfilters==1.5.1 # via nbconvert @@ -523,7 +523,9 @@ typing-extensions==4.10.0 # psycopg # pydantic tzdata==2024.1 - # via celery + # via + # celery + # pandas ua-parser==0.18.0 # via -r requirements/_core.in uri-template==1.3.0