Latest changes - EC2 deployment

.dockerignore

@@ -0,0 +1,4 @@
+**/__pycache__/
+*.py[cod]
+src/data/
+.DS_Store

.env.example

@@ -0,0 +1,12 @@
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+EA_FASTAPI_S3_URL=s3://fcx-downloader/
+EA_FASTAPI_S3_BUCKET=fcx-downloader
+EA_FASTAPI_S3_OBJECT_URL=https://fcx-downloader.s3.amazonaws.com/
+EA_FASTAPI_CLOUDFRONT_URL=https://d1ntjpytjzo3fx.cloudfront.net
+EA_FASTAPI_FILE_DOWNLOAD_PATH=
+
+MACHINE=urs.earthdata.nasa.gov
+EARTHDATA_USERNAME=
+EARTHDATA_PASSWORD=

.gitignore

@@ -1 +1,5 @@
-folder
+.env
+**/__pycache__/
+*.py[cod]
+src/data/
+.DS_Store

Dockerfile

@@ -0,0 +1,15 @@
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.9.1
+RUN apt-get update && apt-get -y install python3-pip --fix-missing
+COPY requirements.txt .
+COPY requirements-additional.txt .
+# RUN ln -s /usr/include/hdf5/serial /usr/include/hdf5/include
+# RUN export HDF5_DIR=/usr/include/hdf5
+# RUN pip install versioned-hdf5 --break-system-packages
+RUN apt-get -y install pkg-config libhdf5-dev
+RUN apt-get -y install python3.12-venv
+RUN pip install --no-binary=h5py h5py --break-system-packages
+RUN pip install -r requirements.txt --break-system-packages
+RUN pip install -r requirements-additional.txt --break-system-packages
+COPY . .
+EXPOSE 8000
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -0,0 +1,30 @@
+# ea-fastapi
+
+In this repo, FastAPI is used as a wrapper around EarthAccess library. There are 4 endpoints defined which can be validated in Postman for tracking,
+1. start_download - Downloads files from search result locally and uploads them to S3. Returns a unique job id of 8 characters.
+2. status - Returns latest status of the job.
+3. get_file_path - Returns S3 path of the uploaded granules.
+4. get_metadata - Displays the metadata of granules.
+
+## Pre-requisites
+
+Accounts in AWS and EarthData are necessary for using this wrapper application. The credentials for these accounts should be entered in .env file.
+Postman and Docker needs to be installed.
+
+## Steps to Use
+
+Run the following commands:
+
+`docker build -t <image-name> .`
+
+`docker run -p 8000:8000 <image-name>`
+
+Once the Docker container is up and running, the following endpoints can be used to send GET/POST/PUT requests from http://0.0.0.0:8000/:
+
+| Request type | Endpoint | Parameters |
+| --- | --- | --- |
+| GET | / | N/A |
+| PUT | /start_download | short_name, date_range, bounding_box(coordinates can be entered manually or passed as a text in Request Body), concept_id(optional) |
+| POST | /status | job_id |
+| POST | /get_file_path | job_id |
+| POST | /get_metadata | job_id |

build_and_docker.sh

@@ -0,0 +1,4 @@
+docker build  -t fcx-fastapi .
+docker stop fcx-fastapi-container
+docker rm fcx-fastapi-container
+docker run -d --name fcx-fastapi-container -p80:80 fcx-fastapi

config.py

@@ -0,0 +1,14 @@
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+s3Url = os.environ.get('EA_FASTAPI_S3_URL')
+cloudfrontUrl = os.environ.get('EA_FASTAPI_CLOUDFRONT_URL')
+s3BucketName = os.environ.get('EA_FASTAPI_S3_BUCKET')
+s3ObjectUrl = os.environ.get('EA_FASTAPI_S3_OBJECT_URL')
+fileDownloadPath = os.environ.get('EA_FASTAPI_FILE_DOWNLOAD_PATH')
+aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID')
+aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+username = os.environ.get('EARTHDATA_USERNAME')
+password = os.environ.get('EARTHDATA_PASSWORD')

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.8'
+services:
+  api:
+    container_name: "docker-fastapi"
+    build: .
+    ports:
+      - 8000:8000
+    volumes:
+      - .:/usr/src/app

main.py

@@ -0,0 +1,81 @@
+import random
+from typing import Any, Dict
+from fastapi import Body, FastAPI, Query, WebSocket
+import string
+
+from starlette.middleware import Middleware
+from starlette.middleware.cors import CORSMiddleware
+
+from src import *
+from src.models.job_model import Job, Metadata, Coord
+from src.login_handler import login_handler
+from src.websocket_handler import ConnectionManager
+from src.root_handler import root_handler
+from http import HTTPStatus
+from fastapi import BackgroundTasks
+from src.websocket_handler import *
+from src.download_handler import *
+from src.file_path_handler import *
+from src.status_handler import *
+from src.metadata_handler import *
+
+# Enable CORS for all origins
+middleware = [
+    Middleware(
+        CORSMiddleware,
+        allow_origins=['*'],
+        allow_credentials=True,
+        allow_methods=['*'],
+        allow_headers=['*']
+    )
+]
+
+# Dict as job storage
+jobs: Dict[str, Job] = {}
+app = FastAPI(middleware=middleware)
+
+# Authenticate user and allow login
+login_handler()
+
+# Root endpoint
+@app.get("/")
+async def root():
+    return await root_handler()
+
+manager = ConnectionManager()
+
+# Establish WebSocket connection
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    return await websocket_endpoint_handler(websocket, manager)
+
+# Start a download job in the background
+@app.put("/start_download", status_code=HTTPStatus.ACCEPTED)
+async def start_download(background_tasks: BackgroundTasks, short_name: str, date_range: list = Query([]) , concept_id: str | None = None, bounding_box: list | None = Query(None), isPangeoForge: bool | None = None):
+
+    # Create a new Job instance and add it to the jobs dictionary
+    # Job should always start with a letter because we use this for pangeo forge jobid which has this specific requirement
+    current_job = Job()
+    key1 = random.choices(string.ascii_lowercase, k=1)[0]
+    key2 = ''.join(random.choices(string.ascii_lowercase + string.digits, k=7))
+    current_job.uid = key1+key2
+    jobs[current_job.uid] = current_job
+
+    # Add the download job to the background tasks
+    background_tasks.add_task(download_handler, current_job, short_name, tuple(date_range), concept_id, bounding_box, manager, isPangeoForge)
+    return current_job.uid
+
+# Check the status of a job
+@app.post("/status")
+async def status(uid: str):
+    return await status_handler(uid, jobs)
+
+# Get the files associated with a job
+@app.post("/get_file_path")
+async def get_file_path(uid: str):
+    return await file_path_handler(uid, jobs)
+
+# Display metadata associated with a job(only earthaccess)
+@app.post("/get_metadata")
+async def get_metadata(uid: str):
+    return await metadata_handler(uid, jobs)

requirements-additional.txt

@@ -0,0 +1,5 @@
+boto3
+dask==2024.3.1
+pangeo-forge-earthdatalogin==0.2
+pangeo-forge-recipes==0.10.5
+pangeo-forge-runner==0.10.2

requirements.txt

@@ -0,0 +1,7 @@
+fastapi[all]==0.104.1 #includes websocket, uvicorn
+earthaccess==0.7.1
+fiona==1.9.5
+xarray==2023.8.0
+geopandas==0.14.1
+python-dotenv==1.0.0
+h5netcdf==1.3.0

src/bake.sh

@@ -0,0 +1 @@
+pangeo-forge-runner bake --config=$CONFIG_FILE --repo=$REPO --Bake.job_name=$JOB_NAME --prune