Merge pull request #24 from KarmaComputing/23-containerization

wip #23 containerization

.env.example

@@ -1,11 +1,17 @@
-SECRET_KEY=changeme
-DEBUG=True
-UPLOAD_FOLDER="/path/to/upload/folder"
-PORT=25
-DOMAIN=127.0.0.1:5000
-SERVER_NAME=example.com
-SMTP_SERVER=email.example.com
-SENDER_EMAIL=[email protected]
-PASSWORD=changeme
-receiver_email_1=[email protected]
-STRIPE_SECRET_KEY=changeme
+SECRET_KEY="ASDSADASDASDASDadsf"
+DEBUG="True"
+APP_UPLOAD_FOLDER="/ecw-to-tiff/uploads"
+DOCKER_SOCKET="/var/run/docker.sock"
+DOCKER_LIB="/usr/bin/docker"
+# change to absolute path where your upload folder inside the app folder is.
+UPLOAD_FOLDER_MOUNT_PATH="/path/to/ecw-to-tiff-conversion/app/uploads"
+SEND_EMAIL="False"
+
+# remove the comments below only if SEND_EMAIL="TRUE"
+#DOMAIN=127.0.0.1:8888
+#PORT=25
+#SMTP_SERVER=email.example.com
+#[email protected]
+#PASSWORD="changeme"
+#receiver_email_1="[email protected]"
+#receiver_email_2="[email protected]"

.gitignore

@@ -2,3 +2,4 @@ __pycache__
 venv/
 .env
 src/uploads
+app/uploads

README.md

@@ -2,9 +2,7 @@
 
 ## How to run it
 ```
-python3 -m venv venv
-. venv/bin/activate
-pip install -r requirements.txt
+# change UPLOAD_FOLDER_MOUNT_PATH to the right setting
 cp .env.example .env
-FLASK_DEBUG=1 flask run
+docker-compose up --build
 ```

app/.dockerignore

@@ -0,0 +1 @@
+uploads/*

app/Dockerfile

@@ -0,0 +1,19 @@
+# Use Ubuntu 20.04 as the base image
+FROM ubuntu:20.04
+
+# Update the package index and install necessary dependencies
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y python3.9 python3.9-dev python3.9-distutils python3-pip
+
+# Make sure pip is up to date
+RUN python3.9 -m pip install --upgrade pip
+
+USER root
+WORKDIR /ecw-to-tiff
+COPY ./ /ecw-to-tiff
+RUN pip3 install -r requirements.txt
+ENTRYPOINT ["/ecw-to-tiff/entrypoint.sh"]
+

app.py → app/app.py

@@ -1,5 +1,6 @@
 import os
 import hashlib
+import stripe
 from flask import (
     render_template,
     Flask,
@@ -20,15 +21,16 @@
     send_notification_upload_email,
     send_email_to_admin,
 )  # noqa: E501
-import stripe
 
 load_dotenv()
 
 SERVER_NAME = os.getenv("SERVER_NAME")
-UPLOAD_FOLDER = os.getenv("UPLOAD_FOLDER")
+UPLOAD_FOLDER = os.getenv("APP_UPLOAD_FOLDER")
+UPLOAD_FOLDER_MOUNT_PATH = os.getenv("UPLOAD_FOLDER_MOUNT_PATH")
 STRIPE_SECRET_KEY = os.getenv("STRIPE_SECRET_KEY")
 ALLOWED_EXTENSIONS_TIFF = {".tiff", ".tif"}
 ALLOWED_EXTENSIONS_ECW = {".ecw"}
+SEND_EMAIL = os.getenv("SEND_EMAIL")
 
 app = Flask(__name__)
 app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
@@ -175,20 +177,22 @@ def convert_ecw_to_cog(app=None, filename=None, email=None):
     print(f"Running background ecw to cog on {filename}")
     with app.app_context():
         subprocess.run(
-            f"./ecw-to-COG.sh {filename} {UPLOAD_FOLDER}",
+            f"./ecw-to-COG.sh {filename} '{UPLOAD_FOLDER_MOUNT_PATH}'",
             shell=True,
         )
-    send_email(email, filename)
-    send_email_to_admin(email, filename)
+    if SEND_EMAIL == "True":
+        send_email(email, filename)
+        send_email_to_admin(email, filename)
 
 
 @background_task
 def convert_tif_to_cog(app=None, filename=None, email=None):
     print(f"Running background tif to cog on {filename}")
     with app.app_context():
         subprocess.run(
-            f"./tif-to-COG.sh {filename} {UPLOAD_FOLDER}",
+            f"./tif-to-COG.sh {filename} '{UPLOAD_FOLDER_MOUNT_PATH}'",
             shell=True,
         )
-    send_email(email, filename)
-    send_email_to_admin(email, filename)
+    if SEND_EMAIL == "True":
+        send_email(email, filename)
+        send_email_to_admin(email, filename)

app/ecw-to-COG.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -exu -o pipefail
+
+MAP_TO_CONVERT=$1
+UPLOAD_FOLDER=$2
+
+# Taking out the extension name
+COG_MAP=`echo $MAP_TO_CONVERT | cut -d '.' -f 1`
+
+docker run --rm -v $UPLOAD_FOLDER:/data geodata/gdal:latest /bin/bash -c "gdal_translate -of GTiff -co COMPRESS=DEFLATE -co PREDICTOR=2 -co ZLEVEL=9 -co BIGTIFF=YES $MAP_TO_CONVERT $COG_MAP'_converted.tiff'"

run.sh → app/entrypoint.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
+set -ex
 
 flask run --host 0.0.0.0 --port 8888

requirements.txt → app/requirements.txt

@@ -1,3 +1,4 @@
 Flask==2.2.2
 python-dotenv==0.21.0
+Werkzeug==2.2.2
 stripe

app/tif-to-COG.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -exu -o pipefail
+
+MAP_TO_CONVERT=$1
+UPLOAD_FOLDER=$2
+
+# Taking out the extension name
+COG_MAP=$(echo "$MAP_TO_CONVERT" | cut -d '.' -f 1)
+
+docker run --rm -v "$UPLOAD_FOLDER":/data osgeo/gdal:latest /bin/bash -c "cd data; gdal_translate -of COG -co NUM_THREADS=ALL_CPUS  -co LEVEL=9 -co COMPRESS=DEFLATE -co BIGTIFF=YES -co PREDICTOR=2 $MAP_TO_CONVERT $COG_MAP'_converted.tiff'"
+

docker-compose.yml

@@ -0,0 +1,12 @@
+version: "3.9"
+services:
+  ecw-to-tiff:
+    build: ./app
+    volumes:
+      - "${UPLOAD_FOLDER_MOUNT_PATH}:/ecw-to-tiff/uploads"
+      - "${DOCKER_SOCKET}:/var/run/docker.sock"
+      - "${DOCKER_BINARY}:/usr/local/bin/docker"
+    ports: 
+      - 8888:8888
+    env_file:
+      - .env