From 4242bfe968fbfe5e15550526e488d9e928a14a05 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Sat, 25 Aug 2018 19:47:49 +0800
Subject: [PATCH 01/22] initial

---
 Dockerfile | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 Dockerfile

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..7a3051d
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,7 @@
+FROM python:2.7-alpine
+
+RUN apk add --no-cache curl git make zip python jq
+RUN apk add --update gcc g++
+RUN apk add --update bash && rm -rf /var/cache/apk/*
+RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
+RUN source /etc/profile; pip install autopep8; pip install pylint; pip install sphinx; pip install sphinx-rtd-theme
\ No newline at end of file

From 161f8295e22930d880b63014827f4c3abbba7db9 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Sat, 25 Aug 2018 20:21:54 +0800
Subject: [PATCH 02/22] change into vanila alpine

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7a3051d..7bbb7cb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
-FROM python:2.7-alpine
+FROM alpine:3.1
 
-RUN apk add --no-cache curl git make zip python jq
+RUN apk add --update curl git make zip python jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user

From 607be57ad13c8f92aa1458a12edd953c185d321c Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Sat, 25 Aug 2018 20:34:19 +0800
Subject: [PATCH 03/22] install pyspark and awsglue library

---
 Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7bbb7cb..7090b8a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,8 @@
 FROM alpine:3.1
 
-RUN apk add --update curl git make zip python jq
+RUN apk add --update curl gift make zip python jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install autopep8; pip install pylint; pip install sphinx; pip install sphinx-rtd-theme
\ No newline at end of file
+RUN source /etc/profile; pip install autopep8; pip install pylint; pip install sphinx; pip install sphinx-rtd-theme
+RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From d9ec268697facd49ae8c9172c432a3a1a12143df Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Fri, 31 Aug 2018 15:23:30 +0800
Subject: [PATCH 04/22] coverage py, and enforce python version

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 7090b8a..0d8a10e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
 FROM alpine:3.1
 
-RUN apk add --update curl gift make zip python jq
+RUN apk add --update curl make zip python=2.7.12-r0 jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install autopep8; pip install pylint; pip install sphinx; pip install sphinx-rtd-theme
+RUN source /etc/profile; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme
 RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 4b5f95c8adec0981313be82e50b341554f845002 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Fri, 31 Aug 2018 15:46:31 +0800
Subject: [PATCH 05/22] jdk8 and python 2.7.15

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0d8a10e..769ea66 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
-FROM alpine:3.1
+FROM anapsix/alpine-java:8u181b13_jdk-dcevm
 
-RUN apk add --update curl make zip python=2.7.12-r0 jq
+RUN apk add --update curl make zip python=2.7.15-r2 jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user

From b9323e29532420e5d0d66e9cf71e61ac892fd329 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Fri, 31 Aug 2018 16:33:36 +0800
Subject: [PATCH 06/22] pip virtualenv

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 769ea66..01877ae 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme
 RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 189bfa2243070df6e94cfb60deb4286895a99c69 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Sun, 2 Sep 2018 22:18:58 +0800
Subject: [PATCH 07/22] add pandas

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 01877ae..743191e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas
 RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 01aa840dac90bda6b049018763a9a3e60eee4e93 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Sun, 2 Sep 2018 22:59:56 +0800
Subject: [PATCH 08/22] add python-dev

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 743191e..56d90c7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install python-dev; pip install pandas
 RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From f724c6bf795e78f87d4c1a71d0e4bac36ecb623d Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Sun, 2 Sep 2018 23:03:57 +0800
Subject: [PATCH 09/22] apk python-dev

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 56d90c7..40df5a5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
 FROM anapsix/alpine-java:8u181b13_jdk-dcevm
 
-RUN apk add --update curl make zip python=2.7.15-r2 jq
+RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install python-dev; pip install pandas
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas
 RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 25d829adf5880b3bc7e5c8369a991c15cbc89aac Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Mon, 3 Sep 2018 18:42:40 +0800
Subject: [PATCH 10/22] re-define the pyspark into 2.2.1

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 40df5a5..6c55397 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,4 +5,4 @@ RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
 RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas
-RUN source /etc/profile; pip install pyspark; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file
+RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From ac14c3ec530f1e7b90cdc464a6b7320f2f4174e1 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Tue, 4 Sep 2018 15:23:56 +0800
Subject: [PATCH 11/22] add boto3

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6c55397..6de9e85 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 6a16026142414244ecf3be86327828f8207dd572 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Thu, 6 Sep 2018 05:55:26 +0800
Subject: [PATCH 12/22] add pyarrow

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6de9e85..4c1fb3d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install pyarrow
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 498f3844e24c53ef86a4d32e2b047fec8c2a4530 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Thu, 6 Sep 2018 09:16:01 +0800
Subject: [PATCH 13/22] add cython

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 4c1fb3d..0b660c3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install pyarrow
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install cython; pip install pyarrow
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 9dd7f39509580bc428ac063ecb6c1d115c8f7efd Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Thu, 6 Sep 2018 10:39:39 +0800
Subject: [PATCH 14/22] remove cython and pyarrow

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 0b660c3..a4d2aa8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install cython; pip install pyarrow
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3;
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 07fb5f40d8d456ea158921e86a4cb4a7173d2107 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Thu, 6 Sep 2018 10:44:12 +0800
Subject: [PATCH 15/22] add cython, pyarrow, cmake

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a4d2aa8..643abcf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
 FROM anapsix/alpine-java:8u181b13_jdk-dcevm
 
-RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
+RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq cmake
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3;
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install cython; pip install pyarrow
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From d61cab9ca6faaefb1f3352c527a5f59dded42c77 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Thu, 6 Sep 2018 13:18:30 +0800
Subject: [PATCH 16/22] replace pyarrow with fastparquet

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 643abcf..6807c69 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,8 @@
 FROM anapsix/alpine-java:8u181b13_jdk-dcevm
 
-RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq cmake
+RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install cython; pip install pyarrow
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install fastparquet
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From 49526ddb4efe1773854217664f20b5b874e61258 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Thu, 6 Sep 2018 16:05:18 +0800
Subject: [PATCH 17/22] remove fastparquet, add py4j

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 6807c69..412fbfb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install fastparquet
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install py4j==0.10.4
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From a6e74db8ea1d1d90b515986e992f9feb53af9365 Mon Sep 17 00:00:00 2001
From: andreas <andreas@reddotpay.com>
Date: Tue, 11 Sep 2018 10:58:00 +0800
Subject: [PATCH 18/22] add s3pypi

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 412fbfb..b939a8b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,5 +4,5 @@ RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
 RUN apk add --update gcc g++
 RUN apk add --update bash && rm -rf /var/cache/apk/*
 RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install py4j==0.10.4
+RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install py4j==0.10.4; pip install s3pypi
 RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file

From c049a85d61f0aefd64099963970e843bb537d4e7 Mon Sep 17 00:00:00 2001
From: piadelosreyes <kristinapia@reddotpay.com>
Date: Tue, 15 Jan 2019 17:34:39 +0800
Subject: [PATCH 19/22] first commit

---
 Dockerfile | 44 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 38 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b939a8b..dc55e18 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,40 @@
 FROM anapsix/alpine-java:8u181b13_jdk-dcevm
 
-RUN apk add --update curl make zip python=2.7.15-r2 python-dev jq
-RUN apk add --update gcc g++
-RUN apk add --update bash && rm -rf /var/cache/apk/*
-RUN curl -O https://bootstrap.pypa.io/get-pip.py; python get-pip.py --user; echo "export PATH=\"\$PATH:/root/.local/bin\"" >> /etc/profile; source /etc/profile; pip install awscli --upgrade --user
-RUN source /etc/profile; pip install virtualenv; pip install autopep8; pip install pylint; pip install coverage; pip install sphinx; pip install sphinx-rtd-theme; pip install pandas; pip install boto3; pip install py4j==0.10.4; pip install s3pypi
-RUN source /etc/profile; pip install pyspark==2.2.1; curl -O https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip; unzip PyGlue.zip; cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
\ No newline at end of file
+ENV PYTHON_VERSION 2.7.15-r2
+ENV PYSPARK_VERSION 2.2.1
+
+ENV PIP get-pip.py
+ADD https://bootstrap.pypa.io/get-pip.py /tmp/
+
+ENV PYGLUE PyGlue.zip
+ADD https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip /tmp/
+
+ENV ROOT_PATH=${PATH}:/root/.local/bin
+
+RUN apk add --update --no-cache \
+  curl \
+  make \
+  zip \
+  python=$PYTHON_VERSION \
+  python-dev \
+  jq \
+  gcc \
+  g++ \
+  bash \
+ && rm -rf /var/cache/apk/*
+
+RUN cd /tmp/ && python /tmp/${PIP}
+RUN pip install awscli --upgrade --user \
+  virtualenv \
+  autopep8 \
+  pylint \
+  coverage \
+  sphinx \
+  sphinx-rtd-theme \
+  pandas \
+  boto3 \
+  py4j==0.10.4 \
+  s3pypi \
+  pyspark==$PYSPARK_VERSION
+
+RUN cd /opt/ && unzip /tmp/$PYGLUE && cp -R awsglue /usr/lib/python2.7/site-packages/awsglue

From ce28c6a0600f093d30b178cedff20554a7ce3093 Mon Sep 17 00:00:00 2001
From: piadelosreyes <kristinapia@reddotpay.com>
Date: Tue, 12 Feb 2019 14:59:48 +0800
Subject: [PATCH 20/22] installation of python dependencies

---
 Dockerfile | 19 ++++++++++++-------
 README.md  |  1 +
 2 files changed, 13 insertions(+), 7 deletions(-)
 create mode 100644 README.md

diff --git a/Dockerfile b/Dockerfile
index dc55e18..c2b8874 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,7 @@
-FROM anapsix/alpine-java:8u181b13_jdk-dcevm
+FROM reddotpay/docker-alpine-java:8u181b13_jdk-dcevm
+
+COPY . /rdp-pyspark
+WORKDIR /rdp-pyspark
 
 ENV PYTHON_VERSION 2.7.15-r2
 ENV PYSPARK_VERSION 2.2.1
@@ -9,7 +12,7 @@ ADD https://bootstrap.pypa.io/get-pip.py /tmp/
 ENV PYGLUE PyGlue.zip
 ADD https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip /tmp/
 
-ENV ROOT_PATH=${PATH}:/root/.local/bin
+ENV ROOT_PATH=${PATH}:~/.local/bin
 
 RUN apk add --update --no-cache \
   curl \
@@ -24,17 +27,19 @@ RUN apk add --update --no-cache \
  && rm -rf /var/cache/apk/*
 
 RUN cd /tmp/ && python /tmp/${PIP}
-RUN pip install awscli --upgrade --user \
-  virtualenv \
+RUN pip install --upgrade setuptools --user python \
+  awscli \
   autopep8 \
+  virtualenv \
   pylint \
   coverage \
   sphinx \
   sphinx-rtd-theme \
-  pandas \
   boto3 \
   py4j==0.10.4 \
   s3pypi \
-  pyspark==$PYSPARK_VERSION
+  pyspark==${PYSPARK_VERSION} \
+  pandas==0.23.4
 
-RUN cd /opt/ && unzip /tmp/$PYGLUE && cp -R awsglue /usr/lib/python2.7/site-packages/awsglue
+RUN cd /opt/ && unzip /tmp/${PYGLUE} && cp -R awsglue /usr/lib/python${PYTHON_VERSION}/site-packages/awsglue
+RUN rm -rf /tmp
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..3b8ed19
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+#### docker-pyrunner

From e8a986b0906af050894943197f90240b71a6e1d2 Mon Sep 17 00:00:00 2001
From: kelvin117 <kelvin117@redddotpay.com>
Date: Fri, 1 Mar 2019 11:11:36 +0800
Subject: [PATCH 21/22] modified for python3

---
 Dockerfile | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)
 mode change 100644 => 100755 Dockerfile

diff --git a/Dockerfile b/Dockerfile
old mode 100644
new mode 100755
index c2b8874..fe402bf
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,33 +1,28 @@
-FROM reddotpay/docker-alpine-java:8u181b13_jdk-dcevm
+FROM alpine:3.8
 
-COPY . /rdp-pyspark
-WORKDIR /rdp-pyspark
+WORKDIR /rdp-python3
 
-ENV PYTHON_VERSION 2.7.15-r2
-ENV PYSPARK_VERSION 2.2.1
+ENV PYTHON_VERSION 3.6.6-r0
 
 ENV PIP get-pip.py
 ADD https://bootstrap.pypa.io/get-pip.py /tmp/
 
-ENV PYGLUE PyGlue.zip
-ADD https://s3.amazonaws.com/aws-glue-jes-prod-us-east-1-assets/etl/python/PyGlue.zip /tmp/
-
 ENV ROOT_PATH=${PATH}:~/.local/bin
 
 RUN apk add --update --no-cache \
   curl \
   make \
   zip \
-  python=$PYTHON_VERSION \
-  python-dev \
+  python3=$PYTHON_VERSION \
+  python3-dev \
   jq \
   gcc \
   g++ \
   bash \
  && rm -rf /var/cache/apk/*
 
-RUN cd /tmp/ && python /tmp/${PIP}
-RUN pip install --upgrade setuptools --user python \
+RUN cd /tmp/ && python3 /tmp/${PIP}
+RUN pip install setuptools \
   awscli \
   autopep8 \
   virtualenv \
@@ -36,10 +31,7 @@ RUN pip install --upgrade setuptools --user python \
   sphinx \
   sphinx-rtd-theme \
   boto3 \
-  py4j==0.10.4 \
   s3pypi \
-  pyspark==${PYSPARK_VERSION} \
   pandas==0.23.4
 
-RUN cd /opt/ && unzip /tmp/${PYGLUE} && cp -R awsglue /usr/lib/python${PYTHON_VERSION}/site-packages/awsglue
 RUN rm -rf /tmp

From 7df305ed1279d4969d7140c846f9d3343b55bfb7 Mon Sep 17 00:00:00 2001
From: jerrypan <jerrypan@reddotpay.com>
Date: Fri, 15 May 2020 17:20:24 +0800
Subject: [PATCH 22/22] added new cicd workflow

---
 Dockerfile       |  2 ++
 getParamStore.sh | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 switchRole.sh    | 22 ++++++++++++++++++++++
 3 files changed, 71 insertions(+)
 create mode 100644 getParamStore.sh
 create mode 100644 switchRole.sh

diff --git a/Dockerfile b/Dockerfile
index fe402bf..cba07b2 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,7 @@
 FROM alpine:3.8
 
+COPY ["switchRole.sh", "getParamStore.sh", "./var/"]
+
 WORKDIR /rdp-python3
 
 ENV PYTHON_VERSION 3.6.6-r0
diff --git a/getParamStore.sh b/getParamStore.sh
new file mode 100644
index 0000000..cd6ad01
--- /dev/null
+++ b/getParamStore.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+#This script will generate a file env.sh
+#To use: bash getEnv.sh [path...]
+#Please execute
+# $ source env.sh
+#to apply environment variables
+
+if [ $# -eq 0 ]; then
+	echo "No arguments provided"
+	exit 1
+fi
+ARGUMENTS="$@"
+OUT_FILE="/var/env.sh"
+echo '#!/bin/sh' > $OUT_FILE
+
+# non greedy capture group 1, | greedy capture till end of string
+
+# use for mac os
+# SED_COMMAND="sed \"s^\([^\|]*\)\|\(.*\)^export \\1='\\2'^\""
+
+# use for aws linux
+SED_COMMAND="sed \"s^\([^|]*\)|\(.*\)^export \\1='\\2'^\""
+
+function exportToFile() {
+	old_IFS=$IFS
+	#set delimiter to only newline
+	IFS=$'\n'
+	for param in `jq -n "$1" | jq -r '.[].Value'`
+	do
+		# echo $param
+		param=$(sed "s^'^'\\\''^" <<< $param) #escape all single quotes
+		# echo $param
+		param=`eval $SED_COMMAND <<< $param`
+		# echo "$param"
+		echo $param >> $OUT_FILE
+		# echo "echo $val" >> $OUT_FILE
+	done
+	#restore delimiter
+	IFS=$old_IFS
+}
+
+for var in $ARGUMENTS
+do
+	PARAMSTORE=`aws ssm get-parameters-by-path --path "$var" --no-paginate --recursive | jq -r '.Parameters'`
+	exportToFile "$PARAMSTORE"
+done
diff --git a/switchRole.sh b/switchRole.sh
new file mode 100644
index 0000000..3ccf7e4
--- /dev/null
+++ b/switchRole.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+if [ $# -ne 2 ]; then
+	echo "Error! Too many arguments provided\nsource /var/switchRole.sh <AWS_ROLE> <AWS_REGION>"
+	exit 1
+fi
+
+echo "Start of Switch Role"
+
+unset AWS_ACCESS_KEY_ID
+unset AWS_SECRET_ACCESS_KEY
+unset AWS_SESSION_TOKEN
+unset AWS_DEFAULT_REGION
+
+# switch role
+aws sts assume-role --role-arn $1 --role-session-name rdp-develop-access > assume-role-output.txt
+export AWS_ACCESS_KEY_ID=`cat assume-role-output.txt | jq -r '.Credentials.AccessKeyId'`
+export AWS_SECRET_ACCESS_KEY=`cat assume-role-output.txt | jq -r '.Credentials.SecretAccessKey'`
+export AWS_SESSION_TOKEN=`cat assume-role-output.txt | jq -r '.Credentials.SessionToken'`
+export AWS_DEFAULT_REGION=$2
+
+echo "End of Switch Role"