Merge pull request #32 from stmandl/master

Updated flavors and small fixes
exasol · Jan 16, 2019 · 0ea38fe · 0ea38fe
2 parents 73ff7cd + ed31610
commit 0ea38fe
Show file tree

Hide file tree

Showing 13 changed files with 79 additions and 52 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -16,8 +16,8 @@ install:
 
 matrix:
   include:
-  - name: mini-python-python
-    env: MATRIX_FLAVOR=mini GENERIC_TEST_LANG=python TEST_FOLDER=python
+  - name: miniEXASOL-6.0.0-python-python
+    env: MATRIX_FLAVOR=mini-EXASOL-6.0.0 GENERIC_TEST_LANG=python TEST_FOLDER=python
   - name: standard-EXASOL-6.0.0-python
     env: MATRIX_FLAVOR=standard-EXASOL-6.0.0 GENERIC_TEST_LANG=python TEST_FOLDER=python
   - name: standard-EXASOL-6.0.0-r

diff --git a/README.md b/README.md
@@ -29,26 +29,26 @@ For running the tests you also need
 * Docker with privileged mode
 
 ## Quickstart
-1. Choose a flavor. Currently we have several pre-defined flavors available, e.g., `mini`, `standard` and `conda`.
+1. Choose a flavor. Currently we have several pre-defined flavors available, e.g., `mini-EXASOL-6.0.0`, and `standard-EXASOL-6.1.0`.
 This project supports different versions of script language environments with different libraries and languages.
 We call these versions _flavors_. The pre-defined flavors can be modified and extended to create customized flavors.
 Each pre-defined flavor has its own set of Docker build-files in a corresponding subfolder of [flavors](flavors).
 2. Create the language container. We choose to use the `mini` flavor which is the smallest of the currently available flavors and which only support the Python language.
 ```bash
-$ ./build --flavor=mini
+$ ./build --flavor=mini-EXASOL-6.0.0
 ```
-(on Mac OS X, use `./build -f mini`)
+(on Mac OS X, use `./build -f mini-EXASOL-6.0.0`)
 
 3. Export it into a standalone archive
 ```bash
-$ ./export --flavor=mini
+$ ./export --flavor=mini-EXASOL-6.0.0
 ```
-(on Mac OS X, use `./export -f mini`)
-This creates the file `mini.tar.gz`.
+(on Mac OS X, use `./export -f mini-EXASOL-6.0.0`)
+This creates the file `mini-EXASOL-6.0.0.tar.gz`.
 
 Optionally, you can run some automated tests for your flavor by using
 ```bash
-$ ./test_complete --flavor=mini
+$ ./test_complete --flavor=mini-EXASOL-6.0.0
 ```
 (on Mac OS X you need to have pip installed for this to work)
 If the test fails with the message
@@ -59,11 +59,11 @@ then run `stop_dockerdb` and restart the test.
 
 4. Upload the file into bucketfs. For the following example we assume the password `pwd` and the bucketname `funwithudfs` in a bucketfs that is running on port `2580` on machine `192.168.122.158`
 ```bash
-curl -v -X PUT -T mini.tar.gz w:[email protected]:2580/funwithudfs/mini.tar.gz
+curl -v -X PUT -T mini-EXASOL-6.0.0.tar.gz w:[email protected]:2580/funwithudfs/mini-EXASOL-6.0.0.tar.gz
 ```
-5. In SQL you activate the Python implementation of the flavor `mini` by using a statement like this
+5. In SQL you activate the Python implementation of the flavor `mini-EXASOL-6.0.0` by using a statement like this
 ```sql
-ALTER SESSION SET SCRIPT_LANGUAGES='MYPYTHON=localzmq+protobuf:///bucketfsname/funwithudfs/mini?lang=python#buckets/bucketfsname/funwithudfs/mini/exaudf/exaudfclient';
+ALTER SESSION SET SCRIPT_LANGUAGES='MYPYTHON=localzmq+protobuf:///bucketfsname/funwithudfs/mini-EXASOL-6.0.0?lang=python#buckets/bucketfsname/funwithudfs/mini-EXASOL-6.0.0/exaudf/exaudfclient';
 ```
 Now the script language `MYPYTHON` can be used to define a script, e.g., 
 ```
@@ -79,7 +79,7 @@ select small_test();
 ```
 6. Afterwards you may choose to remove the Docker images for this flavor. This can be done as follows:
 ```bash
-./clean --flavor=mini
+./clean --flavor=mini-EXASOL-6.0.0
 ```
 Please note that this script does not delete the Linux image that is used as basis for the images that were build in the previous steps.
 

diff --git a/flavors/fancyr-EXASOL-6.0.0/Dockerfile.base_fancyr-EXASOL-6.0.0 b/flavors/fancyr-EXASOL-6.0.0/Dockerfile.base_fancyr-EXASOL-6.0.0
@@ -1,31 +1,30 @@
 FROM ubuntu:16.04
 ARG numberOfCores
 
+
 COPY ext/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc 
 
 RUN apt-get -y update && \
     apt-get -y install software-properties-common && \
-    add-apt-repository ppa:marutter/rrutter3.5 && \
+    add-apt-repository -y ppa:marutter/rrutter3.5 && \
     apt-get -y update && \
     DEBIAN_FRONTEND=noninteractive apt-get -y --allow-unauthenticated install \
         coreutils \
         locales \
         python-dev  \
         libzmq3-dev \
         protobuf-compiler \
-        r-base \
-        r-base-dev \
+        r-base r-base-core r-base-dev \
+	libxml2-dev libcurl4-openssl-dev libssl-dev \
 	libxml2-dev \
         libprotobuf-dev \
         build-essential \
-        vim \
         wget \
         gfortran \
         libssl-dev \
         unzip \
         curl \
-        libcurl4-openssl-dev  \
-        ed openjdk-9-jdk-headless \
+        openjdk-9-jdk-headless \
         libnss-db  && \
     wget prdownloads.sourceforge.net/swig/swig-2.0.4.tar.gz && \
     tar zxf swig-2.0.4.tar.gz && \
@@ -39,10 +38,10 @@ RUN apt-get -y update && \
     addgroup --gid 1000 exasolution && \
     adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
 
-RUN Rscript -e 'install.packages(c("RcppArmadillo","rredis", "data.table", "htmltools", "flashClust", "forecast", "np","abind","assertthat","backports","base64enc","BH","bindr","bindrcpp","bitops","Brobdingnag","broom","callr","car","carData","caTools","cellranger","classInt","clustMixType","combinat","crosstalk","DBI","dbplyr","doParallel","dplyr","e1071","evaluate","forcats","foreach","gdata","glmnet","gmodels","gplots","gtools","haven","hexbin","highr","hms","htmlwidgets","httpuv","httr","imputeTS","ISLR","iterators","jsonlite","KernSmoothIRT","klaR","knitr","labelled","later","lme4","LowRankQP","lubridate","manipulateWidget","maptools","markdown","MASS","mime","miniUI","minqa","mnormt","modelr","MPTinR","multcomp","mvtnorm","nloptr","numDeriv","openssl","openxlsx","pbkrtest","pkgconfig","plogr","plotly","plotrix","praise","processx","promises","psych","purrr","randomForest","RcppEigen","readr","readxl","rematch","reprex","rgl","rio","rmarkdown","ROSE","rprojroot","rstudioapi","rvest","sandwich","selectr","shiny","sourcetools","sp","spData","stinepack","survival","testthat","TH.data","tidyr","tidyselect","tidyverse","tinytex","tree","webshot","whisker","xml2","xtable","yaml","zip","translations","igraph","caret","kernlab","ROCR","gbm","party","arules","RWeka","ipred","lars","earth","CORElearn","mboost"), repos = "http://cran.r-project.org", clean = TRUE, Ncpus = '"$numberOfCores"')'
-ENV LANG en_US.UTF-8  
-ENV LANGUAGE en_US:en  
-ENV LC_ALL en_US.UTF-8  
 
+RUN Rscript -e 'install.packages(c("BH","Brobdingnag","CORElearn","DBI","ISLR","KernSmoothIRT","LowRankQP","MASS","MPTinR","RCurl","ROCR","ROSE","RWeka","RcppArmadillo","RcppEigen","TH.data","abind","arules","assertthat","backports","base64enc","bindr","bindrcpp","bitops","broom","caTools","callr","car","carData","caret","cellranger","classInt","clustMixType","combinat","crosstalk","data.table","dbplyr","doParallel","dplyr","e1071","earth","evaluate","flashClust","forcats","foreach","forecast","gbm","gdata","glmnet","gmodels","gplots","gtools","haven","hexbin","highr","hms","htmltools","htmlwidgets","httpuv","httr","igraph","imputeTS","ipred","iterators","jsonlite","kernlab","klaR","knitr","labelled","lars","later","lme4","lubridate","manipulateWidget","maptools","markdown","mboost","mime","miniUI","minqa","mnormt","modelr","multcomp","mvtnorm","nloptr","np","numDeriv","openssl","openxlsx","party","pbkrtest","pkgconfig","plogr","plotly","plotrix","praise","processx","promises","psych","purrr","randomForest","ranger","readr","readxl","rematch","reprex","rgl","rio","rmarkdown","rprojroot","rredis","rstudioapi","rvest","sandwich","selectr","shiny","sourcetools","sp","spData","stinepack","survival","testthat","tidyr","tidyselect","tidyverse","tinytex","tree","webshot","whisker","XML","xml2","xtable","yaml","zip"), repos = "http://cran.r-project.org", clean = TRUE, Ncpus = "$numberOfCores")'
 
 
+ENV LANG en_US.UTF-8  
+ENV LANGUAGE en_US:en  
+ENV LC_ALL en_US.UTF-8  
diff --git a/flavors/fancyr-EXASOL-6.1.0/Dockerfile.base_fancyr-EXASOL-6.1.0 b/flavors/fancyr-EXASOL-6.1.0/Dockerfile.base_fancyr-EXASOL-6.1.0
@@ -5,26 +5,25 @@ COPY ext/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
 
 RUN apt-get -y update && \
     apt-get -y install software-properties-common && \
+    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
+    add-apt-repository -y 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' && \
     apt-get -y update && \
     DEBIAN_FRONTEND=noninteractive apt-get -y --allow-unauthenticated install \
         coreutils \
         locales \
         python-dev  \
         libzmq3-dev \
         protobuf-compiler \
-        r-base \
-        r-base-dev \
+        r-base r-base-core r-base-dev r-recommended \
+	libxml2-dev libcurl4-openssl-dev libssl-dev \
         libprotobuf-dev \
         build-essential \
-        vim \
         wget \
         gfortran \
         libssl-dev \
         unzip \
         curl \
-        ed openjdk-9-jdk-headless \
-        libcurl4-openssl-dev  \
-        r-cran-rcpparmadillo \
+        openjdk-11-jdk-headless \
         libnss-db  && \
     wget prdownloads.sourceforge.net/swig/swig-2.0.4.tar.gz && \
     tar zxf swig-2.0.4.tar.gz && \
@@ -38,9 +37,8 @@ RUN apt-get -y update && \
     addgroup --gid 1000 exasolution && \
     adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone"
 
-RUN Rscript -e 'install.packages(c("rredis", "data.table", "htmltools", "flashClust", "forecast"), repos = "http://cran.r-project.org")' 
-RUN Rscript -e 'install.packages(c("abind","assertthat","backports","base64enc","BH","bindr","bindrcpp","bitops","Brobdingnag","broom","callr","car","carData","caTools","cellranger","classInt","clustMixType","combinat","crosstalk","DBI","dbplyr","doParallel","dplyr","e1071","evaluate","forcats","foreach","gdata","glmnet","gmodels","gplots","gtools","haven","hexbin","highr","hms","htmlwidgets","httpuv","httr","imputeTS","ISLR","iterators","jsonlite","KernSmoothIRT","klaR","knitr","labelled","later","lme4","LowRankQP","lubridate","manipulateWidget","maptools","markdown","MASS","mime","miniUI","minqa","mnormt","modelr","MPTinR","multcomp","mvtnorm","nloptr","numDeriv","openssl","openxlsx","pbkrtest","pkgconfig","plogr","plotly","plotrix","praise","processx","promises","psych","purrr","randomForest","RcppEigen","readr","readxl","rematch","reprex","rgl","rio","rmarkdown","ROSE","rprojroot","rstudioapi","rvest","sandwich","selectr","shiny","sourcetools","sp","spData","stinepack","survival","testthat","TH.data","tidyr","tidyselect","tidyverse","tinytex","tree","webshot","whisker","xml2","xtable","yaml","zip","translations","igraph","caret","kernlab","ROCR","gbm","party","arules","RWeka","ipred","lars","earth","CORElearn","mboost"), repos = "http://cran.r-project.org")'
 
+RUN Rscript -e 'install.packages(c("BH","Brobdingnag","CORElearn","DBI","ISLR","KernSmoothIRT","LowRankQP","MASS","MPTinR","RCurl","ROCR","ROSE","RWeka","RcppArmadillo","RcppEigen","TH.data","abind","arules","assertthat","backports","base64enc","bindr","bindrcpp","bitops","broom","caTools","callr","car","carData","caret","cellranger","classInt","clustMixType","combinat","crosstalk","data.table","dbplyr","doParallel","dplyr","e1071","earth","evaluate","flashClust","forcats","foreach","forecast","gbm","gdata","glmnet","gmodels","gplots","gtools","haven","hexbin","highr","hms","htmltools","htmlwidgets","httpuv","httr","igraph","imputeTS","ipred","iterators","jsonlite","kernlab","klaR","knitr","labelled","lars","later","lme4","lubridate","manipulateWidget","maptools","markdown","mboost","mime","miniUI","minqa","mnormt","modelr","multcomp","mvtnorm","nloptr","np","numDeriv","openssl","openxlsx","party","pbkrtest","pkgconfig","plogr","plotly","plotrix","praise","processx","promises","psych","purrr","randomForest","ranger","readr","readxl","rematch","reprex","rgl","rio","rmarkdown","rprojroot","rredis","rstudioapi","rvest","sandwich","selectr","shiny","sourcetools","sp","spData","stinepack","survival","testthat","tidyr","tidyselect","tidyverse","tinytex","tree","webshot","whisker","XML","xml2","xtable","yaml","zip"), repos = "http://cran.r-project.org", clean = TRUE, Ncpus = "$numberOfCores")'
 
 ENV LANG en_US.UTF-8  
 ENV LANGUAGE en_US:en  

diff --git a/flavors/fancyr-EXASOL-6.1.0/testconfig b/flavors/fancyr-EXASOL-6.1.0/testconfig
@@ -0,0 +1,4 @@
+language_definition=R=localzmq+protobuf:///bfsdefault/myudfs/fancyr-EXASOL-6.1.0?lang=r#buckets/bfsdefault/myudfs/fancyr-EXASOL-6.1.0/exaudf/exaudfclient PYTHON=builtin_python JAVA=builtin_java
+generic_language_tests=r
+test_folders=r
+# ending new line required!
diff --git a/flavors/mini/Dockerfile.base_mini → ...L-6.0.0/Dockerfile.base_mini-EXASOL-6.0.0 b/flavors/mini/Dockerfile.base_mini → ...L-6.0.0/Dockerfile.base_mini-EXASOL-6.0.0
diff --git a/flavors/mini/Dockerfile.build_mini → ...-6.0.0/Dockerfile.build_mini-EXASOL-6.0.0 b/flavors/mini/Dockerfile.build_mini → ...-6.0.0/Dockerfile.build_mini-EXASOL-6.0.0
@@ -1,4 +1,4 @@
-FROM exaudf/baseimage_mini
+FROM exaudf/baseimage_mini-exasol-6.0.0
 
 RUN mkdir /exaudf_src /exaudf_src/build /exaudf
 

diff --git a/flavors/mini/Dockerfile.release_mini → ....0.0/Dockerfile.release_mini-EXASOL-6.0.0 b/flavors/mini/Dockerfile.release_mini → ....0.0/Dockerfile.release_mini-EXASOL-6.0.0
@@ -1,4 +1,4 @@
-FROM exaudf/baseimage_mini
+FROM exaudf/baseimage_mini-exasol-6.0.0
 ARG extract_dir
 COPY $extract_dir/exaudf /exaudf
 
diff --git a/flavors/mini/Dockerfile.test_mini → ...L-6.0.0/Dockerfile.test_mini-EXASOL-6.0.0 b/flavors/mini/Dockerfile.test_mini → ...L-6.0.0/Dockerfile.test_mini-EXASOL-6.0.0
@@ -1,4 +1,4 @@
-FROM exaudf/release_mini
+FROM exaudf/release_mini-exasol-6.0.0
 
 RUN mkdir /exasol_emulator
 

diff --git a/flavors/mini/testconfig → flavors/mini-EXASOL-6.0.0/testconfig b/flavors/mini/testconfig → flavors/mini-EXASOL-6.0.0/testconfig
@@ -1,4 +1,4 @@
-language_definition=PYTHON=localzmq+protobuf:///bfsdefault/myudfs/mini?lang=python#buckets/bfsdefault/myudfs/mini/exaudf/exaudfclient JAVA=builtin_java R=builtin_r
+language_definition=PYTHON=localzmq+protobuf:///bfsdefault/myudfs/mini-EXASOL-6.0.0?lang=python#buckets/bfsdefault/myudfs/mini-EXASOL-6.0.0/exaudf/exaudfclient JAVA=builtin_java R=builtin_r
 generic_language_tests=python
 test_folders=python
 # ending new line required!
diff --git a/flavors/python3-ds-EXASOL-6.1.0/Dockerfile.base_python3-ds-EXASOL-6.1.0 b/flavors/python3-ds-EXASOL-6.1.0/Dockerfile.base_python3-ds-EXASOL-6.1.0
@@ -28,17 +28,21 @@ RUN apt-get -y update && \
     (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
     rm -rf swig-2.0.4 swig-2.0.4.tar.gz 
 RUN apt-get -y install cython3 libcurl4-openssl-dev
-RUN wget https://files.pythonhosted.org/packages/85/d1/69ee7e757f657e7f527cbf500ec2d295396e5bcec873cf4eb68962c41024/statsmodels-0.9.0-cp36-cp36m-manylinux1_x86_64.whl && \
-    pip3 install statsmodels-0.9.0-cp36-cp36m-manylinux1_x86_64.whl && \
-    rm -f statsmodels-0.9.0-cp36-cp36m-manylinux1_x86_64.whl && \
-    pip3 install keras tensorflow kmodes seaborn matplotlib imbalanced-learn lifelines nltk gensim lxml ujson && \
-    locale-gen en_US.UTF-8 && \
+
+RUN  python3.6 -m pip install --upgrade pip setuptools wheel
+# RUN  curl -OL https://github.com/google/protobuf/releases/download/v3.5.1/protobuf-cpp-3.5.1.zip && \
+#     unzip protobuf-cpp-3.5.1.zip && \
+#     (cd protobuf-3.5.1 ; ./configure ; make -j 4 ; make install ) && \
+#     rm protobuf-cpp-3.5.1.zip  && rm -r protobuf-3.5.1
+
+RUN pip install keras tensorflow kmodes seaborn matplotlib imbalanced-learn lifelines nltk gensim lxml ujson numpy scipy scikit-learn statsmodels joblib pandas pyexasol
+RUN apt-get --yes  remove python3-pycurl
+RUN python3.6 -m pip install --ignore-installed pycurl
+RUN locale-gen en_US.UTF-8 && \
     update-locale LC_ALL=en_US.UTF-8 && \
     apt-get -y clean && \
     apt-get -y autoremove && \
     ldconfig
-RUN apt-get --yes  remove python3-pycurl
-RUN python3.6 -m pip install --ignore-installed pycurl
 RUN addgroup --gid 1000 exasolution
 RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution
 
@@ -48,3 +52,24 @@ ENV LC_ALL en_US.UTF-8
 
 
 
+
+# RUN wget https://files.pythonhosted.org/packages/85/d1/69ee7e757f657e7f527cbf500ec2d295396e5bcec873cf4eb68962c41024/statsmodels-0.9.0-cp36-cp36m-manylinux1_x86_64.whl && \
+#     pip3 install statsmodels-0.9.0-cp36-cp36m-manylinux1_x86_64.whl && \
+#     rm -f statsmodels-0.9.0-cp36-cp36m-manylinux1_x86_64.whl && \
+#     pip3 install keras tensorflow kmodes seaborn matplotlib imbalanced-learn lifelines nltk gensim lxml ujson && \
+#     locale-gen en_US.UTF-8 && \
+#     update-locale LC_ALL=en_US.UTF-8 && \
+#     apt-get -y clean && \
+#     apt-get -y autoremove && \
+#     ldconfig
+# RUN apt-get --yes  remove python3-pycurl
+# RUN python3.6 -m pip install --ignore-installed pycurl
+# RUN addgroup --gid 1000 exasolution
+# RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution
+
+# ENV LANG en_US.UTF-8  
+# ENV LANGUAGE en_US:en  
+# ENV LC_ALL en_US.UTF-8  
+
+
+
diff --git a/src/pythoncontainer.cc b/src/pythoncontainer.cc
@@ -143,26 +143,27 @@ PythonVMImpl::PythonVMImpl(bool checkOnly): m_checkOnly(checkOnly)
 {
     script_code = string("\xEF\xBB\xBF") + string(SWIGVM_params->script_code);    // Magic-Number of UTF-8 files
 
-//    const string nositeKeyword = "%nosite";
-//    const string whitespace = " \t\f\v";
-//    const string lineEnd = ";";
-//    size_t pos;
-//    string nosite = ExecutionGraph::extractOptionLine(script_code, nositeKeyword, whitespace, lineEnd, pos, [&](const char* msg){throw PythonVM::exception(msg);});
+    //const string nositeKeyword = "%nosite";
+    //const string whitespace = " \t\f\v";
+    //const string lineEnd = ";";
+    //size_t pos;
+    //string nosite = ExecutionGraph::extractOptionLine(script_code, nositeKeyword, whitespace, lineEnd, pos, [&](const char* msg){throw PythonVM::exception(msg);});
 
 //    cerr << "VALUE of nosite: |" << nosite << "|" << endl;
 
 
     int noSiteFlag = 0;
-//    if (nosite == "yes") {noSiteFlag=1;}
-//    else if (nosite == "" || nosite == "no") {noSiteFlag=0;}
-//    else throw PythonVM::exception("Invalid value for %nosite option, must be yes or no");
+    //if (nosite == "yes") {noSiteFlag=1;}
+    //else if (nosite == "" || nosite == "no") {noSiteFlag=0;}
+    //else throw PythonVM::exception("Invalid value for %nosite option, must be yes or no");
 
 //    cerr << "Value of noSiteFlag: |" << noSiteFlag << "|" << endl;
 
-//    script_code = string("\xEF\xBB\xBF") + script_code;
+    //script_code = string("\xEF\xBB\xBF") + script_code;
 
 //    cerr << "Script code after extract option line: " << endl << script_code << endl;
 
+    //SWIGVM_params->script_code = strdup(script_code.c_str());
 
     script = exatable = globals = retvalue = NULL;
 #ifndef DISABLE_PYTHON_SUBINTERP

diff --git a/tests/test/generic/unicode.py b/tests/test/generic/unicode.py
@@ -62,7 +62,7 @@ def run(ctx):
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT)
         out, _err = exaplus.communicate(sql)
-
+        pythonVersionInUdf = -1
         for line in out.strip().split('\n'):
             m = re.search(r'Python=(\d)',line)
             if m: