Build and CI changes

.circleci/config.yml

@@ -1,83 +1,64 @@
 version: 2.1
 
 commands: 
+  setup-executor:
+    steps:
+      - run:
+          name: Setup executor
+          command: |
+            apt-get -qq update
+            apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
+            bash <(curl -fsSL https://get.docker.com)
+      - setup_remote_docker:
+          docker_layer_caching: true
+
   setup-automation:
     steps:
       - run:
           name: Setup automation
           command: |
-            git submodule update --init --recursive
             ./deps/readies/bin/getpy3
+            python3 ./system-setup.py
+
+  build-and-test-steps:
+    steps:
+      - checkout
+      - run:
+          name: Checkout submodules
+          command: git submodule update --init --recursive
+      - setup-automation
+      - run:
+          name: Install Redis
+          command: ./deps/readies/bin/getredis --version 6
+      - run:
+          name: Tests
+          command: make tests
+      - run:
+          name: Pack
+          command: make pack
+      - persist_to_workspace:
+          root: artifacts
+          paths:
+            - 'snapshot/*.tgz'
 
 jobs:
   build-and-test-bionic:
       docker:
-        - image: 'ubuntu:bionic'
+        - image: redisfab/rmbuilder:6.0.9-x64-bionic
       steps:
-        - run:
-            name: Setup executor
-            command: |
-              apt-get -qq update
-              apt-get -q install -y git openssh-client curl ca-certificates make tar gzip unzip maven
-        - checkout
-        - setup-automation
-        - run:
-            name: Install Redis
-            command: python3 ./deps/readies/bin/getredis --version 6
-        - run:
-            name: Install RLTest
-            command: python3 -m pip install --no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master
-        - run:
-            name: Tests
-            command: make tests
-        - run:
-            name: Pack
-            command: make pack
-        - persist_to_workspace:
-            root: artifacts
-            paths:
-              - 'snapshot/*.tgz'
+        - build-and-test-steps
+
   build-and-test-centos:
       docker:
-        - image: centos:7.8.2003
+        - image: redisfab/rmbuilder:6.0.9-x64-centos7
       steps:
-        - run:
-            name: Setup executor
-            command: |
-              yum install -y git openssh-client curl ca-certificates make tar gzip unzip maven
-        - checkout
-        - setup-automation
-        - run:
-            name: Install gcc
-            command: ./deps/readies/bin/getgcc --modern
-        - run:
-            name: Install Redis
-            command: . /etc/profile.d/scl-devtoolset-9.sh; python3 ./deps/readies/bin/getredis --version 6
-        - run:
-            name: Install RLTest
-            command: python3 -m pip install --no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master
-        - run:
-            name: Tests
-            command: make tests
-        - run:
-            name: Pack
-            command: make pack
-        - persist_to_workspace:
-            root: artifacts
-            paths:
-              - 'snapshot/*.tgz'
+        - build-and-test-steps
+
   build-docker:
       docker:
-        - image: 'ubuntu:bionic'
+        - image: ubuntu:bionic
       steps:
-        - run:
-            name: Setup executor
-            command: |
-              apt-get -qq update
-              apt-get -q install -y git openssh-client curl ca-certificates
-              bash <(curl -fsSL https://get.docker.com)
-        - setup_remote_docker:
-            docker_layer_caching: true 
+        - setup-executor
         - checkout
         - run:
             name: Checkout submodules
@@ -89,6 +70,7 @@ jobs:
               docker login -u redisfab -p $DOCKER_REDISFAB_PWD
               docker build -t redisfab/redisgearsjvm:edge .
               docker push redisfab/redisgearsjvm:edge
+
   deploy_snapshots:
       docker:
         - image: redisfab/rmbuilder:6.0.9-x64-buster
@@ -101,6 +83,8 @@ jobs:
               for f in artifacts/snapshot/*.tgz; do
                 aws s3 cp $f s3://redismodules/redisgears_plugins/jvm_plugin/snapshots/ --acl public-read
               done
+
+
 on-any-branch: &on-any-branch
   filters:
     branches:

Dockerfile

@@ -1,13 +1,13 @@
 FROM ubuntu:bionic
-RUN apt update
-RUN apt install -y build-essential wget unzip maven
+RUN apt-get update -qq
+RUN apt-get install -q -y build-essential wget unzip maven
 
 WORKDIR /build
 
 ADD . /build
 
 RUN ./deps/readies/bin/getpy3
-
+RUN python ./system-setup.py
 RUN ./deps/readies/bin/getredis --version 6
 
 RUN make all

Install_OpenJDK.sh

@@ -1,3 +1,9 @@
+#!/bin/bash
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT=$HERE
+export READIES=$ROOT/deps/readies
+
 echo "installing OpenJDK"
 
 WORK_DIR=./bin/OpenJDK/
@@ -6,10 +12,10 @@ OPEN_JDK_URL=https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/downloa
 
 mkdir -p $WORK_DIR
 
-if [ -f "$WORK_DIR$OPEN_JDK_ZIP" ]; then
+if [[ -f "$WORK_DIR$OPEN_JDK_ZIP" ]]; then
     echo "Skiping OpenJDK download"
 else 
     echo "Download OpenJDK"
-    wget -P $WORK_DIR $OPEN_JDK_URL
-    tar -C $WORK_DIR -xvf $WORK_DIR$OPEN_JDK_ZIP
+    wget -q -P $WORK_DIR $OPEN_JDK_URL
+    tar -C $WORK_DIR --no-same-owner -xf $WORK_DIR$OPEN_JDK_ZIP
 fi

Install_RedisGears.sh

@@ -1,31 +1,33 @@
-if [ -z ${OS+x} ]
-then
-      echo "os is not set"
-      exit 1
-fi
+#!/bin/bash
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT=$HERE
+export READIES=$ROOT/deps/readies
 
-echo "installing redisgears for :" $OS
+OS=$($READIES/bin/platform --osnick)
 
-WORK_DIR=./bin/RedisGears/
+echo "installing redisgears for: $OS"
+
+WORK_DIR=./bin/RedisGears
 REDISGEARS_ZIP=redisgears.linux-$OS-x64.master.zip
 REDISGEARS_DEPS=redisgears-python.linux-$OS-x64.master.tgz
 REDISGEARS_S3_PATH=http://redismodules.s3.amazonaws.com/redisgears/snapshots/$REDISGEARS_ZIP
 REDISGEARS_DEPS_S3_PATH=http://redismodules.s3.amazonaws.com/redisgears/snapshots/$REDISGEARS_DEPS
 
 mkdir -p $WORK_DIR
 
-if [ -f "$WORK_DIR$REDISGEARS_ZIP" ]; then
+if [[ -f "$WORK_DIR/$REDISGEARS_ZIP" ]]; then
     echo "Skiping RedisGears download"
 else 
     echo "Download RedisGears"
-    wget -P $WORK_DIR $REDISGEARS_S3_PATH
-    unzip $WORK_DIR$REDISGEARS_ZIP -d $WORK_DIR
+    wget -q -P $WORK_DIR $REDISGEARS_S3_PATH
+    unzip -q $WORK_DIR/$REDISGEARS_ZIP -d $WORK_DIR
 fi
 
-if [ -f "$WORK_DIR$REDISGEARS_DEPS" ]; then
+if [[ -f "$WORK_DIR/$REDISGEARS_DEPS" ]]; then
     echo "Skiping RedisGears download"
 else 
     echo "Download RedisGears deps"
-    wget -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH
-    tar -C $WORK_DIR -xvf $WORK_DIR$REDISGEARS_DEPS
-fi
+    wget -q -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH
+    tar -C $WORK_DIR --no-same-owner -xf $WORK_DIR/$REDISGEARS_DEPS
+fi

Makefile

@@ -1,29 +1,31 @@
-OS=$(shell ./deps/readies/bin/platform --osnick)
-GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
-$(info OS=$(OS))
+
+JVM_PATH=./bin/OpenJDK/jdk-11.0.9.1+1/
+GEARS_PATH=./bin/RedisGears/redisgears.so
+JVM_PLUGIN_PATH=./src/gears_jvm.so
+JVM_PLUGIN_CLASSPATH=./gears_runtime/target/gear_runtime-jar-with-dependencies.jar
 
 all: gears_jvm
 
 gears_jvm: InstallRedisGears InstallOpenJDK GearsRuntime
-	make -C ./src/
+	make -C src
 
 InstallRedisGears:
-	OS=$(OS) /bin/bash ./Install_RedisGears.sh
+	./Install_RedisGears.sh
 
 InstallOpenJDK:
-	/bin/bash ./Install_OpenJDK.sh
+	./Install_OpenJDK.sh
 
 GearsRuntime:  
-	cd gears_runtime; mvn package
+	cd gears_runtime; mvn package -q
 
 clean:
-	make -C ./src/ clean
+	make -C src clean
 
 tests: gears_jvm
-	cd ./pytest; ./run_test.sh
-	
+	cd pytest; ./run_test.sh
+
 run: gears_jvm
-	redis-server --loadmodule ./bin/RedisGears/redisgears.so Plugin ./src/gears_jvm.so JvmOptions "-Djava.class.path=./gears_runtime/target/gear_runtime-jar-with-dependencies.jar" JvmPath ./bin/OpenJDK/jdk-11.0.9.1+1/
+	redis-server --loadmodule $(GEARS_PATH) Plugin $(JVM_PLUGIN_PATH) JvmOptions "-Djava.class.path=$(JVM_PLUGIN_CLASSPATH)" JvmPath $(JVM_PATH)
 
 pack: gears_jvm
-	OS=$(OS) GIT_BRANCH=$(GIT_BRANCH) ./pack.sh
+	./pack.sh

pack.sh

@@ -1,8 +1,17 @@
+#!/bin/bash
+
 set -x
 
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT=$HERE
+export READIES=$ROOT/deps/readies
+
+OS=$($READIES/bin/platform --osnick)
+GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
 mkdir -p ./artifacts/snapshot
 tar -czvf ./artifacts/snapshot/gears-jvm.linux-$OS-x64.$GIT_BRANCH.tgz \
---transform "s,^./src/,./plugin/," \
-./bin/OpenJDK/jdk-11.0.9.1+1/* \
-./src/gears_jvm.so \
-./gears_runtime/target/gear_runtime-jar-with-dependencies.jar
+	--transform "s,^./src/,./plugin/," \
+	./bin/OpenJDK/jdk-11.0.9.1+1/* \
+	./src/gears_jvm.so \
+	./gears_runtime/target/gear_runtime-jar-with-dependencies.jar

pytest/run_test.sh

@@ -1,32 +1,62 @@
 #!/bin/bash
+
 set -x
 set -e
 
-mkdir -p ./gears_tests/build/;cd ./gears_tests/build/;../../../bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath ./../../../gears_runtime/target/gear_runtime-jar-with-dependencies.jar ../src/gears_tests/*;../../../bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cvf gears_tests.jar ./gears_tests/
-cd ../../
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT=$(cd $HERE/..; pwd)
+export READIES=$ROOT/deps/readies
 
-JVM_OPTIONS="-Djava.class.path="
-JVM_OPTIONS+="../../gears_runtime/target/gear_runtime-jar-with-dependencies.jar"
-#JVM_OPTIONS+=" -XX:+IdleTuningGcOnIdle";
-#JVM_OPTIONS+=" -Xms10m";
-#JVM_OPTIONS+=" -Xmx2048m";
-#JVM_OPTIONS+=" -Xrs";
-#JVM_OPTIONS+=" -Xcheck:jni";
+mkdir -p $ROOT/pytest/gears_tests/build
+cd $ROOT/pytest/gears_tests/build
+$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath $ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar $ROOT/pytest/gears_tests/src/gears_tests/*.java
+$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cf gears_tests.jar ./gears_tests/
+cd $ROOT
 
-#echo $JVM_OPTIONS
-#JVM_PATH=../../../../deps/openj9-openjdk-jdk14/build/linux-x86_64-server-release/jdk/lib/server/
-JVM_PATH=../../bin/OpenJDK/jdk-11.0.9.1+1/
+JVM_OPTIONS="-Djava.class.path="
+JVM_OPTIONS+="$ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar"
+# JVM_OPTIONS+=" -XX:+IdleTuningGcOnIdle"
+# JVM_OPTIONS+=" -Xms10m"
+# JVM_OPTIONS+=" -Xmx2048m"
+# JVM_OPTIONS+=" -Xrs"
+# JVM_OPTIONS+=" -Xcheck:jni"
+
+# echo $JVM_OPTIONS
+# JVM_PATH=../../../../deps/openj9-openjdk-jdk14/build/linux-x86_64-server-release/jdk/lib/server/
+JVM_PATH=$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/
+
+SRC=$ROOT/src
+BIN=$ROOT/bin
+
+argsf=$(mktemp /tmp/args.XXXXXX)
+cat <<EOF > $argsf
+	--module $BIN/RedisGears/redisgears.so
+	--module-args "
+		Plugin $SRC/gears_jvm.so
+		JvmPath $JVM_PATH
+		JvmOptions $JVM_OPTIONS
+		Plugin $BIN/RedisGears/plugin/gears_python.so
+		CreateVenv 0
+		PythonInstallationDir $BIN/RedisGears/
+		"
+	--clear-logs
+EOF
+
+RLTEST_ARGS=$(readarray -t A < $argsf; IFS=' '; echo "${A[*]}")
+rm -f $argsf
+
+cd $ROOT/pytest
 
 echo oss
-python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs "$@"
+eval python3 -m RLTest $RLTEST_ARGS "$@"
 
 echo cluster 1 shard
-python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs --env oss-cluster --shards-count 1 "$@"
+eval python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 1 "$@"
 
 echo cluster 2 shards
-python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs --env oss-cluster --shards-count 2 "$@"
+eval python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 2 "$@"
 
 echo cluster 3 shards
-python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs --env oss-cluster --shards-count 3 "$@"
+eval python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 3 "$@"
 
-rm -rf ../bin/RedisGears/.venv-*
+rm -rf $ROOT/bin/RedisGears/.venv-*