From 2387927a5fad2d5333fa515d939a799271081baf Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 24 Jan 2021 10:09:04 +0200 Subject: [PATCH 1/8] Build and CI changes --- .circleci/config.yml | 102 ++++++++++++++++++------------------------ Install_OpenJDK.sh | 8 +++- Install_RedisGears.sh | 26 ++++++----- Makefile | 16 +++---- pack.sh | 10 +++-- system-setup.py | 54 ++++++++++++++++++++++ 6 files changed, 132 insertions(+), 84 deletions(-) mode change 100644 => 100755 Install_OpenJDK.sh mode change 100644 => 100755 Install_RedisGears.sh create mode 100755 system-setup.py diff --git a/.circleci/config.yml b/.circleci/config.yml index ace0499..4f16320 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,17 @@ version: 2.1 commands: + setup-executor: + steps: + - run: + name: Setup executor + command: | + apt-get -qq update + apt-get -q install -y git openssh-client curl ca-certificates make tar gzip + bash <(curl -fsSL https://get.docker.com) + - setup_remote_docker: + docker_layer_caching: true + setup-automation: steps: - run: @@ -9,75 +20,45 @@ commands: git submodule update --init --recursive ./deps/readies/bin/getpy3 + build-and-test-steps: + steps: + - checkout + - run: + name: Checkout submodules + command: git submodule update --init --recursive + - setup-automation + - run: + name: Install Redis + command: ./deps/readies/bin/getredis --version 6 + - run: + name: Tests + command: make tests + - run: + name: Pack + command: make pack + - persist_to_workspace: + root: artifacts + paths: + - 'snapshot/*.tgz' + jobs: build-and-test-bionic: docker: - - image: 'ubuntu:bionic' + - image: redisfab/rmbuilder:6.0.9-x64-bionic steps: - - run: - name: Setup executor - command: | - apt-get -qq update - apt-get -q install -y git openssh-client curl ca-certificates make tar gzip unzip maven - - checkout - - setup-automation - - run: - name: Install Redis - command: python3 ./deps/readies/bin/getredis --version 6 - - run: - name: Install RLTest - command: python3 -m pip install --no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master - - run: - name: Tests - command: make tests - - run: - name: Pack - command: make pack - - persist_to_workspace: - root: artifacts - paths: - - 'snapshot/*.tgz' + - build-and-test-steps + build-and-test-centos: docker: - - image: centos:7.8.2003 + - image: redisfab/rmbuilder:6.0.9-x64-centos7 steps: - - run: - name: Setup executor - command: | - yum install -y git openssh-client curl ca-certificates make tar gzip unzip maven - - checkout - - setup-automation - - run: - name: Install gcc - command: ./deps/readies/bin/getgcc --modern - - run: - name: Install Redis - command: . /etc/profile.d/scl-devtoolset-9.sh; python3 ./deps/readies/bin/getredis --version 6 - - run: - name: Install RLTest - command: python3 -m pip install --no-cache-dir git+https://github.com/RedisLabsModules/RLTest.git@master - - run: - name: Tests - command: make tests - - run: - name: Pack - command: make pack - - persist_to_workspace: - root: artifacts - paths: - - 'snapshot/*.tgz' + - build-and-test-steps + build-docker: docker: - - image: 'ubuntu:bionic' + - image: ubuntu:bionic steps: - - run: - name: Setup executor - command: | - apt-get -qq update - apt-get -q install -y git openssh-client curl ca-certificates - bash <(curl -fsSL https://get.docker.com) - - setup_remote_docker: - docker_layer_caching: true + - setup-executor - checkout - run: name: Checkout submodules @@ -89,6 +70,7 @@ jobs: docker login -u redisfab -p $DOCKER_REDISFAB_PWD docker build -t redisfab/redisgearsjvm:edge . docker push redisfab/redisgearsjvm:edge + deploy_snapshots: docker: - image: redisfab/rmbuilder:6.0.9-x64-buster @@ -101,6 +83,8 @@ jobs: for f in artifacts/snapshot/*.tgz; do aws s3 cp $f s3://redismodules/redisgears_plugins/jvm_plugin/snapshots/ --acl public-read done + + on-any-branch: &on-any-branch filters: branches: diff --git a/Install_OpenJDK.sh b/Install_OpenJDK.sh old mode 100644 new mode 100755 index a080983..c54a0d3 --- a/Install_OpenJDK.sh +++ b/Install_OpenJDK.sh @@ -1,3 +1,9 @@ +#!/bin/bash + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT=$HERE +export READIES=$ROOT/deps/readies + echo "installing OpenJDK" WORK_DIR=./bin/OpenJDK/ @@ -6,7 +12,7 @@ OPEN_JDK_URL=https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/downloa mkdir -p $WORK_DIR -if [ -f "$WORK_DIR$OPEN_JDK_ZIP" ]; then +if [[ -f "$WORK_DIR$OPEN_JDK_ZIP" ]]; then echo "Skiping OpenJDK download" else echo "Download OpenJDK" diff --git a/Install_RedisGears.sh b/Install_RedisGears.sh old mode 100644 new mode 100755 index f8c37a0..d7e49ca --- a/Install_RedisGears.sh +++ b/Install_RedisGears.sh @@ -1,10 +1,12 @@ -if [ -z ${OS+x} ] -then - echo "os is not set" - exit 1 -fi +#!/bin/bash + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT=$HERE +export READIES=$ROOT/deps/readies -echo "installing redisgears for :" $OS +OS=$($READIES/bin/platform --os) + +echo "installing redisgears for: $OS" WORK_DIR=./bin/RedisGears/ REDISGEARS_ZIP=redisgears.linux-$OS-x64.master.zip @@ -14,18 +16,18 @@ REDISGEARS_DEPS_S3_PATH=http://redismodules.s3.amazonaws.com/redisgears/snapshot mkdir -p $WORK_DIR -if [ -f "$WORK_DIR$REDISGEARS_ZIP" ]; then +if [[ -f "$WORK_DIR$REDISGEARS_ZIP" ]]; then echo "Skiping RedisGears download" else echo "Download RedisGears" - wget -P $WORK_DIR $REDISGEARS_S3_PATH - unzip $WORK_DIR$REDISGEARS_ZIP -d $WORK_DIR + wget -q -P $WORK_DIR $REDISGEARS_S3_PATH + unzip -q $WORK_DIR$REDISGEARS_ZIP -d $WORK_DIR fi -if [ -f "$WORK_DIR$REDISGEARS_DEPS" ]; then +if [[ -f "$WORK_DIR$REDISGEARS_DEPS" ]]; then echo "Skiping RedisGears download" else echo "Download RedisGears deps" - wget -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH + wget -q -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH tar -C $WORK_DIR -xvf $WORK_DIR$REDISGEARS_DEPS -fi \ No newline at end of file +fi diff --git a/Makefile b/Makefile index 3b5e74f..d40d3e2 100644 --- a/Makefile +++ b/Makefile @@ -1,27 +1,27 @@ -OS=$(shell ./deps/readies/bin/platform --osnick) -GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) +OS:=$(shell ./deps/readies/bin/platform --osnick) +GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) $(info OS=$(OS)) all: gears_jvm gears_jvm: InstallRedisGears InstallOpenJDK GearsRuntime - make -C ./src/ + make -C src InstallRedisGears: - OS=$(OS) /bin/bash ./Install_RedisGears.sh + OS=$(OS) ./Install_RedisGears.sh InstallOpenJDK: - /bin/bash ./Install_OpenJDK.sh + ./Install_OpenJDK.sh GearsRuntime: cd gears_runtime; mvn package clean: - make -C ./src/ clean + make -C src clean tests: gears_jvm - cd ./pytest; ./run_test.sh - + cd pytest; ./run_test.sh + run: gears_jvm redis-server --loadmodule ./bin/RedisGears/redisgears.so Plugin ./src/gears_jvm.so JvmOptions "-Djava.class.path=./gears_runtime/target/gear_runtime-jar-with-dependencies.jar" JvmPath ./bin/OpenJDK/jdk-11.0.9.1+1/ diff --git a/pack.sh b/pack.sh index 3c9b15b..1763e40 100755 --- a/pack.sh +++ b/pack.sh @@ -1,8 +1,10 @@ +#!/bin/bash + set -x mkdir -p ./artifacts/snapshot tar -czvf ./artifacts/snapshot/gears-jvm.linux-$OS-x64.$GIT_BRANCH.tgz \ ---transform "s,^./src/,./plugin/," \ -./bin/OpenJDK/jdk-11.0.9.1+1/* \ -./src/gears_jvm.so \ -./gears_runtime/target/gear_runtime-jar-with-dependencies.jar \ No newline at end of file + --transform "s,^./src/,./plugin/," \ + ./bin/OpenJDK/jdk-11.0.9.1+1/* \ + ./src/gears_jvm.so \ + ./gears_runtime/target/gear_runtime-jar-with-dependencies.jar diff --git a/system-setup.py b/system-setup.py new file mode 100755 index 0000000..8c16d6a --- /dev/null +++ b/system-setup.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python2 + +import sys +import os +import argparse + +ROOT = HERE = os.path.abspath(os.path.dirname(__file__)) +READIES = os.path.join(ROOT, "deps/readies") +sys.path.insert(0, READIES) +import paella + +#---------------------------------------------------------------------------------------------- + +class RedisGearsJVMPluginSetup(paella.Setup): + def __init__(self, nop=False): + paella.Setup.__init__(self, nop) + + def common_first(self): + self.install_downloaders() + self.pip_install("wheel") + self.pip_install("setuptools --upgrade") + + self.install("git tar gzip unzip") + self.install("maven") + + def debian_compat(self): + self.install("libatomic1") + self.run("%s/bin/getgcc" % READIES) + + def redhat_compat(self): + self.install("redhat-lsb-core") + self.install("libatomic") + + self.run("%s/bin/getgcc --modern" % READIES) + + def fedora(self): + self.install("libatomic") + self.run("%s/bin/getgcc" % READIES) + + def macos(self): + self.install_gnu_utils() + self.run("{PYTHON} {READIES}/bin/getredis -v 6 --force".format(PYTHON=self.python, READIES=READIES)) + + def common_last(self): + # self.run("%s/bin/getcmake" % READIES) + self.run("{PYTHON} {READIES}/bin/getrmpytools".format(PYTHON=self.python, READIES=READIES)) + +#---------------------------------------------------------------------------------------------- + +parser = argparse.ArgumentParser(description='Set up system for build.') +parser.add_argument('-n', '--nop', action="store_true", help='no operation') +args = parser.parse_args() + +RedisGearsJVMPluginSetup(nop = args.nop).setup() From a2e2aab950ed4aaf9bf5cb485881e2f0aff89a6d Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 24 Jan 2021 10:29:46 +0200 Subject: [PATCH 2/8] fixes 1 --- Install_RedisGears.sh | 2 +- Makefile | 14 ++++++++------ pack.sh | 7 +++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Install_RedisGears.sh b/Install_RedisGears.sh index d7e49ca..56c501d 100755 --- a/Install_RedisGears.sh +++ b/Install_RedisGears.sh @@ -4,7 +4,7 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" ROOT=$HERE export READIES=$ROOT/deps/readies -OS=$($READIES/bin/platform --os) +OS=$($READIES/bin/platform --osnick) echo "installing redisgears for: $OS" diff --git a/Makefile b/Makefile index d40d3e2..6783117 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ -OS:=$(shell ./deps/readies/bin/platform --osnick) -GIT_BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) -$(info OS=$(OS)) + +JVM_PATH=./bin/OpenJDK/jdk-11.0.9.1+1/ +GEARS_PATH=./bin/RedisGears/redisgears.so +JVM_PLUGIN_PATH=./src/gears_jvm.so +JVM_PLUGIN_CLASSPATH=./gears_runtime/target/gear_runtime-jar-with-dependencies.jar all: gears_jvm @@ -8,7 +10,7 @@ gears_jvm: InstallRedisGears InstallOpenJDK GearsRuntime make -C src InstallRedisGears: - OS=$(OS) ./Install_RedisGears.sh + ./Install_RedisGears.sh InstallOpenJDK: ./Install_OpenJDK.sh @@ -23,7 +25,7 @@ tests: gears_jvm cd pytest; ./run_test.sh run: gears_jvm - redis-server --loadmodule ./bin/RedisGears/redisgears.so Plugin ./src/gears_jvm.so JvmOptions "-Djava.class.path=./gears_runtime/target/gear_runtime-jar-with-dependencies.jar" JvmPath ./bin/OpenJDK/jdk-11.0.9.1+1/ + redis-server --loadmodule $(GEARS_PATH) Plugin $(JVM_PLUGIN_PATH) JvmOptions "-Djava.class.path=$(JVM_PLUGIN_CLASSPATH)" JvmPath $(JVM_PATH) pack: gears_jvm - OS=$(OS) GIT_BRANCH=$(GIT_BRANCH) ./pack.sh + ./pack.sh diff --git a/pack.sh b/pack.sh index 1763e40..c509b50 100755 --- a/pack.sh +++ b/pack.sh @@ -2,6 +2,13 @@ set -x +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT=$HERE +export READIES=$ROOT/deps/readies + +OS=$($READIES/bin/platform --osnick) +GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + mkdir -p ./artifacts/snapshot tar -czvf ./artifacts/snapshot/gears-jvm.linux-$OS-x64.$GIT_BRANCH.tgz \ --transform "s,^./src/,./plugin/," \ From 927ec2a624e3d78ad96350dc63afbde1f03e6f19 Mon Sep 17 00:00:00 2001 From: rafie Date: Sun, 24 Jan 2021 10:58:19 +0200 Subject: [PATCH 3/8] fixes 2 --- Install_OpenJDK.sh | 4 ++-- Install_RedisGears.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Install_OpenJDK.sh b/Install_OpenJDK.sh index c54a0d3..cbdfcbb 100755 --- a/Install_OpenJDK.sh +++ b/Install_OpenJDK.sh @@ -16,6 +16,6 @@ if [[ -f "$WORK_DIR$OPEN_JDK_ZIP" ]]; then echo "Skiping OpenJDK download" else echo "Download OpenJDK" - wget -P $WORK_DIR $OPEN_JDK_URL - tar -C $WORK_DIR -xvf $WORK_DIR$OPEN_JDK_ZIP + wget -q -P $WORK_DIR $OPEN_JDK_URL + tar -C $WORK_DIR -xf $WORK_DIR$OPEN_JDK_ZIP fi diff --git a/Install_RedisGears.sh b/Install_RedisGears.sh index 56c501d..9eed769 100755 --- a/Install_RedisGears.sh +++ b/Install_RedisGears.sh @@ -29,5 +29,5 @@ if [[ -f "$WORK_DIR$REDISGEARS_DEPS" ]]; then else echo "Download RedisGears deps" wget -q -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH - tar -C $WORK_DIR -xvf $WORK_DIR$REDISGEARS_DEPS + tar -C $WORK_DIR -xf $WORK_DIR$REDISGEARS_DEPS fi From 537511a10cb3d97a856a649d0707c4145b40d58c Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 26 Jan 2021 15:10:27 +0200 Subject: [PATCH 4/8] fixes 3 --- .circleci/config.yml | 2 +- Dockerfile | 4 ++-- Install_OpenJDK.sh | 2 +- Install_RedisGears.sh | 2 +- pytest/run_test.sh | 55 ++++++++++++++++++++++++++++++------------- 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f16320..3b7810b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,8 +17,8 @@ commands: - run: name: Setup automation command: | - git submodule update --init --recursive ./deps/readies/bin/getpy3 + python3 ./system-setup.py build-and-test-steps: steps: diff --git a/Dockerfile b/Dockerfile index 5347ac3..631548f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:bionic -RUN apt update -RUN apt install -y build-essential wget unzip maven +RUN apt-get update -qq +RUN apt-get install -q -y build-essential wget unzip maven WORKDIR /build diff --git a/Install_OpenJDK.sh b/Install_OpenJDK.sh index cbdfcbb..52c94a0 100755 --- a/Install_OpenJDK.sh +++ b/Install_OpenJDK.sh @@ -17,5 +17,5 @@ if [[ -f "$WORK_DIR$OPEN_JDK_ZIP" ]]; then else echo "Download OpenJDK" wget -q -P $WORK_DIR $OPEN_JDK_URL - tar -C $WORK_DIR -xf $WORK_DIR$OPEN_JDK_ZIP + tar -C $WORK_DIR --no-same-owner -xf $WORK_DIR$OPEN_JDK_ZIP fi diff --git a/Install_RedisGears.sh b/Install_RedisGears.sh index 9eed769..0878468 100755 --- a/Install_RedisGears.sh +++ b/Install_RedisGears.sh @@ -29,5 +29,5 @@ if [[ -f "$WORK_DIR$REDISGEARS_DEPS" ]]; then else echo "Download RedisGears deps" wget -q -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH - tar -C $WORK_DIR -xf $WORK_DIR$REDISGEARS_DEPS + tar -C $WORK_DIR --no-same-owner -xf $WORK_DIR$REDISGEARS_DEPS fi diff --git a/pytest/run_test.sh b/pytest/run_test.sh index 7b96f30..8f123c7 100755 --- a/pytest/run_test.sh +++ b/pytest/run_test.sh @@ -1,32 +1,53 @@ #!/bin/bash + set -x set -e -mkdir -p ./gears_tests/build/;cd ./gears_tests/build/;../../../bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath ./../../../gears_runtime/target/gear_runtime-jar-with-dependencies.jar ../src/gears_tests/*;../../../bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cvf gears_tests.jar ./gears_tests/ -cd ../../ +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT=$(cd $HERE/..; pwd) +export READIES=$ROOT/deps/readies -JVM_OPTIONS="-Djava.class.path=" -JVM_OPTIONS+="../../gears_runtime/target/gear_runtime-jar-with-dependencies.jar" -#JVM_OPTIONS+=" -XX:+IdleTuningGcOnIdle"; -#JVM_OPTIONS+=" -Xms10m"; -#JVM_OPTIONS+=" -Xmx2048m"; -#JVM_OPTIONS+=" -Xrs"; -#JVM_OPTIONS+=" -Xcheck:jni"; +mkdir -p $ROOT/gears_tests/build +cd $ROOT/gears_tests/build +$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath $ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar $ROOT/src/gears_tests/* +$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cvf gears_tests.jar ./gears_tests/ +cd $ROOT -#echo $JVM_OPTIONS -#JVM_PATH=../../../../deps/openj9-openjdk-jdk14/build/linux-x86_64-server-release/jdk/lib/server/ -JVM_PATH=../../bin/OpenJDK/jdk-11.0.9.1+1/ +JVM_OPTIONS="-Djava.class.path=" +JVM_OPTIONS+="$ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar" +# JVM_OPTIONS+=" -XX:+IdleTuningGcOnIdle" +# JVM_OPTIONS+=" -Xms10m" +# JVM_OPTIONS+=" -Xmx2048m" +# JVM_OPTIONS+=" -Xrs" +# JVM_OPTIONS+=" -Xcheck:jni" + +# echo $JVM_OPTIONS +# JVM_PATH=../../../../deps/openj9-openjdk-jdk14/build/linux-x86_64-server-release/jdk/lib/server/ +JVM_PATH=$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/ + +SRC=$ROOT/src +BIN=$ROOT/bin + +RLTEST_ARGS=\ + --module $BIN/RedisGears/redisgears.so \ + --module-args \ + "Plugin $SRC/gears_jvm.so \ + JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS \ + Plugin $BIN/RedisGears/plugin/gears_python.so \ + CreateVenv 0 \ + PythonInstallationDir $BIN/RedisGears/" \ + --clear-logs echo oss -python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs "$@" +python3 -m RLTest $RLTEST_ARGS "$@" echo cluster 1 shard -python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs --env oss-cluster --shards-count 1 "$@" +python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 1 "$@" echo cluster 2 shards -python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs --env oss-cluster --shards-count 2 "$@" +python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 2 "$@" echo cluster 3 shards -python3 -m RLTest --module ../bin/RedisGears/redisgears.so --module-args "Plugin ../../src/gears_jvm.so JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS Plugin ../../bin/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir ../../bin/RedisGears/" --clear-logs --env oss-cluster --shards-count 3 "$@" +python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 3 "$@" -rm -rf ../bin/RedisGears/.venv-* +rm -rf $ROOT/bin/RedisGears/.venv-* From 0a6f238c6df06e7a71e4c57323e8fb5b60796ccd Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 26 Jan 2021 17:24:15 +0200 Subject: [PATCH 5/8] fixes 4 --- Dockerfile | 2 +- Install_RedisGears.sh | 4 ++-- Makefile | 2 +- pytest/run_test.sh | 28 +++++++++++++++++----------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 631548f..084a72c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /build ADD . /build RUN ./deps/readies/bin/getpy3 - +RUN python ./system-setup.py RUN ./deps/readies/bin/getredis --version 6 RUN make all diff --git a/Install_RedisGears.sh b/Install_RedisGears.sh index 0878468..ba4b49c 100755 --- a/Install_RedisGears.sh +++ b/Install_RedisGears.sh @@ -8,7 +8,7 @@ OS=$($READIES/bin/platform --osnick) echo "installing redisgears for: $OS" -WORK_DIR=./bin/RedisGears/ +WORK_DIR=./bin/RedisGears REDISGEARS_ZIP=redisgears.linux-$OS-x64.master.zip REDISGEARS_DEPS=redisgears-python.linux-$OS-x64.master.tgz REDISGEARS_S3_PATH=http://redismodules.s3.amazonaws.com/redisgears/snapshots/$REDISGEARS_ZIP @@ -29,5 +29,5 @@ if [[ -f "$WORK_DIR$REDISGEARS_DEPS" ]]; then else echo "Download RedisGears deps" wget -q -P $WORK_DIR $REDISGEARS_DEPS_S3_PATH - tar -C $WORK_DIR --no-same-owner -xf $WORK_DIR$REDISGEARS_DEPS + tar -C $WORK_DIR --no-same-owner -xf $WORK_DIR/$REDISGEARS_DEPS fi diff --git a/Makefile b/Makefile index 6783117..4c45adb 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ InstallOpenJDK: ./Install_OpenJDK.sh GearsRuntime: - cd gears_runtime; mvn package + cd gears_runtime; mvn package -q clean: make -C src clean diff --git a/pytest/run_test.sh b/pytest/run_test.sh index 8f123c7..97ca9c6 100755 --- a/pytest/run_test.sh +++ b/pytest/run_test.sh @@ -7,9 +7,9 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" ROOT=$(cd $HERE/..; pwd) export READIES=$ROOT/deps/readies -mkdir -p $ROOT/gears_tests/build -cd $ROOT/gears_tests/build -$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath $ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar $ROOT/src/gears_tests/* +mkdir -p $ROOT/pytest/gears_tests/build +cd $ROOT/pytest/gears_tests/build +$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath $ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar $ROOT/pytest/gears_tests/src/gears_tests/*.java $ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cvf gears_tests.jar ./gears_tests/ cd $ROOT @@ -28,15 +28,21 @@ JVM_PATH=$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/ SRC=$ROOT/src BIN=$ROOT/bin -RLTEST_ARGS=\ - --module $BIN/RedisGears/redisgears.so \ - --module-args \ - "Plugin $SRC/gears_jvm.so \ - JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS \ - Plugin $BIN/RedisGears/plugin/gears_python.so \ - CreateVenv 0 \ - PythonInstallationDir $BIN/RedisGears/" \ +argsf=$(mktemp /tmp/jojo.XXXXXX) +cat < $argf + --module $BIN/RedisGears/redisgears.so + --module-args " + Plugin $SRC/gears_jvm.so + JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS + Plugin $BIN/RedisGears/plugin/gears_python.so + CreateVenv 0 + PythonInstallationDir $BIN/RedisGears/ + " --clear-logs +EOF + +RLTEST_ARGS=$(readarray -t A < $argsf; IFS=' '; echo "${A[*]}") +rm -f $f echo oss python3 -m RLTest $RLTEST_ARGS "$@" From 6fd66322a41586326db18fa8ebb84b05f9494220 Mon Sep 17 00:00:00 2001 From: rafie Date: Tue, 26 Jan 2021 23:10:17 +0200 Subject: [PATCH 6/8] fixes 5 --- pytest/run_test.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pytest/run_test.sh b/pytest/run_test.sh index 97ca9c6..2e86a95 100755 --- a/pytest/run_test.sh +++ b/pytest/run_test.sh @@ -10,7 +10,7 @@ export READIES=$ROOT/deps/readies mkdir -p $ROOT/pytest/gears_tests/build cd $ROOT/pytest/gears_tests/build $ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/javac -d ./ -classpath $ROOT/gears_runtime/target/gear_runtime-jar-with-dependencies.jar $ROOT/pytest/gears_tests/src/gears_tests/*.java -$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cvf gears_tests.jar ./gears_tests/ +$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/bin/jar -cf gears_tests.jar ./gears_tests/ cd $ROOT JVM_OPTIONS="-Djava.class.path=" @@ -28,12 +28,13 @@ JVM_PATH=$ROOT/bin/OpenJDK/jdk-11.0.9.1+1/ SRC=$ROOT/src BIN=$ROOT/bin -argsf=$(mktemp /tmp/jojo.XXXXXX) -cat < $argf +argsf=$(mktemp /tmp/args.XXXXXX) +cat < $argsf --module $BIN/RedisGears/redisgears.so --module-args " Plugin $SRC/gears_jvm.so - JvmPath $JVM_PATH JvmOptions $JVM_OPTIONS + JvmPath $JVM_PATH + JvmOptions $JVM_OPTIONS Plugin $BIN/RedisGears/plugin/gears_python.so CreateVenv 0 PythonInstallationDir $BIN/RedisGears/ @@ -42,7 +43,7 @@ cat < $argf EOF RLTEST_ARGS=$(readarray -t A < $argsf; IFS=' '; echo "${A[*]}") -rm -f $f +rm -f $argsf echo oss python3 -m RLTest $RLTEST_ARGS "$@" From de6ed62b3b2e378bff2797f1033a4ba16d434f11 Mon Sep 17 00:00:00 2001 From: rafie Date: Wed, 27 Jan 2021 01:18:08 +0200 Subject: [PATCH 7/8] fixes 6 --- Install_RedisGears.sh | 6 +++--- pytest/run_test.sh | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Install_RedisGears.sh b/Install_RedisGears.sh index ba4b49c..bd72833 100755 --- a/Install_RedisGears.sh +++ b/Install_RedisGears.sh @@ -16,15 +16,15 @@ REDISGEARS_DEPS_S3_PATH=http://redismodules.s3.amazonaws.com/redisgears/snapshot mkdir -p $WORK_DIR -if [[ -f "$WORK_DIR$REDISGEARS_ZIP" ]]; then +if [[ -f "$WORK_DIR/$REDISGEARS_ZIP" ]]; then echo "Skiping RedisGears download" else echo "Download RedisGears" wget -q -P $WORK_DIR $REDISGEARS_S3_PATH - unzip -q $WORK_DIR$REDISGEARS_ZIP -d $WORK_DIR + unzip -q $WORK_DIR/$REDISGEARS_ZIP -d $WORK_DIR fi -if [[ -f "$WORK_DIR$REDISGEARS_DEPS" ]]; then +if [[ -f "$WORK_DIR/$REDISGEARS_DEPS" ]]; then echo "Skiping RedisGears download" else echo "Download RedisGears deps" diff --git a/pytest/run_test.sh b/pytest/run_test.sh index 2e86a95..77dc084 100755 --- a/pytest/run_test.sh +++ b/pytest/run_test.sh @@ -46,15 +46,15 @@ RLTEST_ARGS=$(readarray -t A < $argsf; IFS=' '; echo "${A[*]}") rm -f $argsf echo oss -python3 -m RLTest $RLTEST_ARGS "$@" +eval python3 -m RLTest $RLTEST_ARGS "$@" echo cluster 1 shard -python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 1 "$@" +eval python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 1 "$@" echo cluster 2 shards -python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 2 "$@" +eval python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 2 "$@" echo cluster 3 shards -python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 3 "$@" +eval python3 -m RLTest $RLTEST_ARGS --env oss-cluster --shards-count 3 "$@" rm -rf $ROOT/bin/RedisGears/.venv-* From 9b06496b11410e6caf21cb574b49e6f8d4ffde36 Mon Sep 17 00:00:00 2001 From: rafie Date: Wed, 27 Jan 2021 01:20:48 +0200 Subject: [PATCH 8/8] fixes 7 --- pytest/run_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest/run_test.sh b/pytest/run_test.sh index 77dc084..03db623 100755 --- a/pytest/run_test.sh +++ b/pytest/run_test.sh @@ -45,6 +45,8 @@ EOF RLTEST_ARGS=$(readarray -t A < $argsf; IFS=' '; echo "${A[*]}") rm -f $argsf +cd $ROOT/pytest + echo oss eval python3 -m RLTest $RLTEST_ARGS "$@"