merge releave/v1.4.0 into master

.github/workflows/build.yaml

@@ -0,0 +1,33 @@
+name: Java CI
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      # Step that does that actual cache save and restore
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Build & Test with Maven
+        id: build
+        run: mvn clean package
+
+    #  - name: Run integration tests
+    #    if: steps.build.outputs.exit_code == 0
+    #    run: mvn -Pintegration-tests clean verify

.travis.yml

@@ -38,7 +38,7 @@ script:
   # Build
   # use travis_wait so it does not time_out after 10 minutes without output (unfortunately that seems to not work)
   # use -q so there's not too much output for travis (4Mb max)
-  - travis_wait mvn clean install -Pintegration-tests -q
+  - travis_wait 60 mvn clean install -Pintegration-tests -q
   # build assembly (there is currently missing jars in assembly when using mvn clean install...)
   - mvn clean package -DskipTests -q
   # Integrations tests

README.rst

@@ -36,6 +36,9 @@ Online documentation
 --------------------
 You can find the latest Logisland documentation, including a programming guide,
 on the `project web page. <http://logisland.readthedocs.io/en/latest/index.html>`_
+
+Or on this `site web as well. <https://logisland.github.io/docs/>`_
+
 This README file only contains basic setup instructions.
 
 Browse the `Java API documentation <http://logisland.readthedocs.io/en/latest/_static/apidocs/>`_ for more information.
@@ -55,7 +58,7 @@ to build from the source just clone source and package with maven (logisland req
     cd logisland
     mvn clean package
 
-the final package is available at `logisland-assembly/target/logisland-1.3.0-bin.tar.gz`
+the final package is available at `logisland-assembly/target/logisland-1.4.0-full-bin.tar.gz`
 
 You can also download the `latest release build <https://github.com/Hurence/logisland/releases>`_
 
@@ -93,9 +96,9 @@ But you should choose the Spark version that is compatible with your environment
     curl -s http://d3kbcqa49mib13.cloudfront.net/spark-<spark-version>-bin-hadoop<hadoop-version>.tgz | tar -xz -C /usr/local/
     export SPARK_HOME=/usr/local/spark-<spark-version>-bin-hadoop<hadoop-version>
 
-    # install Logisland 1.3.0
+    # install Logisland 1.4.0
     curl -s https://github.com/Hurence/logisland/releases/download/v1.0.0-RC2/logisland-1.0.0-RC2-bin.tar.gz  | tar -xz -C /usr/local/
-    cd /usr/local/logisland-1.3.0
+    cd /usr/local/logisland-1.4.0
 
     # launch a logisland job
     bin/logisland.sh --conf conf/index-apache-logs.yml
@@ -124,9 +127,9 @@ Launching logisland streaming apps is just easy as unarchiving logisland distrib
 
 .. code-block:: sh
 
-    # install Logisland 1.3.0
-    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-1.3.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
-    cd /usr/local/logisland-1.3.0
+    # install Logisland 1.4.0
+    curl -s https://github.com/Hurence/logisland/releases/download/v0.10.0/logisland-1.4.0-bin-hdp2.5.tar.gz  | tar -xz -C /usr/local/
+    cd /usr/local/logisland-1.4.0
     bin/logisland.sh --conf conf/index-apache-logs.yml
 
 
@@ -147,7 +150,7 @@ The first part is the `ProcessingEngine` configuration (here a Spark streaming e
 
 .. code-block:: yaml
 
-    version: 1.3.0
+    version: 1.4.0
     documentation: LogIsland job config file
     engine:
       component: com.hurence.logisland.engine.spark.KafkaStreamProcessingEngine
@@ -261,3 +264,8 @@ Contributing
 ------------
 
 Please review the `Contribution to Logisland guide <http://logisland.readthedocs.io/en/latest/developer.html>`_ for information on how to get started contributing to the project.
+
+
+
+Start a stream processing job
+-----------------------------

RELEASING.rst

@@ -3,26 +3,79 @@ Releasing guide
 
 This guide will help you to perform the full release process for Logisland framework.
 
+Logisland sources follow the `GitFlow <https://datasift.github.io/gitflow/IntroducingGitFlow.html>`_ process.
+So be sure that any changes from the latest available release have been back-ported to the master and develop branch.
 
-    git hf release start v1.3.0
+The develop branch must also include all of your new features that you want to release compared to the current existing release.
 
-    # update the version (you should run a dry run first)
-    # you should double escapte dots so it is correctly parsed, otherwise it will be considered as the any character
-    ./update-version.sh -o 0\\.14\\.0 -n 1.3.0 -d
-    ./update-version.sh -o 0\\.14\\.0 -n 1.3.0
+Start a new branch and bump the version
+---------------------------------------
 
-Build the code and run the tests
---------------------------------
+Be sure `HubFlow git extension <https://github.com/datasift/gitflow>`_ is installed on your system. Then:
+
+.. code-block:: sh
+
+    git hf release start v1.4.0
+
+Update the version with the bump_version.sh script. Usage: bump_version.sh <old_version> <new_version>. You should double escape dots (in the old version only) so it is correctly parsed, otherwise it will be considered as the any character
+
+.. code-block:: sh
+
+    bump_version.sh 1\.2\.0 1.4.0
+
+Build the code, run the unit tests as well as integration tests
+---------------------------------------------------------------
+
+The following commands must be run from the top-level directory:
 
+.. code-block:: sh
+
+    mvn -Pintegration-tests clean verify
+
+.. note::
+   When doing a new release, all available unit tests and integration tests should be run and pass.
+   If you know what your are doing, you can however use those additional information:
 
-The following commands must be run from the top-level directory.
+To only build and run unit tests without integration tests:
 
 .. code-block:: sh
 
     mvn clean package
 
-If you wish to skip the unit tests you can do this by adding `-DskipTests` to the command line.
+If you wish to skip the unit tests in any of those commands, you can do this by adding `-DskipTests` to the command line.
+
+Build logisland docker image
+----------------------------
+
+WARNING: you must build the logisland archive with the following command at the root directory of the workspace:
 
+.. code-block:: sh
+
+    mvn -DskipTests clean package
+
+You must not build it with the integration tests run command, as the final packaging will not be spring compliant and
+you will have some errors like this one once you run the docker based quickstarts:
+
+.. code-block:: sh
+
+    Exception in thread "main" java.lang.NoClassDefFoundError: org/springframework/boot/loader/archive/Archive
+
+Once the workspace is built, you must locally create with it the docker image that you will test.
+To do that, follow instructions in logisland-docker/full-container/README.rst, in the 'Build your own' section.
+
+Run manual sanity checking using the QuickStarts
+------------------------------------------------
+
+Run some of the `QuickStarts <https://github.com/Hurence/logisland-quickstarts>`_.
+Those tests use the logisland docker image.
+
+You must at least successfully run the `Getting Started Guide <https://logisland.github.io/docs/guides/getting-started-guide>`_
+and the `Datastore Elasticsearch Guide <https://logisland.github.io/docs/guides/datastore-elasticsearch-guide>`_
+
+Update the release notes
+------------------------
+
+When all the automatic and manual tests are ok, update the release notes with what's new in the new version in logisland-documentation/changes.rst.
 
 Release to maven repositories
 -----------------------------
@@ -34,21 +87,16 @@ to release artifacts (if you're allowed to), follow this guide `release to OSS S
     mvn -DperformRelease=true clean deploy
     mvn versions:commit
 
-
-follow the staging procedure in `oss.sonatype.org <https://oss.sonatype.org/#stagingRepositories>`_ or read `Sonatype book <http://books.sonatype.com/nexus-book/reference/staging-deployment.html#staging-maven>`_
+Follow the staging procedure in `oss.sonatype.org <https://oss.sonatype.org/#stagingRepositories>`_ or read `Sonatype book <http://books.sonatype.com/nexus-book/reference/staging-deployment.html#staging-maven>`_
 
 go to `oss.sonatype.org <https://oss.sonatype.org/#stagingRepositories>`_ to release manually the artifact
 
-
-
 Publish release assets to github
 --------------------------------
 
 please refer to `https://developer.github.com/v3/repos/releases <https://developer.github.com/v3/repos/releases>`_
 
-curl -XPOST https://uploads.github.com/repos/Hurence/logisland/releases/v1.3.0/assets?name=logisland-1.1.2-bin-hdp2.5.tar.gz -v  --data-binary  @logisland-assembly/target/logisland-0.10.3-bin-hdp2.5.tar.gz --user oalam -H 'Content-Type: application/gzip'
-
-
+curl -XPOST https://uploads.github.com/repos/Hurence/logisland/releases/v1.4.0/assets?name=logisland-full-bin.tar.gz -v  --data-binary  @logisland-assembly/target/logisland-full-bin.tar.gz --user oalam -H 'Content-Type: application/gzip'
 
 Publish Docker image
 --------------------
@@ -62,16 +110,23 @@ Building the image
     # verify image build
     docker images
 
-
 then login and push the latest image
 
 .. code-block:: sh
 
     docker login
     docker push hurence/logisland
 
-
 Publish artifact to github
 --------------------------
 
 Tag the release + upload latest tgz
+
+Merge back changes from the release into master and close release branch
+------------------------------------------------------------------------
+
+.. code-block:: sh
+
+    git hf release finish v1.4.0
+
+TBD: does this also generate the release tag?

bump_version.sh

@@ -18,7 +18,7 @@ function bump_files() {
     for i in `find . -name "*.yml"` ; do
         bump $i "version: $current_version" "version: $new_version"
     done
-    for i in `find . -name "*.rst"` ; do
+    for i in `find . -name "*.rst"|grep -v changes.rst|grep -v RELEASING.rst` ; do
         bump $i "$current_version" "$new_version"
     done
 }
@@ -27,7 +27,7 @@ function bump() {
 	echo -n "Updating $1..."
 	tmp_file=$(mktemp)
 	rm -f "$tmp_file"
-	sed -i '' "s/$2/$3/1w $tmp_file" $1
+	sed -i "s/$2/$3/1w $tmp_file" $1
 	if [ -s "$tmp_file" ]; then
 		echo "Done"
 	else