From ce4ad76f40b5d232329c7c58c9a9559d088a4dff Mon Sep 17 00:00:00 2001 From: Robin Edwards <148556345+jrobinedwards@users.noreply.github.com> Date: Fri, 22 Mar 2024 09:56:56 +0100 Subject: [PATCH 1/3] Update install_from_sources.md Add other instructions --- .../install_from_sources.md | 164 +++++++++++++++--- 1 file changed, 137 insertions(+), 27 deletions(-) diff --git a/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md b/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md index f68e5997c..ed4783301 100644 --- a/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md +++ b/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md @@ -4,12 +4,51 @@ Content: * [Prerequisites](#section1) * [Installing cygnus-ngsi](#section2) * [Cloning `fiware-cygnus`](#section2.1) - * [Installing `cygnus-ngsi`](#section2.2) - * [Known issues](#section2.3) -* [Installing dependencies](#section3) + * [Installing `cygnus-common`](#section2.2) + * [Installing `cygnus-ngsi`](#section2.3) + * [Setup `cygnus`](#section2.4) + * [Setup Agents](#section2.5) + * [Running `cygnus-ngsi`](#section2.6) +* [SSL Support](#section3) + * [Prerequisites](#section3.1) + * [Setup](#section3.2) + +Please note that the commands listed here are for Debian systems. ## Prerequisites -[`cygnus-common`](../../cygnus-common/installation_and_administration_guide/install_from_sources.md) must be installed. This includes Maven, `cygnus` user creation, `log4j` path creation, Apache Flume and `cygnus-flume-ng` script installation. +Install JDK 17 + + $ sudo apt-get -y install openjdk-17-jdk openjdk-17-jdk-headless openjdk-17-jre openjdk-17-jre-headless + +Add $JAVA_HOME to $PATH + + $ nano .bashrc + $ export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 + $ export PATH=$PATH:$JAVA_HOME + +Install Maven + + $ sudo apt-get -y install maven + +Create `cygnus` user + + $ adduser cygnus + +Install Apache Flume + + $ wget https://downloads.apache.org/flume/1.11.0/apache-flume-1.11.0-bin.tar.gz + $ tar xzf apache-flume-1.11.0-bin.tar.gz + $ mv apache-flume-1.11.0-bin /opt/apache-flume-1.11.0 + $ mkdir -p /opt/apache-flume-1.11.0/plugins.d/cygnus + $ mkdir -p /opt/apache-flume-1.11.0/plugins.d/cygnus/lib + $ mkdir -p /opt/apache-flume-1.11.0/plugins.d/cygnus/libext + $ chown -R cygnus:cygnus /opt/apache-flume-1.11.0/ + +Add FLUME_HOME to $PATH + + $ nano .bashrc + $ export FLUME_HOME=/opt/apache-flume-1.11.0 + $ export PATH=$PATH:$FLUME_HOME/bin [Top](#top) @@ -25,40 +64,111 @@ Start by cloning the Github repository: [Top](#top) -### Installing `cygnus-ngsi` -`cygnus-ngsi` can be built as a fat Java jar file containing all third-party dependencies (**recommended**): +### Installing `cygnus-common` - $ cd cygnus-ngsi - $ APACHE_MAVEN_HOME/bin/mvn clean compile exec:exec assembly:single - $ cp target/cygnus-ngsi--jar-with-dependencies.jar APACHE_FLUME_HOME/plugins.d/cygnus/lib + $ cd fiware-cygnus/cygnus-common + $ mvn -B -T8 clean compile exec:exec assembly:single + $ cp target/cygnus-common-3.6.0-SNAPSHOT-jar-with-dependencies.jar /opt/apache-flume-1.11.0/plugins.d/cygnus/libext/ + $ mvn install:install-file -Dfile=/opt/apache-flume-1.11.0/plugins.d/cygnus/libext/cygnus-common-3.6.0-SNAPSHOT-jar-with-dependencies.jar -DgroupId=com.telefonica.iot -DartifactId=cygnus-common -Dversion=3.6.0-SNAPSHOT -Dpackaging=jar -DgeneratePom=false + +[Top](#top) -Or as a thin Java jar file: +### Installing `cygnus-ngsi` - $ cd cygnus-ngsi - $ APACHE_MAVEN_HOME/bin/mvn exec:exec package - $ cp target/cygnus-.jar APACHE_FLUME_HOME/plugins.d/cygnus/lib + $ cd fiware-cygnus/cygnus-ngsi + $ mvn -B -T8 clean compile exec:exec assembly:single + $ cp target/cygnus-ngsi-3.6.0-SNAPSHOT-jar-with-dependencies.jar /opt/apache-flume-1.11.0/plugins.d/cygnus/lib/ Finally, please find a `compile.sh` script containing all the commands shown in this section. It must be parameterized with the version of the current branch and the Apache Flume base path. [Top](#top) -### Known issues -It may happen while compiling `cygnus-ngsi` the Maven JVM has not enough memory. This can be changed as detailed at the [Maven official documentation](https://cwiki.apache.org/confluence/display/MAVEN/OutOfMemoryError): +### Setup Cygnus + + $ cp fiware-cygnus/cygnus-common/target/classes/cygnus-flume-ng /opt/apache-flume-1.11.0/bin/ + $ chmod +x /opt/apache-flume-1.11.0/bin/cygnus-flume-ng + $ cp fiware-cygnus/cygnus-common/conf/log4j2.properties.template /opt/apache-flume-1.11.0/conf/log4j2.properties + $ cp fiware-cygnus/cygnus-ngsi/conf/name_mappings.conf.template /opt/apache-flume-1.11.0/conf/name_mappings.conf + $ mkdir -p /var/log/cygnus + $ chown -R cygnus:cygnus /var/log/cygnus + +Clean up + + $ cd fiware-cygnus/cygnus-common + $ mvn -B -T8 clean + $ cd fiware-cygnus/cygnus-ngsi + $ mvn -B -T8 clean> + +[Top](#top) + +### Setup Agents +Setup Cygnus-Common and Cygnus-Ngsi agents + +### Running `cygnus-ngsi` +Please make sure that you have ports 5080 (admin port) and 5050 (default instance port) open. - $ export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=128m" + $ /opt/apache-flume-1.11.0/bin/cygnus-flume-ng agent --conf {{PATH-TO-CONF-FOLDER}} -f {{PATH-TO-AGENT-NGSI-CONF}} -n {{AGENT-NAME}} -p 5080 -Dflume.root.logger=INFO,LOGFILE -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dflume.monitoring.type= -Dflume.monitoring.port=41414 [Top](#top) -## Installing dependencies -These are the packages you will need to install under `APACHE_FLUME_HOME/plugins.d/cygnus/libext/` **if you did not included them in the cygnus-common jar**: - -| Cygnus dependencies | Version | Required by / comments | -|---|---|---| -| mockito-all | 1.9.5 | Unit tests | -| junit | 4.11 | Unit tests | -| httpcore | 4.3.1 | Overwrites the one bundled in Apache Flume | -| log4j | 1.2.17 | Logging | -| slf4j-simple | 1.7.21 | Logging | -| cygnus-common | latest | | +## SSL Support +Apache Flume has SSL support, mentioned here + +### Prerequisites for this is that you should have: +SSL certificated installed (using Certbot & let's encrypt) +Before doing this make sure that port 80 and 443 are open and that you have a domain assoicated with the IP of the server you are installing SSL on. +Install Certbot with snap + + $ sudo apt install snapd + $ sudo snap install core; sudo snap refresh core + $ sudo apt remove certbot + $ sudo snap install --classic certbot + $ sudo ln -s /snap/bin/certbot /usr/bin/certbot + +Issue certificates + + $ sudo certbot certonly --standalone -d DOMAINNAME + +Java Keystore setup using the above certificates. +These commands can be put in a bash file and run as a deployment hook, so whenever the certificates renewed the Java Keystore can be updated. +Replace all values withing {{}} + + $ openssl pkcs12 -export \ + -in {{ABSOLUTE_PATH_TO_LETSENCRYPT_FULLCHAIN}} \ + -inkey {{ABSOLUTE_PATH_TO_LETSENCRYPT_PRIVKEY}} \ + -out /tmp/cert.p12 \ + -name letsencrypt-keystore \ + -passout pass:{{TEMPORARY_PASSWORD}} \ + > /dev/null 2>&1 + +Import PKCS12 file into keystore + + $ keytool -noprompt -importkeystore \ + -srckeystore /tmp/cert.p12 \ + -srcstoretype PKCS12 \ + -srcstorepass {{TEMPORARY_PASSWORD}} \ + -deststorepass {{KEYSTORE_PASSWORD}} \ + -destkeypass {{PRIVATE_KEY_PASSWORD}} \ + -destkeystore {{ABSOLUTE_PATH_TO_KEYSTORE_IN_CONF_FOLDER}} \ + -alias letsencrypt-keystore \ + > /dev/null 2>&1 + +Delete PKCS12 file + + $ rm /tmp/cert.p12 + $ chown cygnus:cygnus /opt/apache-flume-1.11.0/conf/letsencrypt-keystore.jks` + +### Setup +Clone /conf/flume-env.sh.template to /conf/flume-env.sh and edit to add the following: + + $ export JAVA_OPTS="$JAVA_OPTS -Djavax.net.ssl.keyStore=/path/to/letsencrypt-keystore.jks"` + $ export JAVA_OPTS="$JAVA_OPTS -Djavax.net.ssl.keyStorePassword=password"` + $ export FLUME_SSL_KEYSTORE_PATH=/path/to/etsencrypt-keystore.jks` + $ export FLUME_SSL_KEYSTORE_PASSWORD=password` + +Edit agent_ngsi config and add the following lines according to Flume http-source + + $ cygnus-ngsi.sources.http-source.ssl = true` + [Top](#top) From e4e2561b45eb77845b1312f08f8410b65837f32d Mon Sep 17 00:00:00 2001 From: Robin Edwards <148556345+jrobinedwards@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:12:28 +0100 Subject: [PATCH 2/3] Update install_from_sources.md add breaks --- .../install_from_sources.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md b/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md index ed4783301..b5ee42696 100644 --- a/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md +++ b/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md @@ -105,7 +105,7 @@ Clean up Setup Cygnus-Common and Cygnus-Ngsi agents ### Running `cygnus-ngsi` -Please make sure that you have ports 5080 (admin port) and 5050 (default instance port) open. +Please make sure that you have ports 5080 (admin port) and 5050 (default instance port) open.
$ /opt/apache-flume-1.11.0/bin/cygnus-flume-ng agent --conf {{PATH-TO-CONF-FOLDER}} -f {{PATH-TO-AGENT-NGSI-CONF}} -n {{AGENT-NAME}} -p 5080 -Dflume.root.logger=INFO,LOGFILE -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dflume.monitoring.type= -Dflume.monitoring.port=41414 @@ -115,8 +115,8 @@ Please make sure that you have ports 5080 (admin port) and 5050 (default instanc Apache Flume has SSL support, mentioned here ### Prerequisites for this is that you should have: -SSL certificated installed (using Certbot & let's encrypt) -Before doing this make sure that port 80 and 443 are open and that you have a domain assoicated with the IP of the server you are installing SSL on. +SSL certificated installed (using Certbot & let's encrypt)
+Before doing this make sure that port 80 and 443 are open and that you have a domain assoicated with the IP of the server you are installing SSL on.
Install Certbot with snap $ sudo apt install snapd @@ -129,8 +129,8 @@ Issue certificates $ sudo certbot certonly --standalone -d DOMAINNAME -Java Keystore setup using the above certificates. -These commands can be put in a bash file and run as a deployment hook, so whenever the certificates renewed the Java Keystore can be updated. +Java Keystore setup using the above certificates.
+These commands can be put in a bash file and run as a deployment hook, so whenever the certificates renewed the Java Keystore can be updated.
Replace all values withing {{}} $ openssl pkcs12 -export \ From c5bdf16371f1a02afe48ee13fb9d1f1e1deb93c6 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 16 Apr 2024 15:28:33 +0200 Subject: [PATCH 3/3] Update doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- .../install_from_sources.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md b/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md index b5ee42696..2f3ba7ed8 100644 --- a/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md +++ b/doc/cygnus-ngsi/installation_and_administration_guide/install_from_sources.md @@ -112,7 +112,7 @@ Please make sure that you have ports 5080 (admin port) and 5050 (default instanc [Top](#top) ## SSL Support -Apache Flume has SSL support, mentioned here +Apache Flume has SSL support, [mentioned here](https://flume.apache.org/FlumeUserGuide.html#ssl-tls-support) ### Prerequisites for this is that you should have: SSL certificated installed (using Certbot & let's encrypt)