diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 1f689c4813..8cb134acb9 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -146,6 +146,11 @@ solr::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" # Default but needed here to make sure, hue uses the same port solr::server::port: "8983" +knox::gateway::kerberos_realm: "%{hiera('kerberos::site::realm')}" +# Default but needed here to make sure, hue uses the same port +knox::gateway::port: "8443" + +hadoop_oozie::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" hcatalog::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" hcatalog::webhcat::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index fce6ad9ea8..54fb891714 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -19,6 +19,7 @@ hadoop::hadoop_storage_dirs: # - hcat # - hive # - httpfs +# - knox # - mapred-app # - solr # - spark diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index ec81ada94e..46501c73d8 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -125,6 +125,9 @@ }, ranger => { master => ["ranger-server"], + }, + knox => { + master => ["knox-gateway"], } } @@ -189,6 +192,7 @@ "bigtop_utils", "phoenix", "ranger", + "knox", ] node_with_roles::deploy_module { $modules: diff --git a/bigtop-deploy/puppet/modules/knox/manifests/init.pp b/bigtop-deploy/puppet/modules/knox/manifests/init.pp new file mode 100644 index 0000000000..79d2148ae0 --- /dev/null +++ b/bigtop-deploy/puppet/modules/knox/manifests/init.pp @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class knox { + + class deploy ($roles) { + if ('knox-gateway' in $roles) { + include knox::gateway + } + } + + class gateway( + $port = "8443", + ) { + + package { 'knox': + ensure => latest, + } + + file { '/etc/knox/conf/gateway-site.xml': + content => template('knox/gateway-site.xml'), + require => [ Package['knox'] ], + owner => 'knox', + group => 'knox', + } + + file { '/etc/knox/conf/topologies/sandbox.xml': + content => template('knox/sandbox.xml'), + require => [ Package['knox'] ], + owner => 'knox', + group => 'knox', + } + + service { 'knox-gateway': + ensure => running, + subscribe => [ + Package['knox'], + File['/etc/knox/conf/gateway-site.xml'], + ], + hasrestart => true, + hasstatus => true, + } + + if ($kerberos_realm and $kerberos_realm != "") { + require kerberos::client + + kerberos::host_keytab { "knox": + spnego => true, + require => Package["knox"], + before => Service["knox-gateway"], + } + } + } +} diff --git a/bigtop-deploy/puppet/modules/knox/templates/gateway-site.xml b/bigtop-deploy/puppet/modules/knox/templates/gateway-site.xml new file mode 100644 index 0000000000..cfef357303 --- /dev/null +++ b/bigtop-deploy/puppet/modules/knox/templates/gateway-site.xml @@ -0,0 +1,167 @@ + + + + + + gateway.service.alias.impl + org.apache.knox.gateway.services.security.impl.RemoteAliasService + + + gateway.port + <%= @port %> + The HTTP port for the Gateway. + + + + gateway.path + gateway + The default context path for the gateway. + + + + gateway.gateway.conf.dir + deployments + The directory within GATEWAY_HOME that contains gateway topology files and deployments. + + + + gateway.hadoop.kerberos.secured + <% if @hadoop_security_authentication == "kerberos" %>true<% else %>false<% end %> + Boolean flag indicating whether the Hadoop cluster protected by Gateway is secured with Kerberos + +<% if @hadoop_security_authentication == "kerberos" -%> + + java.security.krb5.conf + /etc/krb5.conf + Absolute path to krb5.conf file + + + + java.security.auth.login.config + /etc/knox/conf/krb5JAASLogin.conf + Absolute path to JAAS login config file + + + + sun.security.krb5.debug + false + Boolean flag indicating whether to enable debug messages for krb5 authentication + +<% end -%> + + + + gateway.websocket.feature.enabled + false + Enable/Disable websocket feature. + + + + gateway.scope.cookies.feature.enabled + false + Enable/Disable cookie scoping feature. + + + + gateway.cluster.config.monitor.ambari.enabled + false + Enable/disable Ambari cluster configuration monitoring. + + + + gateway.cluster.config.monitor.ambari.interval + 60 + The interval (in seconds) for polling Ambari for cluster configuration changes. + + + + + knox.homepage.logout.enabled + true + Enable/disable logout from the Knox Homepage. + + + + + gateway.knox.token.eviction.grace.period + 0 + A duration (in seconds) beyond a token’s expiration to wait before evicting its state. This configuration only applies when server-managed token state is enabled either in gateway-site or at the topology level. + + + + + gateway.knox.admin.groups + admin + + + + + gateway.group.config.hadoop.security.group.mapping + org.apache.hadoop.security.LdapGroupsMapping + + + gateway.group.config.hadoop.security.group.mapping.ldap.bind.user + uid=guest,ou=people,dc=hadoop,dc=apache,dc=org + + + gateway.group.config.hadoop.security.group.mapping.ldap.bind.password + guest-password + + + gateway.group.config.hadoop.security.group.mapping.ldap.url + ldap://localhost:33389 + + + gateway.group.config.hadoop.security.group.mapping.ldap.base + + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.filter.user + (&(|(objectclass=person)(objectclass=applicationProcess))(cn={0})) + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.filter.group + (objectclass=groupOfNames) + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.attr.member + member + + + gateway.group.config.hadoop.security.group.mapping.ldap.search.attr.group.name + cn + + + gateway.dispatch.whitelist.services + DATANODE,HBASEUI,HDFSUI,JOBHISTORYUI,NODEUI,YARNUI,knoxauth + The comma-delimited list of service roles for which the gateway.dispatch.whitelist should be applied. + + + gateway.dispatch.whitelist + DEFAULT + The whitelist to be applied for dispatches associated with the service roles specified by gateway.dispatch.whitelist.services. + If the value is DEFAULT, a domain-based whitelist will be derived from the Knox host. + + + gateway.xforwarded.header.context.append.servicename + LIVYSERVER + Add service name to x-forward-context header for the list of services defined above. + + + diff --git a/bigtop-deploy/puppet/modules/knox/templates/sandbox.xml b/bigtop-deploy/puppet/modules/knox/templates/sandbox.xml new file mode 100644 index 0000000000..99a23b2edb --- /dev/null +++ b/bigtop-deploy/puppet/modules/knox/templates/sandbox.xml @@ -0,0 +1,204 @@ + + + + + + + + authentication + ShiroProvider + true + + + sessionTimeout + 30 + + + main.ldapRealm + org.apache.knox.gateway.shirorealm.KnoxLdapRealm + + + main.ldapContextFactory + org.apache.knox.gateway.shirorealm.KnoxLdapContextFactory + + + main.ldapRealm.contextFactory + $ldapContextFactory + + + main.ldapRealm.userDnTemplate + uid={0},ou=people,dc=hadoop,dc=apache,dc=org + + + main.ldapRealm.contextFactory.url + ldap://localhost:33389 + + + main.ldapRealm.contextFactory.authenticationMechanism + simple + + + urls./** + authcBasic + + + + + identity-assertion + Default + true + + + + hostmap + static + true + + localhost + sandbox,sandbox.hortonworks.com + + + + + + + AMBARI + http://localhost:8080 + + + + FLINK + http://localhost:8081 + + + + RESOURCEMANAGER + http://localhost:8088/ws + + + + YARNUI + http://localhost:8088/ws + + + + YARNUIV2 + http://localhost:8088/ws + + + + NAMENODE + hdfs://localhost:8020 + + + + HDFSUI + http://localhost:50070 + + + + WEBHDFS + http://localhost:50070/webhdfs + + + + WEBHCAT + http://localhost:50111/templeton + + + + HIVE + http://localhost:10001/cliservice + + replayBufferSize + 8 + + + + + HBASEUI + http://localhost:16010 + + + + WEBHBASE + http://localhost:60080 + + replayBufferSize + 8 + + + + + OOZIE + http://localhost:11000 + + + + OOZIEUI + http://localhost:11000 + + + + KAFKA + tcp://localhost:9092 + + + + LIVYSERVER + http://localhost:8998 + + + + SOLR + http://localhost:8983 + + + + SPARK3HISTORYUI + http://localhost:8080 + + + + RANGER + http://localhost:6080 + + + + RANGERUI + http://localhost:6080 + + + + ZEPPELINUI + http://localhost:9995 + + + + ZEPPELINWS + http://localhost:9995 + + + + + + \ No newline at end of file diff --git a/bigtop-packages/src/common/knox/do-component-build b/bigtop-packages/src/common/knox/do-component-build new file mode 100644 index 0000000000..9fb9662312 --- /dev/null +++ b/bigtop-packages/src/common/knox/do-component-build @@ -0,0 +1,33 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +. `dirname $0`/bigtop.bom + +if [ $HOSTTYPE = "powerpc64le" ] ; then + echo "Starting quick-package" + ant quick-package + #replace build step with download for test purpases + #mkdir -p target/1.6.1 + #wget --no-check-certificate https://dlcdn.apache.org/knox/1.6.1/knox-1.6.1.tar.gz -P target/1.6.1 +else + echo "Starting quick-package" + ant quick-package + #replace build step with download for test purpases + #mkdir -p target/1.6.1 + #wget --no-check-certificate https://dlcdn.apache.org/knox/1.6.1/knox-1.6.1.tar.gz -P target/1.6.1 +fi \ No newline at end of file diff --git a/bigtop-packages/src/common/knox/install_knox.sh b/bigtop-packages/src/common/knox/install_knox.sh new file mode 100644 index 0000000000..868f112a1f --- /dev/null +++ b/bigtop-packages/src/common/knox/install_knox.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +usage() { + echo " +usage: $0 + Required not-so-options: + --build-dir=DIR path to knox dist.dir + --prefix=PREFIX path to install into + + Optional options: + --lib-dir=DIR path to install knox home [/usr/lib/knox] + --bin-dir=DIR path to install bins [/usr/bin] + --etc-knox=DIR path to install knox conf [/etc/knox] + ... [ see source for more similar options ] + " + exit 1 +} + +OPTS=$(getopt \ + -n $0 \ + -o '' \ + -l 'prefix:' \ + -l 'build-dir:' \ + -l 'bin-dir:' \ + -l 'lib-dir:' \ + -l 'etc-knox:' -- "$@") + +if [ $? != 0 ] ; then + usage +fi + +eval set -- "$OPTS" +while true ; do + case "$1" in + --prefix) + PREFIX=$2 ; shift 2 + ;; + --build-dir) + BUILD_DIR=$2 ; shift 2 + ;; + --bin-dir) + BIN_DIR=$2 ; shift 2 + ;; + --lib-dir) + LIB_DIR=$2 ; shift 2 + ;; + --etc-knox) + ETC_KNOX=$2 ; shift 2 + ;; + --) + shift ; break + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +for var in PREFIX BUILD_DIR ; do + if [ -z "$(eval "echo \$$var")" ]; then + echo Missing param: $var + usage + fi +done + +LIB_DIR=${LIB_DIR:-/usr/lib/knox} +BIN_DIR=${BIN_DIR:-/usr/bin} +ETC_KNOX=${ETC_KNOX:-/etc/knox} +RUN_DIR=${RUN_DIR:-/var/run/knox} +LOG_DIR=${LOG_DIR:-/var/log/knox} + +NP_ETC_KNOX=/etc/knox +NP_VAR_LIB_KNOX_DATA=/var/lib/knox/data + + +install -d -m 0755 $PREFIX/$LIB_DIR +install -d -m 0755 $PREFIX/$LIB_DIR/bin +install -d -m 0755 $PREFIX/$LIB_DIR/lib +install -d -m 0755 $PREFIX/$LIB_DIR/dep +install -d -m 0755 $PREFIX/$NP_ETC_KNOX +install -d -m 0755 $PREFIX/$NP_VAR_LIB_KNOX_DATA +install -d -m 0755 $PREFIX/$ETC_KNOX/conf.dist +install -d -m 0755 $PREFIX/$LIB_DIR/samples +install -d -m 0755 $PREFIX/$LIB_DIR/templates +install -d -m 0755 $PREFIX/$RUN_DIR +install -d -m 0755 $PREFIX/$LOG_DIR + +TMP_DIR=$BUILD_DIR/tmp +mkdir -p $BUILD_DIR/tmp +tar -zxf $BUILD_DIR/target/*.*.*/knox-*.tar.gz -C $TMP_DIR + +cp -ra ${TMP_DIR}/knox-*/dep/* ${PREFIX}/${LIB_DIR}/dep/ +cp -ra ${TMP_DIR}/knox-*/lib/* ${PREFIX}/${LIB_DIR}/lib/ +cp -a ${TMP_DIR}/knox-*/bin/* ${PREFIX}/${LIB_DIR}/bin/ +cp -a ${TMP_DIR}/knox-*/samples/* ${PREFIX}/${LIB_DIR}/samples/ +cp -a ${TMP_DIR}/knox-*/templates/* ${PREFIX}/${LIB_DIR}/templates/ +cp -ra ${TMP_DIR}/knox-*/data/* ${PREFIX}/${NP_VAR_LIB_KNOX_DATA} +cp -ra ${TMP_DIR}/knox-*/conf/* ${PREFIX}/${ETC_KNOX}/conf.dist + +ln -s $NP_ETC_KNOX/conf $PREFIX/$LIB_DIR/conf +ln -s $NP_VAR_LIB_KNOX_DATA $PREFIX/$LIB_DIR/data +ln -s $LOG_DIR $PREFIX/$LIB_DIR/logs +ln -s $RUN_DIR $PREFIX/$LIB_DIR/pids + +rm -rf $TMP_DIR + +# Copy in the /usr/bin/knox wrapper +install -d -m 0755 $PREFIX/$BIN_DIR +cat > $PREFIX/$BIN_DIR/gateway < +Build-Depends: debhelper (>= 9) +Standards-Version: 3.9.4 +Homepage: https://knox.apache.org/ + +Package: knox +Architecture: all +Depends: bigtop-utils (>= 0.7), adduser +Description: The Apache Knox Gateway is an Application Gateway for interacting with the REST APIs and UIs + of Apache Hadoop deployments. + The Knox Gateway provides a single access point for all REST and HTTP interactions with Apache Hadoop + clusters. \ No newline at end of file diff --git a/bigtop-packages/src/deb/knox/copyright b/bigtop-packages/src/deb/knox/copyright new file mode 100644 index 0000000000..c4b187d4cf --- /dev/null +++ b/bigtop-packages/src/deb/knox/copyright @@ -0,0 +1,15 @@ +Format: http://dep.debian.net/deps/dep5 +Source: http://knox.apache.org/ +Upstream-Name: Apache Knox + +Files: * +Copyright: 2013-2015, The Apache Software Foundation +License: Apache-2.0 + +Files debian/* +Copyright: 2011, The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + On Debian systems, the complete text of the Apache 2.0 license + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/bigtop-packages/src/deb/knox/knox.dirs b/bigtop-packages/src/deb/knox/knox.dirs new file mode 100644 index 0000000000..b70369f7ad --- /dev/null +++ b/bigtop-packages/src/deb/knox/knox.dirs @@ -0,0 +1,6 @@ +/usr/lib/knox +/var/log/knox +/var/run/knox +/var/lib/knox +/usr/bin +/etc/knox diff --git a/bigtop-packages/src/deb/knox/knox.install b/bigtop-packages/src/deb/knox/knox.install new file mode 100644 index 0000000000..d936093cfe --- /dev/null +++ b/bigtop-packages/src/deb/knox/knox.install @@ -0,0 +1,4 @@ +/etc/knox +/usr/lib/knox +/var/lib/knox +/usr/bin \ No newline at end of file diff --git a/bigtop-packages/src/deb/knox/knox.postinst b/bigtop-packages/src/deb/knox/knox.postinst new file mode 100644 index 0000000000..9afa2e9b0b --- /dev/null +++ b/bigtop-packages/src/deb/knox/knox.postinst @@ -0,0 +1,37 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + configure) + # Install config alternatives + update-alternatives --install /etc/knox/conf knox-conf /etc/knox/conf.dist 30 + chown -R knox:knox /var/log/knox /var/lib/knox /var/run/knox /etc/knox + chmod g+w /var/log/knox + chmod 0755 /var/run/knox + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# diff --git a/bigtop-packages/src/deb/knox/knox.preinst b/bigtop-packages/src/deb/knox/knox.preinst new file mode 100644 index 0000000000..41c20f7f0c --- /dev/null +++ b/bigtop-packages/src/deb/knox/knox.preinst @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# preinst script for knox +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `install' +# * `install' +# * `upgrade' +# * `abort-upgrade' +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + getent group knox >/dev/null || groupadd -r knox + if ! getent passwd knox >/dev/null; then + # Adding system user: knox . + adduser \ + --system \ + --ingroup knox \ + --home /var/lib/knox \ + --gecos "Knox" \ + --shell /bin/false \ + knox >/dev/null + fi + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/bigtop-packages/src/deb/knox/knox.prerm b/bigtop-packages/src/deb/knox/knox.prerm new file mode 100644 index 0000000000..507dbc26e7 --- /dev/null +++ b/bigtop-packages/src/deb/knox/knox.prerm @@ -0,0 +1,38 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + remove|upgrade|deconfigure) + update-alternatives --remove knox-conf /etc/knox/conf.dist || : + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/bigtop-packages/src/deb/knox/rules b/bigtop-packages/src/deb/knox/rules new file mode 100644 index 0000000000..b6340e4100 --- /dev/null +++ b/bigtop-packages/src/deb/knox/rules @@ -0,0 +1,41 @@ +#!/usr/bin/make -f + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -*- makefile -*- + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_build: + bash debian/do-component-build + +svcs=knox + +$(svcs): debian/init.d.tmpl + bash $< debian/knox-gateway.svc deb debian/$@.init + touch $@ + +override_dh_auto_install: $(svcs) + bash -x debian/install_knox.sh \ + --build-dir=`pwd` \ + --prefix=debian/tmp \ + diff --git a/bigtop-packages/src/deb/knox/source/format b/bigtop-packages/src/deb/knox/source/format new file mode 100644 index 0000000000..163aaf8d82 --- /dev/null +++ b/bigtop-packages/src/deb/knox/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/bigtop-packages/src/rpm/knox/SPECS/knox.spec b/bigtop-packages/src/rpm/knox/SPECS/knox.spec new file mode 100644 index 0000000000..267afaef72 --- /dev/null +++ b/bigtop-packages/src/rpm/knox/SPECS/knox.spec @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +%define knox_name knox +%define knox_pkg_name knox%{pkg_name_suffix} +%define etc_default %{parent_dir}/etc/default +%define usr_lib_knox %{parent_dir}/usr/lib/%{knox_name} +%define etc_knox %{parent_dir}/etc/%{knox_name} +%define bin_dir %{parent_dir}/%{_bindir} + +%define np_var_lib_knox_data /var/lib/%{knox_name}/data +%define np_var_run_knox /var/run/%{knox_name} +%define np_var_log_knox /var/log/%{knox_name} +%define np_etc_knox /etc/%{knox_name} + +%define knox_services gateway + +Name: %{knox_pkg_name} +Version: %{knox_version} +Release: %{knox_release} +BuildArch: noarch +Summary: Knox Gateway +URL: https://knox.apache.org/ +Group: Development/Libraries +License: ASL 2.0 +Source0: %{knox_name}-%{knox_base_version}.zip +Source1: do-component-build +Source2: install_%{knox_name}.sh +Source3: knox-gateway.svc +Source4: bigtop.bom +Source6: init.d.tmpl +Requires: bigtop-utils >= 0.7 +Requires(preun): /sbin/service +%if %{?suse_version:1}0 +# Required for init scripts +Requires: insserv +%else +Requires: /lib/lsb/init-functions +Requires: sh-utils, redhat-lsb +%endif + +AutoProv: no +AutoReqProv: no +#BIGTOP_PATCH_FILES + + +%if %{?suse_version:1}0 +%define alternatives_cmd update-alternatives +%else +%define alternatives_cmd alternatives +%endif + +%description +The Apache Knox Gateway is an Application Gateway for interacting with the REST APIs and UIs +of Apache Hadoop deployments. + +The Knox Gateway provides a single access point for all REST and HTTP interactions with Apache Hadoop +clusters. + +%global initd_dir %{_sysconfdir}/init.d + +%if %{?suse_version:1}0 +# Required for init scripts +Requires: insserv +%global initd_dir %{_sysconfdir}/rc.d + +%else +# Required for init scripts +Requires: /lib/lsb/init-functions + +%global initd_dir %{_sysconfdir}/init.d + +%endif + +# disable repacking jars +%define __os_install_post %{nil} +%define __jar_repack %{nil} + +%clean +%__rm -rf $RPM_BUILD_ROOT + +%prep +%setup -n %{knox_name}-%{version} +#BIGTOP_PATCH_COMMANDS + +%build +bash %{SOURCE1} + +%install +# Init.d scripts +%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/ + +bash -x %{SOURCE2} \ + --prefix=$RPM_BUILD_ROOT \ + --etc-knox=%{etc_knox} \ + --lib-dir=%{usr_lib_knox} \ + --bin-dir=%{bin_dir} \ + --build-dir=`pwd` + +for service in %{knox_services} +do + # Install init script + initd_script=$RPM_BUILD_ROOT/%{initd_dir}/%{knox_name}-${service} + bash %{SOURCE6} $RPM_SOURCE_DIR/%{knox_name}-${service}.svc rpm $initd_script +done + + +%pre +for service in %{knox_services}; do + /sbin/service %{knox_name}-${service} status > /dev/null 2>&1 + if [ $? -eq 0 ]; then + /sbin/service %{knox_name}-${service} stop > /dev/null 2>&1 + fi + chkconfig --del %{knox_name}-${service} +done + +getent group knox >/dev/null || groupadd -r knox +getent passwd knox >/dev/null || useradd -c "Knox" -s /sbin/nologin -g knox -r -d %{usr_lib_knox} knox 2> /dev/null || : + +%post +for service in %{knox_services}; do + chkconfig --add %{knox_name}-${service} +done +%{alternatives_cmd} --install %{np_etc_knox}/conf %{knox_name}-conf %{etc_knox}/conf.dist 30 + +%preun +if [ "$1" = 0 ]; then + %{alternatives_cmd} --remove %{knox_name}-conf %{etc_knox}/conf.dist || : +fi + +%postun +for service in %{knox_services}; do + if [ $1 -ge 1 ]; then + service %{knox_name}-${service} condrestart >/dev/null 2>&1 + fi +done + +%files +%defattr(-,root,root) +%attr(0755,knox,knox) %config(noreplace) %{np_etc_knox} +%config(noreplace) %{etc_knox}/conf.dist +%attr(0755,knox,knox) %config(noreplace) %{initd_dir}/%{knox_name}-gateway +%dir %{_sysconfdir}/%{knox_name} + +%attr(0755,knox,knox) %{np_var_log_knox} +%attr(0755,knox,knox) %{np_var_lib_knox_data} +%attr(0755,knox,knox) %{np_var_run_knox} + +%{usr_lib_knox}/data +%{usr_lib_knox}/conf +%{usr_lib_knox}/logs +%{usr_lib_knox}/pids +%{usr_lib_knox}/bin +%{usr_lib_knox}/dep +%{usr_lib_knox}/lib +%{usr_lib_knox}/samples +%{usr_lib_knox}/templates + +%{bin_dir}/gateway \ No newline at end of file diff --git a/bigtop-tests/smoke-tests/knox/TestKnox.groovy b/bigtop-tests/smoke-tests/knox/TestKnox.groovy new file mode 100644 index 0000000000..38bfabcbe1 --- /dev/null +++ b/bigtop-tests/smoke-tests/knox/TestKnox.groovy @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.junit.BeforeClass +import org.junit.AfterClass + +import org.apache.bigtop.itest.shell.Shell +import static org.junit.Assert.assertTrue +import org.junit.Test +import org.apache.commons.logging.LogFactory +import org.apache.commons.logging.Log + +import static org.apache.bigtop.itest.LogErrorsUtils.logError + +class TestKnox { + static private Log LOG = LogFactory.getLog(Object.class) + + static Shell sh = new Shell("/bin/bash -s") + + @BeforeClass + static void setUp() { + // noop for now. + } + + @AfterClass + public static void tearDown() { + // noop for now + } + + @Test + void testKnox() { + // Basic test to verify that Knox cli can list its topologies. + + + sh.exec("/usr/lib/knox/bin/knoxcli.sh list-topologies"); + String ret = sh.getOut() + assertTrue(ret.contains("sandbox")); + LOG.info('Test Knox cli finished'); + LOG.info('Found topology '+ret); + + + LOG.info('Running systemctl status knox-gateway'); + sh.exec("systemctl status knox-gateway | grep \"active (running)\""); + logError(sh); + assertTrue("Failed ...", sh.getRet() == 0); + } +} diff --git a/bigtop-tests/smoke-tests/knox/build.gradle b/bigtop-tests/smoke-tests/knox/build.gradle new file mode 100644 index 0000000000..4a8446ac42 --- /dev/null +++ b/bigtop-tests/smoke-tests/knox/build.gradle @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +def tests_to_include() { + return [ + "TestKnox.groovy" + ]; +} + +dependencies { + compile 'com.google.code.gson:gson:2.3' +} + +sourceSets { + test { + groovy { + srcDirs = ["${BIGTOP_HOME}/bigtop-tests/smoke-tests/knox/"] + } + } +} \ No newline at end of file diff --git a/bigtop.bom b/bigtop.bom index 3cd1025b65..1fdd0ce031 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -118,7 +118,7 @@ bigtop { */ dependencies = [ 'bigtop-utils':['bigtop-groovy', 'bigtop-jsvc', - 'flink', 'zookeeper'], + 'flink', 'zookeeper', 'knox'], 'bigtop-groovy':['hadoop'], 'bigtop-jsvc':['hadoop'], zookeeper:['hadoop', 'hbase', 'kafka'], @@ -353,5 +353,16 @@ bigtop { url { site = "https://github.com/apache/ranger/archive/refs/tags" archive = site } } + 'knox' { + name = 'knox' + rpm_pkg_suffix = "_" + bigtop.base_version.replace(".", "_") + relNotes = 'Apache Knox' + version { base = '2.0.0'; pkg = base-"-release"; release = 1 } + tarball { destination = "$name-${version.base}.zip" + source = "$name-${version.base}-src.zip" } + url { download_path = "$name/${version.base}" + site = "${apache.APACHE_MIRROR}/${download_path}" + archive = "${apache.APACHE_ARCHIVE}/${download_path}" } + } } }