Merge pull request #1348 from GlobalNOC/2.0.14-dev

2.0.14

.env.example

@@ -1,6 +1,92 @@
-MYSQL_PASSWORD=
+OESS_VERSION=2.0.13
+
+
+# Default root password set for installed MySQL Server.
+#
+MYSQL_ROOT_PASSWORD=test
+#
+# Default password set for installed RabbitMQ Server.
+#
+RABBITMQ_DEFAULT_PASS=guest
+
+
+# Authentication details to MySQL Server.
+#
+MYSQL_USER=root
+MYSQL_PASS=test
+MYSQL_HOST=localhost
+MYSQL_PORT=3306
+MYSQL_DATABASE=oess
+
+
+# Authentication details for RabbitMQ Server.
+#
+RABBITMQ_USER=guest
+RABBITMQ_PASS=guest
+RABBITMQ_HOST=rabbitmq
+RABBITMQ_PORT=5672
+RABBITMQ_VHOST=/
+
+
+# Email notifications
+#
+SMTP_FROM_ADDR=oess@localhost
+SMTP_FROM_NAME=OESS Notifier
+SMTP_IMAGE_URL=http://localhost:8080/oess/notification-img
+
+
+# Optional: WebAPI details for GlobalNOC TSDS instance; Is used to
+# store time series data like network statistics for connections.
+#
+# TSDS_USERNAME=
+# TSDS_PASSWORD=
+# TSDS_URL=
+# TSDS_REALM=
+
+
+# OESS password for `admin` user
+#
 OESS_PASSWORD=
+#
+# Base OESS URL. Required to properly set frontend URLs
+#
+OESS_BASE_URL=http://localhost:8080/oess
+#
+# Used as ASN for OESS side of all VRF peerings.
+#
+OESS_LOCAL_ASN=55038
+#
+# Southbound interface to network devices. Options:
+#   openflow:     deprecated
+#   vpn-mpls:     netconf provisioned l2vpn, l2ccc, l3vpn (Juniper MX)
+#   evpn-vxlan:   netconf provisioned vxlan (Juniper QFX)
+#   nso:          nso based southbound
+#   nso+vpn-mpls: runs both vpn-mpls and nso southbounds simultaneously
+#
 OESS_NETWORK_TYPE=vpn-mpls
-NSO_HOST=
-NSO_PASSWORD=
-NSO_USERNAME=
+
+
+# Authentication details for NETCONF connection to network devices. Is
+# required for vpn-mpls, evpn-vxlan, and nso+vpn-mpls
+# OESS_NETWORK_TYPEs.
+#
+NETCONF_USERNAME=
+NETCONF_PASSWORD=
+#
+# Interval on which OESS evaluates and resolves difference between
+# expected and actual network configuration.
+#
+NETCONF_DIFF_INTERVAL=60
+
+
+# Authentication details for NSO REST API. Is required for nso
+# OESS_NETWORK_TYPEs.
+#
+# NSO_USERNAME=
+# NSO_PASSWORD=
+# NSO_HOST=
+#
+# Interval on which OESS evaluates and resolves difference between
+# expected and actual network configuration.
+#
+# NSO_DIFF_INTERVAL=60

.gitignore

@@ -40,6 +40,9 @@ perl-lib/OESS/MYMETA.json
 perl-lib/OESS/MYMETA.yml
 perl-lib/OESS/lib/OESS/Collector
 perl-lib/OESS/lib/OESS/Collector.pm
+perl-lib/OESS/share/scripts/oess-migrate/connection_id_list.csv
+perl-lib/OESS/share/scripts/oess-migrate/shim_interface_mapping.csv
+perl-lib/OESS/share/scripts/oess-migrate/config.json
 etc/packer-config/output-virtualbox-iso/
 etc/packer-config/packer_cache/
 perl-lib/OESS/blib/
@@ -52,3 +55,4 @@ perl-lib/OESS/lib/OESS/t/conf/database.xml
 frontend/.htpasswd
 *.DS_Store
 .env
+.vscode

Dockerfile

@@ -6,5 +6,8 @@ RUN yum -y install epel-release
 RUN yum -y install perl mariadb-server
 RUN yum -y install perl-Carp-Always perl-Test-Deep perl-Test-Exception perl-Test-Pod perl-Test-Pod-Coverage perl-Devel-Cover perl-AnyEvent-HTTP
 RUN yum -y install perl-OESS oess-core oess-frontend
+
+COPY perl-lib/OESS /OESS
+
 RUN chmod 777 /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

Dockerfile.dev

@@ -7,7 +7,7 @@ RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-serve
 RUN yum makecache
 RUN yum -y install epel-release
 RUN yum -y install perl httpd mariadb-server rabbitmq-server
-RUN yum -y install perl-Carp-Always perl-Test-Deep perl-Test-Exception perl-Test-Pod perl-Test-Pod-Coverage perl-Devel-Cover perl-Net-AMQP-RabbitMQ perl-LWP-Protocol-https perl-AnyEvent-HTTP
+RUN yum -y install perl-Carp-Always perl-Test-Deep perl-Test-Exception perl-Test-Pod perl-Test-Pod-Coverage perl-Devel-Cover perl-Net-AMQP-RabbitMQ perl-LWP-Protocol-https perl-AnyEvent-HTTP perl-GRNOC-CLI perl-Text-CSV
 RUN yum -y install perl-OESS oess-core oess-frontend yui2
 
 COPY app/mpls/mpls_discovery.pl /usr/bin/mpls_discovery.pl

Makefile

@@ -1,4 +1,4 @@
-OESS_VERSION=2.0.13
+OESS_VERSION=2.0.14
 OESS_NETWORK=oess
 
 container:
@@ -9,8 +9,14 @@ container:
 #
 # Allow container to attach to host port via host.docker.internal hostname:
 # --add-host=host.docker.internal:host-gateway
+#
+# To support packet captures:
+# --cap-add=NET_RAW
+# --cap-add=NET_ADMIN
+#
 dev:
 	docker run -it \
+	--rm \
 	--env-file .env \
 	--publish 8000:80 \
 	--publish 5672:5672 \
@@ -22,3 +28,7 @@ dev:
 
 documentation:
 	perl docs/generate-webservice-docs.pl
+
+test:
+	docker build . -f Dockerfile -t oess-test
+	docker run --rm -it -e OESS_TEST_FILES="$(TEST_FILES)" --volume ${PWD}/perl-lib/OESS:/OESS oess-test

app/Makefile

@@ -1,5 +1,5 @@
 NAME= oess-core
-VERSION = 2.0.13
+VERSION = 2.0.14
 
 rpm:	dist
 	rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz

app/oess-core.spec

@@ -1,5 +1,5 @@
 Name:		oess-core
-Version:	2.0.13
+Version:	2.0.14
 Release:	1%{?dist}
 Summary:	The core OESS service providers
 
@@ -20,7 +20,7 @@ Requires: /bin/bash
 Requires: /usr/bin/perl
 Requires: perl(base), perl(constant), perl(strict), perl(warnings)
 
-Requires: perl-OESS >= 2.0.13
+Requires: perl-OESS >= 2.0.14
 
 Requires: perl(AnyEvent), perl(AnyEvent::DBus), perl(AnyEvent::RabbitMQ)
 Requires: perl(CPAN), perl(CPAN::Shell)

frontend/Makefile

@@ -1,5 +1,5 @@
 NAME= oess-frontend
-VERSION = 2.0.13
+VERSION = 2.0.14
 
 rpm:	dist
 	rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz

frontend/oess-frontend.spec

@@ -1,5 +1,5 @@
 Name:		oess-frontend
-Version:	2.0.13
+Version:	2.0.14
 Release:	1%{?dist}
 Summary:	The OESS webservices and user interface
 
@@ -13,14 +13,14 @@ BuildRequires: perl
 BuildRequires: python >= 2.6, python-libs >= 2.6
 BuildRequires: python-simplejson
 
-Requires: oess-core >= 2.0.13
+Requires: oess-core >= 2.0.14
 Requires: yui
 Requires: httpd, mod_ssl
 Requires: nddi-tiles
 Requires: perl-Crypt-SSLeay
 Requires: xmlsec1, xmlsec1-openssl
 
-Requires: perl-OESS >= 2.0.13
+Requires: perl-OESS >= 2.0.14
 
 Requires: perl(strict), perl(warnings)
 Requires: perl(AnyEvent)

frontend/webservice/circuit.cgi

@@ -274,11 +274,6 @@ sub provision {
         return;
     }
 
-    if (@{$args->{endpoint}->{value}} > 2) {
-        $method->set_error("Support for Multi-Point Layer 2 Connections is currently disabled. Please contact your OESS administrator for more information.");
-        return;
-    }
-
     # Endpoint: { entity: 'entity name', bandwidth: 0, tag: 100, inner_tag: 100, peerings: [{ version: 4 }]  }
     foreach my $value (@{$args->{endpoint}->{value}}) {
         my $ep;
@@ -509,11 +504,6 @@ sub update {
     my $add_endpoints = [];
     my $del_endpoints = [];
 
-    if (@{$args->{endpoint}->{value}} > 2) {
-        $method->set_error("Support for Multi-Point Layer 2 Connections is currently disabled. Please contact your OESS administrator for more information.");
-        return;
-    }
-
     foreach my $value (@{$args->{endpoint}->{value}}) {
         my $ep;
         eval{

frontend/webservice/monitoring.cgi

@@ -36,6 +36,7 @@ use Log::Log4perl;
 
 use OESS::Database;
 use OESS::RabbitMQ::Client;
+use OESS::RabbitMQ::Topic qw(fwdctl_switch_topic_for_node);
 use GRNOC::WebService;
 
 Log::Log4perl::init('/etc/oess/logging.conf');
@@ -201,7 +202,7 @@ sub get_mpls_node_status{
 	return $tmp;
     }
 
-    $mq->{'topic'} = 'MPLS.FWDCTL.Switch.' . $node->{'mgmt_addr'};
+    $mq->{'topic'} = fwdctl_switch_topic_for_node(mgmt_addr => $node->{'mgmt_addr'}, tcp_port => $node->{'tcp_port'});
     my $result = $mq->is_connected();
     warn Dumper($result);
     $result = int($result->{'results'}->{'connected'});

frontend/www/js_templates/modify_cloud.js

@@ -34,7 +34,12 @@ class GlobalState extends Component {
 
       let iframe3 = document.getElementById(`endpoints-statistics-iframe-route`);
       iframe3.dataset.vrf = this.connection.vrf_id;
-      iframe3.src = `${iframe3.dataset.url}&var-table=OESS-L3VPN-${this.connection.vrf_id}.inet.0&from=now-1h&to=now`;
+
+      if (this.connection.endpoints.length == 0 || this.connection.endpoints[0].controller === "nso") {
+        iframe3.src = `${iframe3.dataset.url}&var-table=OESS-VRF-${this.connection.vrf_id}&from=now-1h&to=now`;
+      } else {
+        iframe3.src = `${iframe3.dataset.url}&var-table=OESS-L3VPN-${this.connection.vrf_id}.inet.0&from=now-1h&to=now`;
+      }
 
       this.connection.endpoints.forEach(function(endpoint, eIndex) {
 
@@ -54,7 +59,7 @@ class GlobalState extends Component {
 
         let peerSelections = document.getElementById('peering-selection');
         peerSelections.appendChild(select);
-
+        
         let statGraph = `
 <div id="endpoints-statistics-${eIndex}" class="panel panel-default endpoints-statistics" style="display: none;">
   <div class="panel-heading" style="height: 40px;">
@@ -65,7 +70,7 @@ class GlobalState extends Component {
 
   <div style="padding-left: 15px; padding-right: 15px">
     <iframe id="endpoints-statistics-iframe-${eIndex}" data-url="[% g_port %]" data-node="${endpoint.node}" data-interface="${endpoint.interface}" data-unit="${endpoint.unit}" width="100%" height="300" frameborder="0"></iframe>
-    <iframe id="endpoints-statistics-iframe-peer-${eIndex}" data-url="[% g_peer %]" data-node="${endpoint.node}" data-vrf="${this.connection.vrf_id}" width="100%" height="300" frameborder="0"></iframe>
+    <iframe id="endpoints-statistics-iframe-peer-${eIndex}" data-url="[% g_peer %]" data-controller="${endpoint.controller}" data-node="${endpoint.node}" data-vrf="${this.connection.vrf_id}" width="100%" height="300" frameborder="0"></iframe>
   </div>
 </div>`;
 
@@ -126,7 +131,6 @@ class GlobalState extends Component {
 
   // Named saveCircuit to work with Object built from Layer2 Conns
   async saveCircuit() {
-    console.log('Connection:', this.connection);
 
     let addNetworkLoadingModal = $('#add-connection-loading');
     addNetworkLoadingModal.modal('show');
@@ -266,8 +270,9 @@ function updateStatisticsIFrame() {
     iframe.src = `${iframe.dataset.url}&var-node=${iframe.dataset.node}&var-interface=${iframe.dataset.interface}.${iframe.dataset.unit}` + range.value;
 
     let iframe2 = document.getElementById(`endpoints-statistics-iframe-peer-${container.value}`);
-    iframe2.src = `${iframe2.dataset.url}&var-node=${iframe2.dataset.node}&var-vrf=OESS-L3VPN-${iframe2.dataset.vrf}&var-peer=${peer.value}` + range.value;
-
-    let iframe3 = document.getElementById(`endpoints-statistics-iframe-route`);
-    iframe3.src = `${iframe3.dataset.url}&var-table=OESS-L3VPN-${iframe3.dataset.vrf}.inet.0` + range.value;
+    if (iframe2.dataset.controller === "nso") {
+      iframe2.src = `${iframe2.dataset.url}&var-node=${iframe2.dataset.node}&var-vrf=OESS-VRF-${iframe2.dataset.vrf}&var-peer=${peer.value.split('/')[0]}` + range.value;
+    } else {
+      iframe2.src = `${iframe2.dataset.url}&var-node=${iframe2.dataset.node}&var-vrf=OESS-L3VPN-${iframe2.dataset.vrf}&var-peer=${peer.value}` + range.value;
+    }
 }

oess.spec

@@ -1,14 +1,14 @@
 Summary: OESS Metapackage 
 Name: oess
-Version: 2.0.13
+Version: 2.0.14
 Release: 1%{?dist}
 License: Apache
 Group: GRNOC
 #Source: 
 URL: http://globalnoc.iu.edu
 Buildroot: %{_tmppath}/%{name}-root
-Requires: oess-core >= 2.0.13
-Requires: oess-frontend >= 2.0.13
+Requires: oess-core >= 2.0.14
+Requires: oess-frontend >= 2.0.14
 
 %description
 Package that installs all of the OESS packages

perl-lib/OESS/MANIFEST

@@ -83,7 +83,7 @@ lib/OESS/Webservice.pm
 lib/OESS/Workgroup.pm
 Makefile.PL
 MANIFEST			This list of files
-perl-OESS-2.0.13.tar.gz
+perl-OESS-2.0.14.tar.gz
 perl-OESS.spec
 share/customer-templates/cisco/2900/ios124/template.txt
 share/customer-templates/cisco/3700/ios124/template.txt
@@ -147,6 +147,7 @@ share/upgrade/oess-2.0.1-2.0.2
 share/upgrade/oess-2.0.10-2.0.11
 share/upgrade/oess-2.0.11-2.0.12
 share/upgrade/oess-2.0.12-2.0.13
+share/upgrade/oess-2.0.13-2.0.14
 share/upgrade/oess-2.0.2-2.0.3
 share/upgrade/oess-2.0.3-2.0.4
 share/upgrade/oess-2.0.4-2.0.5

perl-lib/OESS/entrypoint.dev.sh

@@ -37,10 +37,11 @@ sed -i "s/NSO_HOST/$NSO_HOST/" /etc/oess/database.xml
 sed -i "s/NSO_PASSWORD/$NSO_PASSWORD/" /etc/oess/database.xml
 sed -i "s/NSO_USERNAME/$NSO_USERNAME/" /etc/oess/database.xml
 sed -i "s/OESS_LOCAL_ASN/$OESS_LOCAL_ASN/" /etc/oess/database.xml
-sed -i "s/TSDS_URL/$TSDS_URL/" /etc/oess/database.xml
+sed -i "s@TSDS_URL@$TSDS_URL@" /etc/oess/database.xml
 sed -i "s/TSDS_PASSWORD/$TSDS_PASSWORD/" /etc/oess/database.xml
 sed -i "s/TSDS_USERNAME/$TSDS_USERNAME/" /etc/oess/database.xml
-sed -i "s/TSDS_REALM/$TSDS_REALM/" /etc/oess/database.xml
+sed -i "s@TSDS_REALM@$TSDS_REALM@" /etc/oess/database.xml
+sed -i "s@GRAFANA_URL@$GRAFANA_URL@" /etc/oess/database.xml
 
 sed -i "s/root/$NETCONF_USERNAME/" /etc/oess/.passwd.xml
 sed -i "s/test/$NETCONF_PASSWORD/" /etc/oess/.passwd.xml

perl-lib/OESS/entrypoint.sh

@@ -3,8 +3,8 @@
 pwd
 ls -la
 
-cp perl-lib/OESS/t/conf/database.xml /etc/oess/database.xml
-cp perl-lib/OESS/t/conf/passwd.xml /etc/oess/.passwd.xml
+cp /OESS/t/conf/database.xml /etc/oess/database.xml
+cp /OESS/t/conf/passwd.xml /etc/oess/.passwd.xml
 
 rm -rf /usr/share/perl5/vendor_perl/OESS
 
@@ -14,8 +14,16 @@ sleep 3
 
 /usr/bin/mysqladmin -u root password test
 
+cd /OESS
 
-cd perl-lib/OESS
-perl Makefile.PL
-make
-cover -test
+if [ -z "$OESS_TEST_FILES" ]; then
+    echo "Running all tests:"
+    perl Makefile.PL
+    make
+    cover -test
+else
+    echo "Running select tests: $OESS_TEST_FILES"
+    perl Makefile.PL
+    make
+    make test TEST_FILES="$OESS_TEST_FILES"
+fi