From 5a3834c4e7449efb9eb59bf78f9c05392e2da908 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Wed, 21 Oct 2020 16:32:31 +0000 Subject: [PATCH 001/156] add initial nso discovery module --- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 160 ++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 perl-lib/OESS/lib/OESS/NSO/Discovery.pm diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm new file mode 100644 index 000000000..390bf93e3 --- /dev/null +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -0,0 +1,160 @@ +use strict; +use warnings; + +package OESS::NSO::Discovery; + +use AnyEvent; +use Data::Dumper; +use GRNOC::RabbitMQ::Method; +use JSON; +use Log::Log4per; + +use OESS::Config; +use OESS::DB; +use OESS::RabbitMQ::Dispatcher; + +=head1 OESS::NSO::Discovery + +=cut + +=head2 new + +=cut +sub new { + my $class = shift; + my $args = { + config => '/etc/oess/database.xml', + logger => Log::Log4perl->get_logger('OESS.NSO.Discovery'), + @_ + }; + my $self = bless $args, $class; + + $self->{config_filename} = $self->{config}; + $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + $self->{db} = new OESS::DB(config => $self->{config_filename}); + + # When this process receives sigterm send an event to notify all + # children to exit cleanly. + $SIG{TERM} = sub { + $self->stop; + }; + + return $self; +} + +=head2 connection_handler + +=cut +sub connection_handler { + my $self = shift; + + return 1; +} + +=head2 device_handler + +=cut +sub device_handler { + my $self = shift; + + return 1; +} + +=head2 interface_handler + +=cut +sub interface_handler { + my $self = shift; + + return 1; +} + +=head2 link_handler + +=cut +sub link_handler { + my $self = shift; + + return 1; +} + +=head2 new_switch + +=cut +sub new_switch { + my $self = shift; + + return 1; +} + +=head2 start + +=cut +sub start { + my $self = shift; + + $self->{connection_timer} = AnyEvent->timer( + after => 20, + interval => 60, + cb => sub { $self->connection_handler(@_); } + ); + $self->{device_timer} = AnyEvent->timer( + after => 10, + interval => 60, + cb => sub { $self->device_handler(@_); } + ); + $self->{interface_timer} = AnyEvent->timer( + after => 60, + interval => 120, + cb => sub { $self->interface_handler(@_); } + ); + $self->{link_timer} = AnyEvent->timer( + after => 80, + interval => 120, + cb => sub { $self->link_handler(@_); } + ); + + $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( + queue => 'oess-discovery', + topic => 'oess.discovery.rpc' + ); + + my $new_switch = new GRNOC::RabbitMQ::Method( + name => 'new_switch', + description => 'Add a new switch to the database', + async => 1, + callback => sub { $self->new_switch(@_); } + ); + $new_switch->add_input_parameter( + name => 'node_id', + description => 'Id of the new node', + required => 1, + pattern => $GRNOC::WebService::Regex::NUMBER_ID + ); + $self->{dispatcher}->register_method($new_switch); + + my $is_online = new GRNOC::RabbitMQ::Method( + name => "is_online", + description => 'Return if this service is online', + async => 1, + callback => sub { + my $method = shift; + return $method->{success_callback}({ successful => 1 }); + } + ); + $self->{dispatcher}->register_method($is_online); + + $self->{dispatcher}->start_consuming; + return 1; +} + +=head2 stop + +=cut +sub stop { + my $self = shift; + $self->{logger}->info('Stopping OESS::NSO::Discovery.'); + $self->{dispatcher}->stop_consuming; +} + +1; From 766b8bc0dd05d78e89b13ea50ce599fec64ef8f2 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 22 Oct 2020 16:20:54 -0400 Subject: [PATCH 002/156] remove ssl from apache example config --- frontend/conf/oe-ss.conf.example | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/frontend/conf/oe-ss.conf.example b/frontend/conf/oe-ss.conf.example index 00118ed92..3fb242976 100644 --- a/frontend/conf/oe-ss.conf.example +++ b/frontend/conf/oe-ss.conf.example @@ -1,23 +1,22 @@ -Alias /oess/tiles /usr/share/nddi-tiles/ -Alias /oess/services /usr/share/oess-frontend/webservice -Alias /oess/yui/build /usr/share/yui2 -Alias /oess/notification-img/ /usr/share/oess-frontend/www/media/notification -Alias /oess /usr/share/oess-frontend/www -Alias /idc /usr/share/oess-frontend/webservice/idc +Alias /oess/tiles /usr/share/nddi-tiles/ +Alias /oess/services /usr/share/oess-frontend/webservice +Alias /oess/yui/build /usr/share/yui2 +Alias /oess/notification-img /usr/share/oess-frontend/www/media/notification +Alias /oess /usr/share/oess-frontend/www +Alias /idc /usr/share/oess-frontend/webservice/idc + - SSLRequireSSL - AddHandler cgi-script .cgi - DirectoryIndex index.cgi - Options ExecCGI - Order allow,deny - Allow from all + AddHandler cgi-script .cgi + DirectoryIndex index.cgi + Options ExecCGI + Order allow,deny + Allow from all Satisfy any - SSLRequireSSL AddHandler cgi-script .cgi DirectoryIndex index.cgi Options ExecCGI @@ -30,15 +29,14 @@ Alias /idc /usr/share/oess-frontend/webservice/idc Require valid-user + - SSLRequireSSL Satisfy Any Allow from all - SSLRequireSSL AddHandler cgi-script .cgi Options ExecCGI Order allow,deny @@ -50,6 +48,4 @@ Alias /idc /usr/share/oess-frontend/webservice/idc Require valid-user - - -Redirect 301 /oess/admin/admin_index.cgi /oess/admin \ No newline at end of file +Redirect 301 /oess/admin/admin_index.cgi /oess/admin From e6cd07286e09b58d649935305b6b1ec558c5e149 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 22 Oct 2020 16:25:11 -0400 Subject: [PATCH 003/156] ignore mac dirs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 513d6ab88..2b3cb16b9 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ perl-lib/OESS/lib/OESS/pod2htmd.tmp perl-lib/OESS/lib/OESS/pod2htmi.tmp perl-lib/OESS/lib/OESS/t/conf/database.xml frontend/.htpasswd +*.DS_Store From 2bb382a79e6d46e0cd893f481ef3491e3e67fa82 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 22 Oct 2020 16:59:13 -0400 Subject: [PATCH 004/156] add base user and workgroup for new installs --- perl-lib/OESS/share/nddi.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/share/nddi.sql b/perl-lib/OESS/share/nddi.sql index caa94476f..d4c247ff0 100755 --- a/perl-lib/OESS/share/nddi.sql +++ b/perl-lib/OESS/share/nddi.sql @@ -768,7 +768,7 @@ CREATE TABLE `remote_auth` ( CONSTRAINT `user_auth_values_fk` FOREIGN KEY (`user_id`) REFERENCES `user` (`user_id`) ON DELETE NO ACTION ON UPDATE NO ACTION ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; - +INSERT INTO `remote_auth` (`auth_name`,`user_id`) VALUES ('admin',1); -- -- Dumping data for table `remote_auth` -- @@ -881,6 +881,7 @@ CREATE TABLE `user` ( KEY `user_idx` (`email`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `user` (`user_id`,`email`,`given_names`,`family_name`,`is_admin`,`status`) VALUES (1,'admin@localhost','admin','admin',1,'active'); -- -- Table structure for table `user_entity_membership` @@ -896,6 +897,7 @@ CREATE TABLE `user_entity_membership` ( KEY `user` (`user_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `user_entity_membership` (`user_id`,`entity_id`) VALUES (1,1); -- -- Table structure for table `user_workgroup_membership` @@ -914,6 +916,7 @@ CREATE TABLE `user_workgroup_membership` ( CONSTRAINT `workgroups_user_workgroup_membership_fk` FOREIGN KEY (`workgroup_id`) REFERENCES `workgroup` (`workgroup_id`) ON DELETE NO ACTION ON UPDATE NO ACTION ) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `user_workgroup_membership` (`workgroup_id`,`user_id`,`role`) VALUES (1,1,'admin'); -- -- Table structure for table `vrf` @@ -1043,6 +1046,7 @@ CREATE TABLE `workgroup` ( UNIQUE KEY `workgroups_idx` (`name`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `workgroup` (`workgroup_id`,`description`,`name`,`type`) VALUES (1,'admin','admin','admin'); -- -- Dumping data for table `workgroup` From 2d93246763075705a4f13f2415f2d64ebe52ef81 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 22 Oct 2020 17:05:19 -0400 Subject: [PATCH 005/156] add basic docker setup for local dev --- .env.example | 2 ++ .gitignore | 1 + Dockerfile.dev | 22 ++++++++++++++++++++++ Makefile | 12 ++++++++++++ perl-lib/OESS/entrypoint.dev.sh | 28 ++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+) create mode 100644 .env.example create mode 100644 Dockerfile.dev create mode 100644 Makefile create mode 100644 perl-lib/OESS/entrypoint.dev.sh diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..334e744ae --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +MYSQL_PASSWORD= +OESS_PASSWORD= diff --git a/.gitignore b/.gitignore index 2b3cb16b9..3ecfcc91d 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ perl-lib/OESS/lib/OESS/pod2htmi.tmp perl-lib/OESS/lib/OESS/t/conf/database.xml frontend/.htpasswd *.DS_Store +.env diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..c50cb0785 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,22 @@ +FROM centos:7 + +COPY globalnoc-public-el7.repo /etc/yum.repos.d/globalnoc-public-el7.repo +RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/erlang/script.rpm.sh | bash +RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.rpm.sh | bash + +RUN yum makecache +RUN yum -y install epel-release +RUN yum -y install perl httpd mariadb-server rabbitmq-server +RUN yum -y install perl-Carp-Always perl-Test-Deep perl-Test-Exception perl-Test-Pod perl-Test-Pod-Coverage perl-Devel-Cover +RUN yum -y install perl-OESS oess-core oess-frontend yui2 + +COPY app/mpls/mpls_discovery.pl /usr/bin/mpls_discovery.pl +COPY app/mpls/mpls_fwdctl.pl /usr/bin/mpls_fwdctl.pl + +COPY frontend/conf/oe-ss.conf.example /etc/httpd/conf.d/oe-ss.conf +COPY app/etc/firmware.xml /etc/oess/firmware.xml +COPY perl-lib/OESS/t/conf/database.xml /etc/oess/database.xml +COPY perl-lib/OESS/t/conf/passwd.xml /etc/oess/.passwd.xml + +COPY perl-lib/OESS/entrypoint.dev.sh /entrypoint.sh +RUN chmod 777 /entrypoint.sh diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..9d5e0045f --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +OESS_VERSION=2.0.11 + +container: + docker build -f Dockerfile.dev --tag oess:${OESS_VERSION} . +dev: + docker run -it \ + --env-file .env \ + --publish 8000:80 \ + --mount type=bind,src=${PWD}/perl-lib/OESS/lib/OESS,dst=/usr/share/perl5/vendor_perl/OESS \ + --mount type=bind,src=${PWD}/frontend,dst=/usr/share/oess-frontend \ + --mount type=bind,src=${PWD}/perl-lib/OESS/share,dst=/usr/share/doc/perl-OESS-2.0.10/share \ + oess:v1 /bin/bash diff --git a/perl-lib/OESS/entrypoint.dev.sh b/perl-lib/OESS/entrypoint.dev.sh new file mode 100644 index 000000000..058e0a704 --- /dev/null +++ b/perl-lib/OESS/entrypoint.dev.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Configure and start all oess services + +# Start httpd +htpasswd -b -c /usr/share/oess-frontend/www/.htpasswd admin ${OESS_PASSWORD} +/usr/sbin/httpd +sleep 1 + +# Start mysql +/usr/bin/mysql_install_db --user=mysql --ldata=/var/lib/mysql --force +/usr/bin/mysqld_safe --datadir='/var/lib/mysql' & +sleep 3 +/usr/bin/mysqladmin -u root password ${MYSQL_PASSWORD} +/usr/bin/mysql --user=root --password=${MYSQL_PASSWORD} < /usr/share/doc/perl-OESS-2.0.10/share/nddi.sql + +# Start RabbitMQ +rabbitmq-server start -detached +sleep 7 + +# Populate OESS config with mysql credentials +sed -i "s/oess_test/oess/" /etc/oess/database.xml +sed -i "s/test/$MYSQL_PASSWORD/" /etc/oess/database.xml + +# Start OESS +/usr/bin/oess-notify.pl & +/usr/bin/mpls_discovery.pl & +/usr/bin/mpls_fwdctl.pl & +/usr/bin/oess-nsi & From 02ef0561cc802af293f706d0212f28c4a8ac6992 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 22 Oct 2020 17:12:47 -0400 Subject: [PATCH 006/156] update docker run tag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9d5e0045f..c66709ce5 100644 --- a/Makefile +++ b/Makefile @@ -9,4 +9,4 @@ dev: --mount type=bind,src=${PWD}/perl-lib/OESS/lib/OESS,dst=/usr/share/perl5/vendor_perl/OESS \ --mount type=bind,src=${PWD}/frontend,dst=/usr/share/oess-frontend \ --mount type=bind,src=${PWD}/perl-lib/OESS/share,dst=/usr/share/doc/perl-OESS-2.0.10/share \ - oess:v1 /bin/bash + oess:${OESS_VERSION} /bin/bash From 11c3b0d1e9296178920a275da1e3dc8849390ce3 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 23 Oct 2020 11:36:23 -0400 Subject: [PATCH 007/156] add docker network config --- .env.example | 1 + Makefile | 2 ++ perl-lib/OESS/entrypoint.dev.sh | 1 + perl-lib/OESS/t/conf/database.xml | 2 +- 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 334e744ae..ba9d87154 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,3 @@ MYSQL_PASSWORD= OESS_PASSWORD= +OESS_NETWORK_TYPE=vpn-mpls diff --git a/Makefile b/Makefile index c66709ce5..d34adeeed 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ OESS_VERSION=2.0.11 +OESS_NETWORK=oess-network container: docker build -f Dockerfile.dev --tag oess:${OESS_VERSION} . @@ -6,6 +7,7 @@ dev: docker run -it \ --env-file .env \ --publish 8000:80 \ + --network ${OESS_NETWORK} \ --mount type=bind,src=${PWD}/perl-lib/OESS/lib/OESS,dst=/usr/share/perl5/vendor_perl/OESS \ --mount type=bind,src=${PWD}/frontend,dst=/usr/share/oess-frontend \ --mount type=bind,src=${PWD}/perl-lib/OESS/share,dst=/usr/share/doc/perl-OESS-2.0.10/share \ diff --git a/perl-lib/OESS/entrypoint.dev.sh b/perl-lib/OESS/entrypoint.dev.sh index 058e0a704..d777f8b52 100644 --- a/perl-lib/OESS/entrypoint.dev.sh +++ b/perl-lib/OESS/entrypoint.dev.sh @@ -20,6 +20,7 @@ sleep 7 # Populate OESS config with mysql credentials sed -i "s/oess_test/oess/" /etc/oess/database.xml sed -i "s/test/$MYSQL_PASSWORD/" /etc/oess/database.xml +sed -i "s/vpn\-mpls/$OESS_NETWORK_TYPE/" /etc/oess/database.xml # Start OESS /usr/bin/oess-notify.pl & diff --git a/perl-lib/OESS/t/conf/database.xml b/perl-lib/OESS/t/conf/database.xml index b6324fd21..23390abb6 100644 --- a/perl-lib/OESS/t/conf/database.xml +++ b/perl-lib/OESS/t/conf/database.xml @@ -1,4 +1,4 @@ - + From 757832fb434e4f70c3b88b2514991ea52f96ea09 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 23 Oct 2020 11:37:37 -0400 Subject: [PATCH 008/156] load nso discovery module when set in database.xml --- app/mpls/mpls_discovery.pl | 18 ++++++++++++-- perl-lib/OESS/lib/OESS/Config.pm | 10 +++++++- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 31 +++++++++++++++++-------- perl-lib/OESS/share/nddi.sql | 2 ++ 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/app/mpls/mpls_discovery.pl b/app/mpls/mpls_discovery.pl index c8b281b76..189525189 100755 --- a/app/mpls/mpls_discovery.pl +++ b/app/mpls/mpls_discovery.pl @@ -9,17 +9,31 @@ use Proc::Daemon; use Data::Dumper; +use OESS::Config; use OESS::Database; use OESS::MPLS::Discovery; +use OESS::NSO::Discovery; my $pid_file = "/var/run/oess/mpls_discovery.pid"; +my $cnf_file = "/etc/oess/database.xml"; sub core{ #basic init stuffs Log::Log4perl::init_and_watch('/etc/oess/logging.conf',10); - my $discovery = OESS::MPLS::Discovery->new(); - AnyEvent->condvar->recv; + my $config = new OESS::Config(config_filename => $cnf_file); + if ($config->network_type eq 'nso') { + my $discovery = OESS::NSO::Discovery->new(config => $config); + $discovery->start; + AnyEvent->condvar->recv; + } + elsif ($config->network_type eq 'vpn-mpls' || $config->network_type eq 'evpn-vxlan') { + my $discovery = OESS::MPLS::Discovery->new(config => $config); + AnyEvent->condvar->recv; + } + else { + die "Unexpected network type configured." + } } sub main{ diff --git a/perl-lib/OESS/lib/OESS/Config.pm b/perl-lib/OESS/lib/OESS/Config.pm index 83cc43407..9d91e742a 100644 --- a/perl-lib/OESS/lib/OESS/Config.pm +++ b/perl-lib/OESS/lib/OESS/Config.pm @@ -82,6 +82,14 @@ sub db_credentials { password => $password}; } +=head2 filename + +=cut +sub filename { + my $self = shift; + return $self->{config_filename}; +} + =head2 fwdctl_enabled =cut @@ -138,7 +146,7 @@ sub network_type { } my $type = $self->{'config'}->{'network_type'}; - my $valid_types = ['openflow', 'vpn-mpls', 'evpn-vxlan']; + my $valid_types = ['openflow', 'vpn-mpls', 'evpn-vxlan', 'nso']; foreach my $valid_type (@$valid_types) { if ($type eq $valid_type) { return $type; diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index 390bf93e3..98b93404f 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -7,7 +7,7 @@ use AnyEvent; use Data::Dumper; use GRNOC::RabbitMQ::Method; use JSON; -use Log::Log4per; +use Log::Log4perl; use OESS::Config; use OESS::DB; @@ -23,15 +23,17 @@ use OESS::RabbitMQ::Dispatcher; sub new { my $class = shift; my $args = { - config => '/etc/oess/database.xml', - logger => Log::Log4perl->get_logger('OESS.NSO.Discovery'), + config => undef, + config_filename => '/etc/oess/database.xml', + logger => Log::Log4perl->get_logger('OESS.NSO.Discovery'), @_ }; my $self = bless $args, $class; - $self->{config_filename} = $self->{config}; - $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); - $self->{db} = new OESS::DB(config => $self->{config_filename}); + if (!defined $self->{config}) { + $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + } + $self->{db} = new OESS::DB(config => $self->{config}->filename); # When this process receives sigterm send an event to notify all # children to exit cleanly. @@ -82,9 +84,16 @@ sub link_handler { =cut sub new_switch { - my $self = shift; + my $self = shift; + my $method = shift; + my $params = shift; - return 1; + my $success = $method->{'success_callback'}; + my $error = $method->{'error_callback'}; + + warn "new_switch: $params->{node_id}{value}"; + + return &$success({ status => 1 }); } =head2 start @@ -115,8 +124,10 @@ sub start { ); $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( - queue => 'oess-discovery', - topic => 'oess.discovery.rpc' + # queue => 'oess-discovery', + # topic => 'oess.discovery.rpc' + queue => 'MPLS-Discovery', + topic => 'MPLS.Discovery.RPC' ); my $new_switch = new GRNOC::RabbitMQ::Method( diff --git a/perl-lib/OESS/share/nddi.sql b/perl-lib/OESS/share/nddi.sql index d4c247ff0..719b10689 100755 --- a/perl-lib/OESS/share/nddi.sql +++ b/perl-lib/OESS/share/nddi.sql @@ -544,6 +544,8 @@ CREATE TABLE `network` ( UNIQUE KEY `network_idx` (`name`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; /*!40101 SET character_set_client = @saved_cs_client */; +INSERT INTO `network` (`network_id`,`name`,`longitude`,`latitude`,`is_local`) VALUES (1,'oess',0,0,1); + -- -- Table structure for table `node` -- From b1cbc4e6e8c1c5365258f1632816c52e2ca53f73 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 23 Oct 2020 14:26:54 -0400 Subject: [PATCH 009/156] checkpoint save --- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index 98b93404f..ee7a38c25 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -11,6 +11,7 @@ use Log::Log4perl; use OESS::Config; use OESS::DB; +use OESS::DB::Node; use OESS::RabbitMQ::Dispatcher; =head1 OESS::NSO::Discovery @@ -34,6 +35,7 @@ sub new { $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); } $self->{db} = new OESS::DB(config => $self->{config}->filename); + $self->{nodes} = {}; # When this process receives sigterm send an event to notify all # children to exit cleanly. @@ -55,19 +57,34 @@ sub connection_handler { =head2 device_handler +device_handler queries each devices for basic system info: +- loopback address +- firmware version + =cut sub device_handler { my $self = shift; + $self->{logger}->info("Calling device_handler."); + foreach my $key (%{$self->{nodes}}) { + + } return 1; } =head2 interface_handler +interface_handler queries each device for interface configuration and +operational state. + =cut sub interface_handler { my $self = shift; + $self->{logger}->info("Calling interface_handler."); + foreach my $key (%{$self->{nodes}}) { + + } return 1; } @@ -91,7 +108,13 @@ sub new_switch { my $success = $method->{'success_callback'}; my $error = $method->{'error_callback'}; - warn "new_switch: $params->{node_id}{value}"; + my $node = OESS::DB::Node::fetch(db => $self->{db}, node_id => $params->{node_id}{value}); + if (!defined $node) { + my $err = "Couldn't lookup node $params->{node_id}{value}. Discovery will not properly complete on this node."; + $self->{logger}->error($err); + &$error($err); + } + $self->{nodes}->{$params->{node_id}{value}} = $node; return &$success({ status => 1 }); } From de55aabecaac7552e4352303ba53e52c03b51095 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 23 Oct 2020 20:41:21 +0000 Subject: [PATCH 010/156] add ability to modify peer md5_key --- perl-lib/OESS/lib/OESS/DB/Peer.pm | 4 ++++ perl-lib/OESS/lib/OESS/Peer.pm | 6 +++++- perl-lib/OESS/t/z-DB/Peer.update.00.t | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/lib/OESS/DB/Peer.pm b/perl-lib/OESS/lib/OESS/DB/Peer.pm index 918030ff3..5416e6e0a 100644 --- a/perl-lib/OESS/lib/OESS/DB/Peer.pm +++ b/perl-lib/OESS/lib/OESS/DB/Peer.pm @@ -200,6 +200,10 @@ sub update { push @$params, 'bfd=?'; push @$values, $args->{peer}->{bfd}; } + if (defined $args->{peer}->{md5_key}) { + push @$params, 'md5_key=?'; + push @$values, $args->{peer}->{md5_key}; + } my $fields = join(', ', @$params); push @$values, $args->{peer}->{vrf_ep_peer_id}; diff --git a/perl-lib/OESS/lib/OESS/Peer.pm b/perl-lib/OESS/lib/OESS/Peer.pm index f6a65ccfc..a6a1e88b2 100644 --- a/perl-lib/OESS/lib/OESS/Peer.pm +++ b/perl-lib/OESS/lib/OESS/Peer.pm @@ -162,7 +162,11 @@ sub peer_asn{ =cut sub md5_key{ my $self = shift; - return $self->{'md5_key'}; + my $md5_key = shift; + if (defined $md5_key) { + $self->{md5_key} = $md5_key; + } + return $self->{md5_key}; } =head2 vrf_ep_id diff --git a/perl-lib/OESS/t/z-DB/Peer.update.00.t b/perl-lib/OESS/t/z-DB/Peer.update.00.t index 710725571..a0e913b31 100644 --- a/perl-lib/OESS/t/z-DB/Peer.update.00.t +++ b/perl-lib/OESS/t/z-DB/Peer.update.00.t @@ -55,6 +55,7 @@ warn $err if defined $err; $model->{vrf_ep_peer_id} = $id; $model->{local_ip} = '192.168.3.2/31'; $model->{peer_ip} = '192.168.3.3/31'; +$model->{md5_key} = 'a new key'; my $err3 = OESS::DB::Peer::update(db => $db, peer => $model); ok(!defined $err3, "no error on standard update"); From 1ab926de44bea98eb69ca6520abfeb2d359ac5b8 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 5 Nov 2020 15:28:12 -0500 Subject: [PATCH 011/156] add logic to fetch device state from nso --- .env.example | 3 + perl-lib/OESS/entrypoint.dev.sh | 3 + perl-lib/OESS/lib/OESS/Config.pm | 27 +++++++ perl-lib/OESS/lib/OESS/DB/Node.pm | 10 +++ perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 97 +++++++++++++++++++++++-- perl-lib/OESS/t/conf/database.xml | 1 + 6 files changed, 136 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index ba9d87154..9a4f2e13b 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,6 @@ MYSQL_PASSWORD= OESS_PASSWORD= OESS_NETWORK_TYPE=vpn-mpls +NSO_HOST= +NSO_PASSWORD= +NSO_USERNAME= diff --git a/perl-lib/OESS/entrypoint.dev.sh b/perl-lib/OESS/entrypoint.dev.sh index d777f8b52..4c61f8b73 100644 --- a/perl-lib/OESS/entrypoint.dev.sh +++ b/perl-lib/OESS/entrypoint.dev.sh @@ -21,6 +21,9 @@ sleep 7 sed -i "s/oess_test/oess/" /etc/oess/database.xml sed -i "s/test/$MYSQL_PASSWORD/" /etc/oess/database.xml sed -i "s/vpn\-mpls/$OESS_NETWORK_TYPE/" /etc/oess/database.xml +sed -i "s/NSO_HOST/$NSO_HOST/" /etc/oess/database.xml +sed -i "s/NSO_PASSWORD/$NSO_PASSWORD/" /etc/oess/database.xml +sed -i "s/NSO_USERNAME/$NSO_USERNAME/" /etc/oess/database.xml # Start OESS /usr/bin/oess-notify.pl & diff --git a/perl-lib/OESS/lib/OESS/Config.pm b/perl-lib/OESS/lib/OESS/Config.pm index 9d91e742a..d2d7738c7 100644 --- a/perl-lib/OESS/lib/OESS/Config.pm +++ b/perl-lib/OESS/lib/OESS/Config.pm @@ -244,4 +244,31 @@ sub third_party_mgmt { return $self->{'config'}->{'third_party_mgmt'}; } +=head2 nso_host + +=cut +sub nso_host { + my $self = shift; + return if (!defined $self->{config}->{nso}); + return $self->{config}->{nso}->{host}; +} + +=head2 nso_password + +=cut +sub nso_password { + my $self = shift; + return if (!defined $self->{config}->{nso}); + return $self->{config}->{nso}->{password}; +} + +=head2 nso_username + +=cut +sub nso_username { + my $self = shift; + return if (!defined $self->{config}->{nso}); + return $self->{config}->{nso}->{username}; +} + 1; diff --git a/perl-lib/OESS/lib/OESS/DB/Node.pm b/perl-lib/OESS/lib/OESS/DB/Node.pm index 20294a920..65c29daed 100644 --- a/perl-lib/OESS/lib/OESS/DB/Node.pm +++ b/perl-lib/OESS/lib/OESS/DB/Node.pm @@ -34,6 +34,16 @@ sub fetch{ return $node->[0]; } +=head2 fetch_all + +=cut +sub fetch_all { + my %params = @_; + my $db = $params{'db'}; + + return $db->execute_query("select * from node join node_instantiation on node.node_id=node_instantiation.node_id where node_instantiation.end_epoch=-1", []); +} + =head2 get_node_interfaces =cut diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index ee7a38c25..cd0a1ba97 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -6,9 +6,11 @@ package OESS::NSO::Discovery; use AnyEvent; use Data::Dumper; use GRNOC::RabbitMQ::Method; +use HTTP::Request::Common; use JSON; use Log::Log4perl; - +use LWP::UserAgent; +use XML::LibXML; use OESS::Config; use OESS::DB; use OESS::DB::Node; @@ -37,6 +39,11 @@ sub new { $self->{db} = new OESS::DB(config => $self->{config}->filename); $self->{nodes} = {}; + $self->{www} = new LWP::UserAgent; + my $host = $self->{config}->nso_host; + $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string + $self->{www}->credentials($host, "restconf", $self->{config}->nso_username, $self->{config}->nso_password); + # When this process receives sigterm send an event to notify all # children to exit cleanly. $SIG{TERM} = sub { @@ -64,10 +71,38 @@ device_handler queries each devices for basic system info: =cut sub device_handler { my $self = shift; - $self->{logger}->info("Calling device_handler."); - foreach my $key (%{$self->{nodes}}) { + foreach my $key (keys %{$self->{nodes}}) { + my $node = $self->{nodes}->{$key}; + + my $dom = eval { + my $res = $self->{www}->get($self->{config}->nso_host . "/restconf/data/tailf-ncs:devices/device=$node->{name}"); + return XML::LibXML->load_xml(string => $res->content); + }; + if ($@) { + warn 'tailf-ncs:devices:' . $@; + $self->{logger}->error('tailf-ncs:devices:' . $@); + next; + } + + my $device = eval { + return { + name => $dom->findvalue('/ncs:device/ncs:name'), + platform => $dom->findvalue('/ncs:device/ncs:platform/ncs:name'), + version => $dom->findvalue('/ncs:device/ncs:platform/ncs:version'), + model => $dom->findvalue('/ncs:device/ncs:platform/ncs:model'), + serial_number => $dom->findvalue('/ncs:device/ncs:platform/ncs:serial-number') + }; + }; + if ($@) { + warn 'device:' . $@; + $self->{logger}->error('device:' . $@); + next; + } + + # TODO save nso queried data into db + warn 'device: ' . Dumper($device); } return 1; } @@ -80,10 +115,45 @@ operational state. =cut sub interface_handler { my $self = shift; - $self->{logger}->info("Calling interface_handler."); - foreach my $key (%{$self->{nodes}}) { + foreach my $key (keys %{$self->{nodes}}) { + my $node = $self->{nodes}->{$key}; + + my $dom = eval { + my $res = $self->{www}->get($self->{config}->nso_host . "/restconf/data/tailf-ncs:devices/device=$node->{name}/config/tailf-ned-cisco-ios-xr:interface"); + return XML::LibXML->load_xml(string => $res->content); + }; + if ($@) { + warn 'tailf-ned-cisco-ios-xr:interface:' . $@; + $self->{logger}->error('tailf-ned-cisco-ios-xr:interface:' . $@); + next; + } + + my $ports = eval { + my $result = []; + + my @gb_ports = $dom->findnodes('/cisco-ios-xr:interface/cisco-ios-xr:GigabitEthernet'); + foreach my $port (@gb_ports) { + my $port_info = { + admin_state => $port->exists('./cisco-ios-xr:shutdown') ? 'down' : 'up', + bandwidth => $port->findvalue('./cisco-ios-xr:speed') || 1000, + description => $port->findvalue('./cisco-ios-xr:description') || '', + mtu => $port->findvalue('./cisco-ios-xr:mtu') || 0, + name => $port->findvalue('./cisco-ios-xr:id') + }; + push @$result, $port_info; + } + return $result; + }; + if ($@) { + warn 'port_info:' . $@; + $self->{logger}->error('port_info:' . $@); + next; + } + + # TODO save nso queried data into db + warn 'ports: ' . Dumper($ports); } return 1; } @@ -116,6 +186,12 @@ sub new_switch { } $self->{nodes}->{$params->{node_id}{value}} = $node; + warn "Switch $node->{name} registered with discovery."; + $self->{logger}->info("Switch $node->{name} registered with discovery."); + + $self->device_handler; + $self->interface_handler; + return &$success({ status => 1 }); } @@ -125,6 +201,17 @@ sub new_switch { sub start { my $self = shift; + # Load devices from database + my $nodes = OESS::DB::Node::fetch_all(db => $self->{db}); + if (!defined $nodes) { + warn "Couldn't lookup nodes. Discovery will not collect data on any existing nodes."; + $self->{logger}->error("Couldn't lookup nodes. Discovery will not collect data on any existing nodes."); + } + foreach my $node (@$nodes) { + $self->{nodes}->{$node->{node_id}} = $node; + } + + # Setup polling subroutines $self->{connection_timer} = AnyEvent->timer( after => 20, interval => 60, diff --git a/perl-lib/OESS/t/conf/database.xml b/perl-lib/OESS/t/conf/database.xml index 23390abb6..34990800c 100644 --- a/perl-lib/OESS/t/conf/database.xml +++ b/perl-lib/OESS/t/conf/database.xml @@ -1,4 +1,5 @@ + From 4274da759767bf1374583efc552e7251aa8dfcc7 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 5 Nov 2020 22:44:30 +0000 Subject: [PATCH 012/156] add update method for nodes --- perl-lib/OESS/lib/OESS/DB/Node.pm | 154 ++++++++++++++++++++++++++ perl-lib/OESS/t/z-DB/Node.update.00.t | 105 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 perl-lib/OESS/t/z-DB/Node.update.00.t diff --git a/perl-lib/OESS/lib/OESS/DB/Node.pm b/perl-lib/OESS/lib/OESS/DB/Node.pm index 20294a920..6004a7d70 100644 --- a/perl-lib/OESS/lib/OESS/DB/Node.pm +++ b/perl-lib/OESS/lib/OESS/DB/Node.pm @@ -147,4 +147,158 @@ sub _process_tag_string{ return \@tags; } +=head2 update + + my $err = OESS::DB::Node::update( + db => $db, + node => { + node_id => 1, + name => 'host.examle.com', # Optional + latitude => 1, # Optional + longitude => 1, # Optional + operational_state_mpls => 'up', # Optional + vlan_tag_range => '1-4095', # Optional + pending_diff => 1 # Optional + short_name => 'host', # Optional + + admin_state => 'active', # Optional + vendor => 'juniper', # Optional + model => 'mx', # Optional + sw_version => '13.3R3', # Optional + mgmt_addr => '192.168.1.1', # Optional + loopback_addr => '10.0.0.1', # Optional + tcp_port => 830 # Optional + } + ); + die $err if defined $err; + +=cut +sub update { + my $args = { + db => undef, + node => undef, + @_ + }; + + return 'Required argument `db` is missing.' if !defined $args->{db}; + return 'Required argument `node` is missing.' if !defined $args->{node}; + return 'Required argument `node->node_id` is missing.' if !defined $args->{node}->{node_id}; + + my $params = []; + my $values = []; + if (exists $args->{node}->{name}) { + push @$params, 'name=?'; + push @$values, $args->{node}->{name}; + } + if (exists $args->{node}->{longitude}) { + push @$params, 'longitude=?'; + push @$values, $args->{node}->{longitude}; + } + if (exists $args->{node}->{latitude}) { + push @$params, 'latitude=?'; + push @$values, $args->{node}->{latitude}; + } + if (exists $args->{node}->{operational_state_mpls}) { + push @$params, 'operational_state_mpls=?'; + push @$values, $args->{node}->{operational_state_mpls}; + } + if (exists $args->{node}->{vlan_tag_range}) { + push @$params, 'vlan_tag_range=?'; + push @$values, $args->{node}->{vlan_tag_range}; + } + if (exists $args->{node}->{pending_diff}) { + push @$params, 'pending_diff=?'; + push @$values, $args->{node}->{pending_diff}; + } + if (exists $args->{node}->{short_name}) { + push @$params, 'short_name=?'; + push @$values, $args->{node}->{short_name}; + } + my $fields = join(', ', @$params); + push @$values, $args->{node}->{node_id}; + + if ($fields ne ""){ + my $ok = $args->{db}->execute_query( + "UPDATE node SET $fields WHERE node_id=?", + $values + ); + if (!defined $ok) { + return $args->{db}->get_error; + } + } + + my $iparams = []; + my $ivalues = []; + + my $node = $args->{db}->execute_query( + "select * from node_instantiation where end_epoch=-1 and node_id=?", + [ $args->{node}->{node_id} ] + ); + return $args->{db}->get_error if !defined $node; + return "Couldn't find instantiation for node $args->{node}->{node_id}." if !defined $node->[0]; + $node = $node->[0]; + + my $modified = 0; + if (exists $args->{node}->{admin_state} && $args->{node}->{admin_state} ne $node->{admin_state}) { + $modified = 1; + $node->{admin_state} = $args->{node}->{admin_state}; + } + if (exists $args->{node}->{vendor} && $args->{node}->{vendor} ne $node->{vendor}) { + $modified = 1; + $node->{vendor} = $args->{node}->{vendor}; + } + if (exists $args->{node}->{model} && $args->{node}->{model} ne $node->{model}) { + $modified = 1; + $node->{model} = $args->{node}->{model}; + } + if (exists $args->{node}->{sw_version} && $args->{node}->{sw_version} ne $node->{sw_version}) { + $modified = 1; + $node->{sw_version} = $args->{node}->{sw_version}; + } + if (exists $args->{node}->{mgmt_addr} && $args->{node}->{mgmt_addr} ne $node->{mgmt_addr}) { + $modified = 1; + $node->{mgmt_addr} = $args->{node}->{mgmt_addr}; + } + if (exists $args->{node}->{loopback_address} && $args->{node}->{loopback_address} ne $node->{loopback_address}) { + $modified = 1; + $node->{loopback_address} = $args->{node}->{loopback_address}; + } + if (exists $args->{node}->{tcp_port} && $args->{node}->{tcp_port} != $node->{tcp_port}) { + $modified = 1; + $node->{tcp_port} = $args->{node}->{tcp_port}; + } + + # No changes required to instantiation table + return if (!$modified); + + my $inst_ok = $args->{db}->execute_query( + "UPDATE node_instantiation SET end_epoch=UNIX_TIMESTAMP(NOW()) WHERE node_id=? and end_epoch=-1", + [$args->{node}->{node_id}] + ); + if (!defined $inst_ok) { + return $args->{db}->get_error; + } + $inst_ok = $args->{db}->execute_query(" + INSERT INTO node_instantiation ( + admin_state, vendor, model, sw_version, mgmt_addr, + loopback_address, tcp_port, node_id, start_epoch, end_epoch + ) VALUES (?,?,?,?,?,?,?,?,UNIX_TIMESTAMP(NOW()),-1) + ", + [ + $node->{admin_state}, + $node->{vendor}, + $node->{model}, + $node->{sw_version}, + $node->{mgmt_addr}, + $node->{loopback_address}, + $node->{tcp_port}, + $args->{node}->{node_id}] + ); + if (!defined $inst_ok) { + return $args->{db}->get_error; + } + + return; +} + 1; diff --git a/perl-lib/OESS/t/z-DB/Node.update.00.t b/perl-lib/OESS/t/z-DB/Node.update.00.t new file mode 100644 index 000000000..ce61a307d --- /dev/null +++ b/perl-lib/OESS/t/z-DB/Node.update.00.t @@ -0,0 +1,105 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use FindBin; +my $path; + +BEGIN { + if ($FindBin::Bin =~ /(.*)/) { + $path = $1; + } +} +use lib "$path/.."; + + +use Data::Dumper; +use Test::More tests => 20; + +use OESSDatabaseTester; + +use OESS::DB; +use OESS::DB::Node; + +# Purpose: +# +# Verify interface updates are correctly saved into the database. + + +OESSDatabaseTester::resetOESSDB( + config => "$path/../conf/database.xml", + dbdump => "$path/../conf/oess_known_state.sql" +); + +my $db = new OESS::DB( + config => "$path/../conf/database.xml" +); + + +my $model = { + node_id => 1, + name => 'host.examle.com', # Optional + latitude => 1, # Optional + longitude => 1, # Optional + operational_state_mpls => 'up', # Optional + vlan_tag_range => '1-4095', # Optional + pending_diff => 1, # Optional + short_name => 'host', # Optional + admin_state => 'active', # Optional + vendor => 'juniper', # Optional + model => 'mx', # Optional + sw_version => '13.3R3', # Optional + mgmt_addr => '192.168.1.1', # Optional + loopback_address => '10.0.0.1', # Optional + tcp_port => 830 # Optional +}; + + +my $i1 = $db->execute_query("select * from node_instantiation where node_id=1"); +my $icount1 = @$i1; + +my $err = OESS::DB::Node::update( + db => $db, + node => $model +); +ok(!defined $err, 'Node updated'); +warn $err if defined $err; + +# Verify created instantiation entry +my $i2 = $db->execute_query("select * from node_instantiation where node_id=1"); +my $icount2 = @$i2; +ok($icount2 == $icount1+1, "Got expected number of instantiation entries."); + +# Verify non-effective edit creates no new instantiation entries +OESS::DB::Node::update( + db => $db, + node => $model +); +my $i3 = $db->execute_query("select * from node_instantiation where node_id=1"); +my $icount3 = @$i2; +ok($icount3 == $icount1+1, "Got expected number of instantiation entries."); + +my $intf = OESS::DB::Node::fetch( + db => $db, + node_id => 1 +); + +foreach my $key (keys %$model) { + ok($intf->{$key} eq $model->{$key}, "got expected $key from db"); +} + + +my $err1 = OESS::DB::Node::update( + db => $db, + node => undef +); +ok(defined $err1, "Got expected error $err1"); + + +delete $model->{node_id}; +my $err2 = OESS::DB::Node::update( + db => $db, + node => $model +); +ok(defined $err2, "Got expected error $err2"); From b311352817814b34d082a27b1f34504cf32c1d91 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 6 Nov 2020 15:43:08 -0500 Subject: [PATCH 013/156] save nso device data into oess db --- perl-lib/OESS/lib/OESS/DB/Node.pm | 12 +- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 22 +- perl-lib/OESS/lib/OESS/Node.pm | 265 +++++++++++++++++++++--- 3 files changed, 262 insertions(+), 37 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/DB/Node.pm b/perl-lib/OESS/lib/OESS/DB/Node.pm index 08559bd77..f999c7315 100644 --- a/perl-lib/OESS/lib/OESS/DB/Node.pm +++ b/perl-lib/OESS/lib/OESS/DB/Node.pm @@ -176,7 +176,7 @@ sub _process_tag_string{ model => 'mx', # Optional sw_version => '13.3R3', # Optional mgmt_addr => '192.168.1.1', # Optional - loopback_addr => '10.0.0.1', # Optional + loopback_address => '10.0.0.1', # Optional tcp_port => 830 # Optional } ); @@ -291,8 +291,9 @@ sub update { $inst_ok = $args->{db}->execute_query(" INSERT INTO node_instantiation ( admin_state, vendor, model, sw_version, mgmt_addr, - loopback_address, tcp_port, node_id, start_epoch, end_epoch - ) VALUES (?,?,?,?,?,?,?,?,UNIX_TIMESTAMP(NOW()),-1) + loopback_address, tcp_port, node_id, openflow, mpls, + start_epoch, end_epoch + ) VALUES (?,?,?,?,?,?,?,?,?,?,UNIX_TIMESTAMP(NOW()),-1) ", [ $node->{admin_state}, @@ -302,7 +303,10 @@ sub update { $node->{mgmt_addr}, $node->{loopback_address}, $node->{tcp_port}, - $args->{node}->{node_id}] + $args->{node}->{node_id}, + $args->{node}->{openflow} || 0, + $args->{node}->{mpls} || 1 + ] ); if (!defined $inst_ok) { return $args->{db}->get_error; diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index cd0a1ba97..86f98fb8e 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -11,9 +11,12 @@ use JSON; use Log::Log4perl; use LWP::UserAgent; use XML::LibXML; + use OESS::Config; use OESS::DB; +use OESS::DB::Interface; use OESS::DB::Node; +use OESS::Node; use OESS::RabbitMQ::Dispatcher; =head1 OESS::NSO::Discovery @@ -86,7 +89,7 @@ sub device_handler { next; } - my $device = eval { + my $data = eval { return { name => $dom->findvalue('/ncs:device/ncs:name'), platform => $dom->findvalue('/ncs:device/ncs:platform/ncs:name'), @@ -101,8 +104,14 @@ sub device_handler { next; } - # TODO save nso queried data into db - warn 'device: ' . Dumper($device); + $self->{db}->start_transaction; + my $device = new OESS::Node(db => $self->{db}, name => $data->{name}); + $device->name($data->{name}); + $device->model($data->{model}); + $device->sw_version($data->{version}); + $device->vendor('Cisco') if ($data->{platform} eq 'ios-xr'); + $device->update; + $self->{db}->commit; } return 1; } @@ -125,8 +134,11 @@ sub interface_handler { return XML::LibXML->load_xml(string => $res->content); }; if ($@) { - warn 'tailf-ned-cisco-ios-xr:interface:' . $@; - $self->{logger}->error('tailf-ned-cisco-ios-xr:interface:' . $@); + # Don't log Empty String error as there are simply no interfaces + if ($@ !~ /Empty String/g) { + warn 'tailf-ned-cisco-ios-xr:interface:' . $@; + $self->{logger}->error('tailf-ned-cisco-ios-xr:interface:' . $@); + } next; } diff --git a/perl-lib/OESS/lib/OESS/Node.pm b/perl-lib/OESS/lib/OESS/Node.pm index 2bd15de17..2b935528c 100644 --- a/perl-lib/OESS/lib/OESS/Node.pm +++ b/perl-lib/OESS/lib/OESS/Node.pm @@ -14,21 +14,18 @@ sub new{ my $that = shift; my $class = ref($that) || $that; - my $logger = Log::Log4perl->get_logger("OESS.Node"); - my %args = ( + db => undef, + logger => Log::Log4perl->get_logger("OESS.Node"), + name => undef, node_id => undef, - db => undef, @_ - ); - + ); my $self = \%args; bless $self, $class; - $self->{'logger'} = $logger; - - if(!defined($self->{'db'})){ + if (!defined $self->{db}) { $self->{'logger'}->error("No Database Object specified"); return; } @@ -41,27 +38,51 @@ sub new{ =head2 from_hash =cut -sub from_hash{ +sub from_hash { my $self = shift; my $hash = shift; - $self->{'node_id'} = $hash->{'node_id'}; - $self->{'name'} = $hash->{'name'}; - $self->{'latitude'} = $hash->{'latitude'}; - $self->{'longitude'} = $hash->{'longitude'}; - + $self->{node_id} = $hash->{node_id}; + $self->{name} = $hash->{name}; + $self->{latitude} = $hash->{latitude}; + $self->{longitude} = $hash->{longitude}; + $self->{vlan_tag_range} = $hash->{vlan_tag_range}; + $self->{operational_state_mpls} = $hash->{operational_state_mpls} || 'up'; + $self->{pending_diff} = $hash->{pending_diff} || 0; + $self->{admin_state} = $hash->{admin_state} || 'active'; + $self->{short_name} = $hash->{short_name}; + $self->{vendor} = $hash->{vendor}; + $self->{model} = $hash->{model}; + $self->{sw_version} = $hash->{sw_version}; + $self->{mgmt_addr} = $hash->{mgmt_addr}; + $self->{loopback_address} = $hash->{loopback_address}; + $self->{tcp_port} = $self->{tcp_port} || 830; + + return 1; } =head2 to_hash =cut -sub to_hash{ +sub to_hash { my $self = shift; - my $obj = { node_id => $self->{'node_id'}, - name => $self->{'name'}, - latitude => $self->{'latitude'}, - longitude => $self->{'longitude'}}; - + my $obj = { + node_id => $self->{'node_id'}, + name => $self->{'name'}, + latitude => $self->{'latitude'}, + longitude => $self->{'longitude'}, + vlan_tag_range => $self->{vlan_tag_range}, + operational_state_mpls => $self->{operational_state_mpls}, + pending_diff => $self->{pending_diff}, + admin_state => $self->{admin_state}, + short_name => $self->{short_name}, + vendor => $self->{vendor}, + model => $self->{model}, + sw_version => $self->{sw_version}, + mgmt_addr => $self->{mgmt_addr}, + loopback_address => $self->{loopback_address}, + tcp_port => $self->{tcp_port} + }; return $obj; } @@ -71,30 +92,49 @@ sub to_hash{ sub _fetch_from_db{ my $self = shift; my $db = $self->{'db'}; - my $hash = OESS::DB::Node::fetch(db => $db, node_id => $self->{'node_id'}); + my $hash = OESS::DB::Node::fetch( + db => $db, + name => $self->{name}, + node_id => $self->{node_id} + ); $self->from_hash($hash); } -=head2 node_id +=head2 update =cut -sub node_id{ +sub update { my $self = shift; - return $self->{'node_id'}; + + if (!defined $self->{db}) { + $self->{'logger'}->error("Could not update Node: No database object specified."); + return; + } + + my $err = OESS::DB::Node::update( + db => $self->{db}, + node => $self->to_hash + ); + if (defined $err) { + $self->{'logger'}->error("Could not update Node: $err"); + return; + } + + return 1; } -=head2 name +=head2 node_id =cut -sub name{ +sub node_id { my $self = shift; - return $self->{'name'}; + return $self->{'node_id'}; } =head2 interfaces =cut -sub interfaces{ +sub interfaces { my $self = shift; my $interfaces = shift; @@ -111,4 +151,173 @@ sub interfaces{ } } +=head2 latitude + +=cut +sub latitude { + my $self = shift; + my $latitude = shift; + + if (defined $latitude) { + $self->{latitude} = $latitude; + } + return $self->{latitude}; +} + +=head2 longitude + +=cut +sub longitude { + my $self = shift; + my $longitude = shift; + + if (defined $longitude) { + $self->{longitude} = $longitude; + } + return $self->{longitude}; +} + +=head2 mgmt_addr + +=cut +sub mgmt_addr { + my $self = shift; + my $mgmt_addr = shift; + + if (defined $mgmt_addr) { + $self->{mgmt_addr} = $mgmt_addr; + } + return $self->{mgmt_addr}; +} + +=head2 loopback_address + +=cut +sub loopback_address { + my $self = shift; + my $loopback_address = shift; + + if (defined $loopback_address) { + $self->{loopback_address} = $loopback_address; + } + return $self->{loopback_address}; +} + +=head2 model + +=cut +sub model { + my $self = shift; + my $model = shift; + + if (defined $model) { + $self->{model} = $model; + } + return $self->{model}; +} + +=head2 name + +=cut +sub name { + my $self = shift; + my $name = shift; + + if (defined $name) { + $self->{name} = $name; + } + return $self->{name}; +} + +=head2 pending_diff + +=cut +sub pending_diff { + my $self = shift; + my $pending_diff = shift; + + if (defined $pending_diff) { + $self->{pending_diff} = $pending_diff; + } + return $self->{pending_diff}; +} + +=head2 short_name + +=cut +sub short_name { + my $self = shift; + my $short_name = shift; + + if (defined $short_name) { + $self->{short_name} = $short_name; + } + return $self->{short_name}; +} + +=head2 sw_version + +=cut +sub sw_version { + my $self = shift; + my $sw_version = shift; + + if (defined $sw_version) { + $self->{sw_version} = $sw_version; + } + return $self->{sw_version}; +} + +=head2 tcp_port + +=cut +sub tcp_port { + my $self = shift; + my $tcp_port = shift; + + if (defined $tcp_port) { + $self->{tcp_port} = $tcp_port; + } + return $self->{tcp_port}; +} + +=head2 vendor + +=cut +sub vendor { + my $self = shift; + my $vendor = shift; + + if (defined $vendor) { + $self->{vendor} = $vendor; + } + return $self->{vendor}; +} + +=head2 vlan_tag_range + +=cut +sub vlan_tag_range { + my $self = shift; + my $vlan_tag_range = shift; + + if (defined $vlan_tag_range) { + $self->{vlan_tag_range} = $vlan_tag_range; + } + return $self->{vlan_tag_range}; +} + +=head2 operational_state_mpls + +=cut +sub operational_state_mpls { + my $self = shift; + my $operational_state_mpls = shift; + + if (defined $operational_state_mpls) { + $self->{operational_state_mpls} = $operational_state_mpls; + } + return $self->{operational_state_mpls}; +} + 1; From 6f505875732beacbd1bfe372ff4eefc8d950797d Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 9 Nov 2020 09:56:22 -0500 Subject: [PATCH 014/156] save discovered interface state to db --- perl-lib/OESS/lib/OESS/Interface.pm | 91 ++++++++++++++++++------- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 34 +++++++++ 2 files changed, 100 insertions(+), 25 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/Interface.pm b/perl-lib/OESS/lib/OESS/Interface.pm index 467e33c75..38cf9792b 100644 --- a/perl-lib/OESS/lib/OESS/Interface.pm +++ b/perl-lib/OESS/lib/OESS/Interface.pm @@ -24,11 +24,10 @@ sub new{ my $that = shift; my $class = ref($that) || $that; - my $logger = Log::Log4perl->get_logger("OESS.Interface"); - my %args = ( db => undef, interface_id => undef, + logger => Log::Log4perl->get_logger("OESS.Interface"), model => undef, @_ ); @@ -36,9 +35,6 @@ sub new{ my $self = \%args; bless $self, $class; - $self->{'logger'} = $logger; - - if (!defined $self->{'db'}) { $self->{'logger'}->warn("No Database Object specified"); } @@ -67,6 +63,7 @@ sub from_hash{ $self->{'name'} = $hash->{'name'}; $self->{'interface_id'} = $hash->{'interface_id'}; + $self->{'node_id'} = $hash->{'node_id'}; $self->{'node'} = $hash->{'node'}; $self->{'cloud_interconnect_id'} = $hash->{'cloud_interconnect_id'}; $self->{'cloud_interconnect_type'} = $hash->{'cloud_interconnect_type'}; @@ -90,26 +87,32 @@ sub from_hash{ sub to_hash{ my $self = shift; - my $acl_models = []; - foreach my $acl (@{$self->acls()}) { - push @$acl_models, $acl->to_hash(); - } - - my $res = { name => $self->name(), - cloud_interconnect_id => $self->cloud_interconnect_id(), - cloud_interconnect_type => $self->cloud_interconnect_type(), - description => $self->description(), - interface_id => $self->interface_id(), - node_id => $self->node()->node_id(), - node => $self->node()->name(), - acls => $acl_models, - admin_state => $self->{'admin_state'}, - operational_state => $self->{'operational_state'}, - workgroup_id => $self->workgroup_id(), - utilized_bandwidth => $self->{'utilized_bandwidth'}, - bandwidth => $self->{'bandwidth'}, - mtu => $self->{'mtu'} }; + my $res = { + name => $self->name(), + cloud_interconnect_id => $self->cloud_interconnect_id(), + cloud_interconnect_type => $self->cloud_interconnect_type(), + description => $self->description(), + interface_id => $self->interface_id(), + node_id => $self->{node_id}, + admin_state => $self->{'admin_state'}, + operational_state => $self->{'operational_state'}, + workgroup_id => $self->workgroup_id(), + utilized_bandwidth => $self->{'utilized_bandwidth'}, + bandwidth => $self->{'bandwidth'}, + mtu => $self->{'mtu'} + }; + if (defined $self->{acls}) { + my $acl_models = []; + foreach my $acl (@{$self->{acls}}) { + push @$acl_models, $acl->to_hash; + } + $res->{acls} = $acl_models; + } + if (defined $self->{node}) { + $res->{node} = $self->node()->name(); + $res->{node_id} = $self->node()->node_id(); + } return $res; } @@ -123,6 +126,7 @@ sub _fetch_from_db{ if (defined $self->{'name'} && defined $self->{'node'}) { my $interface_id = OESS::DB::Interface::get_interface(db => $self->{'db'}, interface => $self->{'name'}, node => $self->{'node'}); if (!defined $interface_id) { + warn "Unable to fetch interface $self->{name} on $self->{node} from the db!"; $self->{'logger'}->error("Unable to fetch interface $self->{name} on $self->{node} from the db!"); return; } @@ -142,6 +146,38 @@ sub _fetch_from_db{ return $self->from_hash($info); } +=head2 create + +=cut +sub create { + my $self = shift; + my $args = { + node_id => undef, + @_ + }; + + if (!defined $self->{db}) { + $self->{logger}->error("Couldn't create Interface; DB handle is missing."); + return (undef, "Couldn't create Interface; DB handle is missing."); + } + if (!defined $args->{node_id}) { + $self->{logger}->error("Couldn't create Interface; node_id is missing."); + return (undef, "Couldn't create Interface; node_id is missing."); + } + + my $model = $self->to_hash; + $model->{node_id} = $args->{node_id}; + + my ($id, $err) = OESS::DB::Interface::create( + db => $self->{db}, + model => $model + ); + return (undef, $err) if defined $err; + + $self->{interface_id} = $id; + return ($id, undef); +} + =head2 update_db =cut @@ -254,7 +290,12 @@ sub cloud_interconnect_type{ =cut sub description{ my $self = shift; - return $self->{'description'}; + my $description = shift; + + if (defined $description) { + $self->{description} = $description; + } + return $self->{description}; } =head2 port_number diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index 86f98fb8e..929a86137 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -106,6 +106,10 @@ sub device_handler { $self->{db}->start_transaction; my $device = new OESS::Node(db => $self->{db}, name => $data->{name}); + if (!defined $device) { + warn "Couldn't find node $data->{name}."; + $self->{logger}->error("Couldn't find node $data->{name}."); + } $device->name($data->{name}); $device->model($data->{model}); $device->sw_version($data->{version}); @@ -166,6 +170,36 @@ sub interface_handler { # TODO save nso queried data into db warn 'ports: ' . Dumper($ports); + $self->{db}->start_transaction; + foreach my $data (@$ports) { + my $port = new OESS::Interface(db => $self->{db}, node => $node->{name}, name => $data->{name}); + if (defined $port) { + $port->admin_state($data->{admin_state}); + $port->bandwidth($data->{bandwidth}); + $port->description($data->{description}); + $port->mtu($data->{mtu}); + $port->update_db; + } else { + warn "Couldn't find interface $node->{name} $data->{name}; Creating interface."; + $self->{logger}->warn("Couldn't find interface $node->{name} $data->{name}; Creating interface."); + + $port = new OESS::Interface(db => $self->{db}, model => { + admin_state => $data->{admin_state}, + bandwidth => $data->{bandwidth}, + description => $data->{description}, + mtu => $data->{mtu}, + name => $data->{name}, + operational_state => $data->{admin_state} # Using admin_state as best guess for now + }); + + my ($port_id, $port_err) = $port->create(node_id => $node->{node_id}); + if (defined $port_err) { + warn "Couldn't create interface $node->{name} $data->{name}."; + $self->{logger}->error("Couldn't create interface $node->{name} $data->{name}.") + } + } + } + $self->{db}->commit; } return 1; } From 7cba075ca53c905c13166b26f1fbbf68554a9299 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 9 Nov 2020 18:53:19 +0000 Subject: [PATCH 015/156] limit modification of remote_auth entires when editing users preivously edit_user would create a new entriy in the remote_auth table for each username associated with a user, even if there was no change to those usernames. This change modifies this behavior to only make changes to the remote_auth entries when requried. --- perl-lib/OESS/lib/OESS/DB/User.pm | 57 ++++++---- perl-lib/OESS/t/z-DB/User.edit_user.01.t | 137 +++++++++++++++++++++++ 2 files changed, 172 insertions(+), 22 deletions(-) create mode 100644 perl-lib/OESS/t/z-DB/User.edit_user.01.t diff --git a/perl-lib/OESS/lib/OESS/DB/User.pm b/perl-lib/OESS/lib/OESS/DB/User.pm index 1f7fef100..a2a3999c6 100644 --- a/perl-lib/OESS/lib/OESS/DB/User.pm +++ b/perl-lib/OESS/lib/OESS/DB/User.pm @@ -302,7 +302,7 @@ sub delete_user { sub edit_user { my %params = @_; my $db = $params{'db'}; - + my $user_id = $params{'user_id'}; my $given_name = $params{'given_name'}; my $family_name = $params{'family_name'}; @@ -317,37 +317,50 @@ sub edit_user { return (undef, 'Required argument `email` is missing.') if !defined $email; return (undef, 'Required argument `auth_names` is missing.') if !defined $auth_names; return (undef, 'Required argument `status` is missing.') if !defined $status; - + if ($given_name =~ /^system$/ || $family_name =~ /^system$/) { return(undef, "User 'system' is reserved."); } + my $query = "UPDATE user SET email = ?, given_names = ?, family_name = ?, status = ? WHERE user_id = ?"; - my $query = "UPDATE user SET email = ?, given_names = ?, family_name = ?, status = ? WHERE user_id = $user_id"; - - my $results = $db->execute_query($query, [$email, $given_name, $family_name, $status]); - + my $results = $db->execute_query($query, [$email, $given_name, $family_name, $status, $user_id]); if (!defined $user_id || $results == 0) { return (undef, "Unable to edit user - does this user actually exist?"); } - # TODO Blindly removing and adding remote_auth entries for a user - # causes a lot of churn in database ids. Modify this logic to only - # update the remote_auth table when required. - $db->execute_query("DELETE FROM remote_auth WHERE user_id = ?", [$user_id]); - - if (ref($auth_names) eq 'ARRAY') { - foreach my $name (@$auth_names){ - if (length $name >=1) { - $query = "INSERT INTO remote_auth (auth_name, user_id) VALUES (?, ?)"; - $db->execute_query($query, [$name,$user_id]); - } + if (ref($auth_names) ne 'ARRAY') { + $auth_names = [$auth_names]; + } + + my $uindex = {}; + my $usernames = $db->execute_query("select * from remote_auth where user_id=?", [$user_id]); + if (!defined $usernames) { + warn "edit_user called on user without any known usernames."; + $usernames = []; + } + foreach my $u (@$usernames) { + $uindex->{$u->{auth_name}} = $u; + } + + foreach my $name (@$auth_names) { + if (defined $uindex->{$name}) { + delete $uindex->{$name}; + next; } - } else { - return (undef, 'Auth_Names is required to be at least 1 character.') if length $auth_names <1; - $query = "INSERT INTO remote_auth (auth_name, user_id) VALUES (?, ?)"; - $db->execute_query($query, [$auth_names, $user_id]); - } + + return (undef, 'auth_names is required to be at least 1 character.') if length $auth_names < 1; + + my $ok = $db->execute_query("INSERT INTO remote_auth (auth_name, user_id) VALUES (?, ?)", [$name, $user_id]); + return (undef, $db->get_error) if !defined $ok; + + delete $uindex->{$name}; + } + + foreach my $name (keys %$uindex) { + my $ok = $db->execute_query("delete from remote_auth where auth_name=?", [$name]); + return (undef, $db->get_error) if !defined $ok; + } return (1,undef) } diff --git a/perl-lib/OESS/t/z-DB/User.edit_user.01.t b/perl-lib/OESS/t/z-DB/User.edit_user.01.t new file mode 100644 index 000000000..a61a29923 --- /dev/null +++ b/perl-lib/OESS/t/z-DB/User.edit_user.01.t @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use FindBin; +my $path; + +BEGIN { + if ($FindBin::Bin =~ /(.*)/) { + $path = $1; + } +} +use lib "$path/.."; + +use Data::Dumper; +use Test::More tests => 28; + +use OESSDatabaseTester; + +use OESS::DB; +use OESS::DB::User; + +# Purpose: +# +# Verify user creation errors when bad type specified. + +OESSDatabaseTester::resetOESSDB( + config => "$path/../conf/database.xml", + dbdump => "$path/../conf/oess_known_state.sql" +); + +my $db = new OESS::DB( + config => "$path/../conf/database.xml" +); +my $model = { + given_names => 'Testerfield', + family_name => 'Testerton', + email => 'ttesterton@testertonestates.com', + username => 'ttesterton', + status => 'active' +}; + +my ($id, $err) = OESS::DB::User::add_user( + db => $db, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => $model->{username} +); +ok(defined $id, "User entry was created"); + +my $user = OESS::DB::User::fetch(db => $db, user_id => $id); +foreach my $key (keys %$model) { + ok($user->{$key} eq $model->{$key}, "got expected initial $key from db"); +} +$model->{family_name} = 'Please-Ignore'; + + +my ($res, $err2) = OESS::DB::User::edit_user( + db => $db, + user_id => $id, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => [ $model->{username}, 'ttesterton2', 'ttesterton3' ], + status => $model->{status} +); +print ("ERR = $err2") if defined $err2; +ok($res == 1, "Editing User was successful"); + + +my $index = { 'ttesterton' => 962, 'ttesterton2' => 963, 'ttesterton3' => 964 }; +my $users = $db->execute_query("select * from remote_auth where user_id=?", [$id]); + + +foreach my $user (@$users) { + my $name = $user->{auth_name}; + my $uid = $user->{auth_id}; + ok(defined $index->{$name}, "found expected username $name"); + ok($index->{$name} == $uid, "found expected user id $uid"); + delete $index->{$name}; +} + +ok(keys %$index == 0, "all expected usernames accounted for"); + + +my ($res3, $err3) = OESS::DB::User::edit_user( + db => $db, + user_id => $id, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => [ $model->{username}, 'ttesterton3' ], + status => $model->{status} +); +print ("ERR = $err3") if defined $err3; +ok($res3 == 1, "Editing User was successful"); + +$index = { 'ttesterton' => 962, 'ttesterton3' => 964 }; +$users = $db->execute_query("select * from remote_auth where user_id=?", [$id]); + +foreach my $user (@$users) { + my $name = $user->{auth_name}; + my $uid = $user->{auth_id}; + ok(defined $index->{$name}, "found expected username $name"); + ok($index->{$name} == $uid, "found expected user id $uid"); + delete $index->{$name}; +} + +ok(keys %$index == 0, "all expected usernames accounted for"); + + +my ($res4, $err4) = OESS::DB::User::edit_user( + db => $db, + user_id => $id, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => [ $model->{username}, 'ttesterton3', 'ttesterton4' ], + status => $model->{status} +); +print ("ERR = $err4") if defined $err4; +ok($res4 == 1, "Editing User was successful"); + +$index = { 'ttesterton' => 962, 'ttesterton3' => 964, 'ttesterton4' => 965 }; +$users = $db->execute_query("select * from remote_auth where user_id=?", [$id]); + +foreach my $user (@$users) { + my $name = $user->{auth_name}; + my $uid = $user->{auth_id}; + ok(defined $index->{$name}, "found expected username $name"); + ok($index->{$name} == $uid, "found expected user id $uid"); + delete $index->{$name}; +} + +ok(keys %$index == 0, "all expected usernames accounted for"); From 963191f2042d3a56522bdc8d733fa95ccd3e4c63 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 28 Dec 2020 17:11:47 -0500 Subject: [PATCH 016/156] add initial nso version of fwdctl --- app/mpls/mpls_discovery.pl | 8 +- app/mpls/mpls_fwdctl.pl | 28 +- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 12 +- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 415 ++++++++++++++++++++++++ 4 files changed, 449 insertions(+), 14 deletions(-) create mode 100644 perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm diff --git a/app/mpls/mpls_discovery.pl b/app/mpls/mpls_discovery.pl index 189525189..0124982f5 100755 --- a/app/mpls/mpls_discovery.pl +++ b/app/mpls/mpls_discovery.pl @@ -10,7 +10,6 @@ use Data::Dumper; use OESS::Config; -use OESS::Database; use OESS::MPLS::Discovery; use OESS::NSO::Discovery; @@ -18,8 +17,7 @@ my $cnf_file = "/etc/oess/database.xml"; sub core{ - #basic init stuffs - Log::Log4perl::init_and_watch('/etc/oess/logging.conf',10); + Log::Log4perl::init_and_watch('/etc/oess/logging.conf', 10); my $config = new OESS::Config(config_filename => $cnf_file); if ($config->network_type eq 'nso') { @@ -32,8 +30,10 @@ sub core{ AnyEvent->condvar->recv; } else { - die "Unexpected network type configured." + die "Unexpected network type configured."; } + + Log::Log4perl->get_logger('OESS.MPLS.Discovery.APP')->info("Starting OESS.MPLS.Discovery event loop."); } sub main{ diff --git a/app/mpls/mpls_fwdctl.pl b/app/mpls/mpls_fwdctl.pl index d06c8c5d2..d0d859d03 100755 --- a/app/mpls/mpls_fwdctl.pl +++ b/app/mpls/mpls_fwdctl.pl @@ -2,7 +2,9 @@ use strict; use warnings; +use OESS::Config; use OESS::MPLS::FWDCTL; +use OESS::NSO::FWDCTL; use English; use Data::Dumper; @@ -12,6 +14,7 @@ use XML::Simple; my $pid_file = "/var/run/oess/mpls_fwdctl.pid"; +my $cnf_file = "/etc/oess/database.xml"; sub get_diff_interval{ eval { @@ -27,13 +30,24 @@ sub get_diff_interval{ sub core{ Log::Log4perl::init_and_watch('/etc/oess/logging.conf', 10); - my $FWDCTL = OESS::MPLS::FWDCTL->new(); - my $reaper = AnyEvent->timer( after => 3600, interval => 3600, cb => sub { $FWDCTL->reap_old_events() } ); - my $status = AnyEvent->timer( after => 10, interval => 60, cb => sub { $FWDCTL->save_mpls_nodes_status() } ); - my $differ = AnyEvent->timer( after => 5, interval => get_diff_interval(), cb => sub { $FWDCTL->diff() } ); + my $config = new OESS::Config(config_filename => $cnf_file); + if ($config->network_type eq 'nso') { + my $fwdctl = OESS::NSO::FWDCTL->new(config => $config); + $fwdctl->start; + AnyEvent->condvar->recv; + } + elsif ($config->network_type eq 'vpn-mpls' || $config->network_type eq 'evpn-vxlan') { + my $FWDCTL = OESS::MPLS::FWDCTL->new(); + my $reaper = AnyEvent->timer( after => 3600, interval => 3600, cb => sub { $FWDCTL->reap_old_events() } ); + my $status = AnyEvent->timer( after => 10, interval => 60, cb => sub { $FWDCTL->save_mpls_nodes_status() } ); + my $differ = AnyEvent->timer( after => 5, interval => get_diff_interval(), cb => sub { $FWDCTL->diff() } ); + AnyEvent->condvar->recv; + } + else { + die "Unexpected network type configured."; + } Log::Log4perl->get_logger('OESS.MPLS.FWDCTL.APP')->info("Starting OESS.MPLS.FWDCTL event loop."); - AnyEvent->condvar->recv; } sub main{ @@ -42,8 +56,8 @@ sub main{ my $username; #remove the ready file - # This directory is auto-removed on reboot. Create the directory - # if not already created. + # This directory is auto-removed on reboot. Create the directory if not + # already created. This is used to store connection cache files. if (!-d "/var/run/oess/") { `/usr/bin/mkdir /var/run/oess`; `/usr/bin/chown _oess:_oess /var/run/oess`; diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index 929a86137..baec8a1dc 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -224,6 +224,11 @@ sub new_switch { my $success = $method->{'success_callback'}; my $error = $method->{'error_callback'}; + if (defined $self->{nodes}->{$params->{node_id}{value}}) { + $self->{logger}->warn("Node $params->{node_id}{value} already registered with Discovery."); + return &$success({ status => 1 }); + } + my $node = OESS::DB::Node::fetch(db => $self->{db}, node_id => $params->{node_id}{value}); if (!defined $node) { my $err = "Couldn't lookup node $params->{node_id}{value}. Discovery will not properly complete on this node."; @@ -232,9 +237,10 @@ sub new_switch { } $self->{nodes}->{$params->{node_id}{value}} = $node; - warn "Switch $node->{name} registered with discovery."; - $self->{logger}->info("Switch $node->{name} registered with discovery."); + warn "Switch $node->{name} registered with Discovery."; + $self->{logger}->info("Switch $node->{name} registered with Discovery."); + # Make first invocation of polling subroutines $self->device_handler; $self->interface_handler; @@ -288,7 +294,7 @@ sub start { my $new_switch = new GRNOC::RabbitMQ::Method( name => 'new_switch', - description => 'Add a new switch to the database', + description => 'Add a new switch process to Discovery', async => 1, callback => sub { $self->new_switch(@_); } ); diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm new file mode 100644 index 000000000..865fdfac2 --- /dev/null +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -0,0 +1,415 @@ +package OESS::NSO::FWDCTL; + +use AnyEvent; +use Data::Dumper; +use GRNOC::RabbitMQ::Method; +use GRNOC::WebService::Regex; +use HTTP::Request::Common; +use JSON; +use Log::Log4perl; +use LWP::UserAgent; +use XML::LibXML; + +use OESS::Config; +use OESS::DB; +use OESS::DB::Node; +use OESS::Node; +use OESS::RabbitMQ::Dispatcher; + +use constant FWDCTL_WAITING => 2; +use constant FWDCTL_SUCCESS => 1; +use constant FWDCTL_FAILURE => 0; +use constant FWDCTL_UNKNOWN => 3; +use constant FWDCTL_BLOCKED => 4; + +=head1 OESS::NSO::FWDCTL + +=cut + +=head2 new + +=cut +sub new { + my $class = shift; + my $args = { + config => undef, + config_filename => '/etc/oess/database.xml', + logger => Log::Log4perl->get_logger('OESS.NSO.FWDCTL'), + @_ + }; + my $self = bless $args, $class; + + if (!defined $self->{config}) { + $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + } + $self->{db} = new OESS::DB(config => $self->{config}->filename); + $self->{nodes} = {}; + + $self->{www} = new LWP::UserAgent; + my $host = $self->{config}->nso_host; + $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string + $self->{www}->credentials($host, "restconf", $self->{config}->nso_username, $self->{config}->nso_password); + + # When this process receives sigterm send an event to notify all + # children to exit cleanly. + $SIG{TERM} = sub { + $self->stop; + }; + + return $self; +} + +=head2 start + +=cut +sub start { + my $self = shift; + + # Load devices from database + my $nodes = OESS::DB::Node::fetch_all(db => $self->{db}); + if (!defined $nodes) { + warn "Couldn't lookup nodes. FWDCTL will not provision on any existing nodes."; + $self->{logger}->error("Couldn't lookup nodes. Discovery will not provision on any existing nodes."); + } + foreach my $node (@$nodes) { + $self->{nodes}->{$node->{node_id}} = $node; + } + + # Setup polling subroutines here: + # See OESS::NSO::Discovery::start for examples + + $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( + # queue => 'oess-fwdctl', + # topic => 'oess.fwdctl.rpc' + queue => 'MPLS-FWDCTL', + topic => 'MPLS.FWDCTL.RPC' + ); + + my $add_vlan = GRNOC::RabbitMQ::Method->new( + name => "addVlan", + async => 1, + callback => sub { $self->addVlan(@_) }, + description => "addVlan provisions a l2 connection" + ); + $add_vlan->add_input_parameter( + name => "circuit_id", + description => "Id of the l2 connection to add", + required => 1, + attern => $GRNOC::WebService::Regex::INTEGER + ); + $self->{dispatcher}->register_method($add_vlan); + + my $delete_vlan = GRNOC::RabbitMQ::Method->new( + name => "deleteVlan", + async => 1, + callback => sub { $self->deleteVlan(@_) }, + description => "deleteVlan removes a l2 connection" + ); + $delete_vlan->add_input_parameter( + name => "circuit_id", + description => "Id of the l2 connection to delete", + required => 1, + pattern => $GRNOC::WebService::Regex::INTEGER + ); + $self->{dispatcher}->register_method($delete_vlan); + + my $modify_vlan = GRNOC::RabbitMQ::Method->new( + name => "modifyVlan", + async => 1, + callback => sub { $self->modifyVlan(@_) }, + description => "modifyVlan modifies an existing l2 connection" + ); + $modify_vlan->add_input_parameter( + name => "circuit_id", + description => "Id of l2 connection to be modified.", + required => 1, + pattern => $GRNOC::WebService::Regex::INTEGER + ); + $modify_vlan->add_input_parameter( + name => "previous", + description => "Previous version of the modified l2 connection.", + required => 1, + pattern => $GRNOC::WebService::Regex::TEXT + ); + $modify_vlan->add_input_parameter( + name => "pending", + description => "Pending version of the modified l2 connection.", + required => 1, + pattern => $GRNOC::WebService::Regex::TEXT + ); + $self->{dispatcher}->register_method($modify_vlan); + + my $add_vrf = GRNOC::RabbitMQ::Method->new( + name => "addVrf", + async => 1, + callback => sub { $self->addVrf(@_) }, + description => "addVrf provisions a l3 connection" + ); + $self->{dispatcher}->register_method($add_vrf); + + my $delete_vrf = GRNOC::RabbitMQ::Method->new( + name => "delVrf", + async => 1, + callback => sub { $self->delVrf(@_) }, + description => "delVrf removes a l3 connection" + ); + $self->{dispatcher}->register_method($delete_vrf); + + my $modify_vrf = GRNOC::RabbitMQ::Method->new( + name => "modifyVrf", + async => 1, + callback => sub { $self->modifyVrf(@_) }, + description => "modifyVrf modifies an existing l3 connection" + ); + $self->{dispatcher}->register_method($modify_vrf); + + # NOTE It's not expected that any children processes will exist in this + # version of FWDCTL. Result is hardcoded. + my $check_child_status = GRNOC::RabbitMQ::Method->new( + name => "check_child_status", + description => "check_child_status returns an event id which will return the final status of all children", + callback => sub { + my $method = shift; + return { status => 1, event_id => 1 }; + } + ); + $self->{dispatcher}->register_method($check_child_status); + + # NOTE It's not expected that any children processes will exist in this + # version of FWDCTL. Result is hardcoded. + my $get_event_status = GRNOC::RabbitMQ::Method->new( + name => "get_event_status", + description => "get_event_status returns the current status of the event", + callback => sub { + my $method = shift; + return { status => 1 }; + } + ); + $get_event_status->add_input_parameter( + name => "event_id", + description => "the event id to fetch the current state of", + required => 1, + pattern => $GRNOC::WebService::Regex::NAME_ID + ); + $self->{dispatcher}->register_method($get_event_status); + + # TODO It's not clear if both is_online and echo are required; Please + # investigate. + my $echo = GRNOC::RabbitMQ::Method->new( + name => "echo", + description => "echo always returns 1", + callback => sub { + my $method = shift; + return { status => 1 }; + } + ); + $self->{dispatcher}->register_method($echo); + + my $get_diff_text = GRNOC::RabbitMQ::Method->new( + name => 'get_diff_text', + async => 1, + callback => sub { $self->get_diff_text(@_); }, + description => "Returns a human readable diff for node_id" + ); + $get_diff_text->add_input_parameter( + name => "node_id", + description => "The node ID to lookup", + required => 1, + pattern => $GRNOC::WebService::Regex::INTEGER + ); + $self->{dispatcher}->register_method($get_diff_text); + + # TODO It's not clear if both is_online and echo are required; Please + # investigate. + my $is_online = new GRNOC::RabbitMQ::Method( + name => "is_online", + description => 'is_online returns 1 if this service is available', + async => 1, + callback => sub { + my $method = shift; + return $method->{success_callback}({ successful => 1 }); + } + ); + $self->{dispatcher}->register_method($is_online); + + my $new_switch = new GRNOC::RabbitMQ::Method( + name => 'new_switch', + description => 'new_switch adds a new switch to FWDCTL', + async => 1, + callback => sub { $self->new_switch(@_); } + ); + $new_switch->add_input_parameter( + name => 'node_id', + description => 'Id of the new node', + required => 1, + pattern => $GRNOC::WebService::Regex::NUMBER_ID + ); + $self->{dispatcher}->register_method($new_switch); + + my $update_cache = GRNOC::RabbitMQ::Method->new( + name => 'update_cache', + async => 1, + callback => sub { $self->update_cache(@_) }, + description => "Rewrites the connection cache file" + ); + $self->{dispatcher}->register_method($update_cache); + + $self->{dispatcher}->start_consuming; + return 1; +} + +=head2 stop + +=cut +sub stop { + my $self = shift; + $self->{logger}->info('Stopping OESS::NSO::FWDCTL.'); + $self->{dispatcher}->stop_consuming; +} + +=head2 addVlan + +=cut +sub addVlan { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 deleteVlan + +=cut +sub deleteVlan { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 modifyVlan + +=cut +sub modifyVlan { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 addVrf + +=cut +sub addVrf { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 deleteVrf + +=cut +sub deleteVrf { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 modifyVrf + +=cut +sub modifyVrf { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 get_diff_text + +=cut +sub get_diff_text { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +=head2 new_switch + +=cut +sub new_switch { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{'success_callback'}; + my $error = $method->{'error_callback'}; + + if (defined $self->{nodes}->{$params->{node_id}{value}}) { + $self->{logger}->warn("Node $params->{node_id}{value} already registered with FWDCTL."); + return &$success({ status => 1 }); + } + + my $node = OESS::DB::Node::fetch(db => $self->{db}, node_id => $params->{node_id}{value}); + if (!defined $node) { + my $err = "Couldn't lookup node $params->{node_id}{value}. FWDCTL will not properly provision on this node."; + $self->{logger}->error($err); + &$error($err); + } + $self->{nodes}->{$params->{node_id}{value}} = $node; + + warn "Switch $node->{name} registered with FWDCTL."; + $self->{logger}->info("Switch $node->{name} registered with FWDCTL."); + + # Make first invocation of polling subroutines here: + # See OESS::NSO::Discovery::start for examples + + return &$success({ status => 1 }); +} + +=head2 update_cache + +=cut +sub update_cache { + my $self = shift; + my $method = shift; + my $params = shift; + + my $success = $method->{success_callback}; + my $error = $method->{error_callback}; + + return &$success({ status => 1 }); +} + +1; From 252d0527ac4781130477c1320021e0b48f363762 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Tue, 29 Dec 2020 11:02:41 -0500 Subject: [PATCH 017/156] add diff method to fwdctl --- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 865fdfac2..e27a2fcab 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -75,8 +75,12 @@ sub start { $self->{nodes}->{$node->{node_id}} = $node; } - # Setup polling subroutines here: - # See OESS::NSO::Discovery::start for examples + # Setup polling subroutines + $self->{connection_timer} = AnyEvent->timer( + after => 5, + interval => 300, + cb => sub { $self->diff(@_); } + ); $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( # queue => 'oess-fwdctl', @@ -351,6 +355,19 @@ sub modifyVrf { return &$success({ status => 1 }); } +=head2 diff + +diff reads all connections from cache, loads all connections from nso, +determines if a configuration change within nso is required, and if so, make +the change. + +=cut +sub diff { + my $self = shift; + + return 1; +} + =head2 get_diff_text =cut @@ -392,8 +409,8 @@ sub new_switch { warn "Switch $node->{name} registered with FWDCTL."; $self->{logger}->info("Switch $node->{name} registered with FWDCTL."); - # Make first invocation of polling subroutines here: - # See OESS::NSO::Discovery::start for examples + # Make first invocation of polling subroutines + $self->diff; return &$success({ status => 1 }); } From 60435ed57392606646a57552c147a3fda0a4046b Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Tue, 29 Dec 2020 14:25:29 -0500 Subject: [PATCH 018/156] update discovery to get all port types --- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index baec8a1dc..da0ba990b 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -148,17 +148,20 @@ sub interface_handler { my $ports = eval { my $result = []; - - my @gb_ports = $dom->findnodes('/cisco-ios-xr:interface/cisco-ios-xr:GigabitEthernet'); - foreach my $port (@gb_ports) { - my $port_info = { - admin_state => $port->exists('./cisco-ios-xr:shutdown') ? 'down' : 'up', - bandwidth => $port->findvalue('./cisco-ios-xr:speed') || 1000, - description => $port->findvalue('./cisco-ios-xr:description') || '', - mtu => $port->findvalue('./cisco-ios-xr:mtu') || 0, - name => $port->findvalue('./cisco-ios-xr:id') - }; - push @$result, $port_info; + my $types = ["GigabitEthernet", "TenGigE", "FortyGigE", "HundredGigE", "FourHundredGigE"]; + + foreach my $type (@$types) { + my @gb_ports = $dom->findnodes("/cisco-ios-xr:interface/cisco-ios-xr:$type"); + foreach my $port (@gb_ports) { + my $port_info = { + admin_state => $port->exists('./cisco-ios-xr:shutdown') ? 'down' : 'up', + bandwidth => $port->findvalue('./cisco-ios-xr:speed') || 1000, + description => $port->findvalue('./cisco-ios-xr:description') || '', + mtu => $port->findvalue('./cisco-ios-xr:mtu') || 0, + name => $type . $port->findvalue('./cisco-ios-xr:id') + }; + push @$result, $port_info; + } } return $result; }; @@ -168,8 +171,6 @@ sub interface_handler { next; } - # TODO save nso queried data into db - warn 'ports: ' . Dumper($ports); $self->{db}->start_transaction; foreach my $data (@$ports) { my $port = new OESS::Interface(db => $self->{db}, node => $node->{name}, name => $data->{name}); From dc59f24c20e2abdaadd4aa2929fbcc5d2cec978b Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Wed, 30 Dec 2020 11:02:08 -0500 Subject: [PATCH 019/156] add initial calls to nso rest api --- perl-lib/OESS/lib/OESS/NSO/Client.pm | 141 +++++++++++++++++++++++++++ perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 29 ++++-- 2 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 perl-lib/OESS/lib/OESS/NSO/Client.pm diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm new file mode 100644 index 000000000..2f2bdd258 --- /dev/null +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -0,0 +1,141 @@ +package OESS::NSO::Client; + +use Data::Dumper; +use HTTP::Request::Common; +use JSON; +use Log::Log4perl; +use LWP::UserAgent; +use XML::LibXML; + +=head1 OESS::NSO::Client + +OESS::NSO::Client provides a perl interface to the NSO web api. + +=cut + +=head2 new + +=cut +sub new { + my $class = shift; + my $args = { + config => undef, + config_filename => '/etc/oess/database.xml', + logger => Log::Log4perl->get_logger('OESS.NSO.Client'), + @_ + }; + my $self = bless $args, $class; + + if (!defined $self->{config}) { + $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + } + + $self->{www} = new LWP::UserAgent; + my $host = $self->{config}->nso_host; + $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string + $self->{www}->credentials($host, "restconf", $self->{config}->nso_username, $self->{config}->nso_password); + + return $self; +} + +=head2 create_l2connection + + my $err = create_l2connection($l2connection); + +=cut +sub create_l2connection { + my $self = shift; + my $conn = shift; # OESS::L2Circuit + warn Dumper($conn->to_hash); + + my $eps = []; + foreach my $ep (@{$conn->endpoints}) { + my $obj = { + endpoint_id => $ep->circuit_ep_id, + bandwidth => $ep->bandwidth, + device => $ep->node, + interface => $ep->interface, + tag => $ep->tag + }; + if (defined $ep->inner_tag) { + $obj->{inner_tag} = $ep->inner_tag; + } + push(@$eps, $obj); + } + + my $payload = { + "internet2-l2connection:internet2-l2connection" => [ + { + "connection_id" => $conn->circuit_id, + "endpoint" => $eps + } + ] + }; + + eval { + my $res = $self->{www}->post( + $self->{config}->nso_host . "/restconf/data/", + 'Content-type' => 'application/yang-data+json', + 'Content' => encode_json($payload) + ); + return if ($res->content eq ''); # Empty payload indicates success + + my $result = decode_json($res->content); + die $self->get_json_errors($result->{errors}) if (defined $result->{errors}); + }; + if ($@) { + my $err = $@; + warn $err; + return $err; + } + return; +} + +=head2 delete_l2connection + + my $err = delete_l2connection($l2connection_id); + +=cut +sub delete_l2connection { + my $self = shift; + my $conn_id = shift; # OESS::L2Circuit->circuit_id + + eval { + my $res = $self->{www}->delete( + $self->{config}->nso_host . "/restconf/data/internet2-l2connection:internet2-l2connection=$conn_id", + 'Content-type' => 'application/yang-data+json' + ); + return if ($res->content eq ''); # Empty payload indicates success + + my $result = decode_json($res->content); + die $self->get_json_errors($result->{errors}) if (defined $result->{errors}); + }; + if ($@) { + my $err = $@; + warn $err; + return $err; + } + return; +} + + +=head2 get_json_errors + +get_json_errors is a helper method to extract errors returned from nso's rest +api. + +=cut +sub get_json_errors { + my $self = shift; + my $errs = shift; + + my $errors = []; + foreach my $err (@{$errs->{error}}) { + push(@$errors, $err->{'error-message'}); + } + + my $r = join(". ", @$errors); + return $r; +} + +1; diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index e27a2fcab..902e2ed97 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -13,8 +13,11 @@ use XML::LibXML; use OESS::Config; use OESS::DB; use OESS::DB::Node; +use OESS::L2Circuit; use OESS::Node; +use OESS::NSO::Client; use OESS::RabbitMQ::Dispatcher; +use OESS::VRF; use constant FWDCTL_WAITING => 2; use constant FWDCTL_SUCCESS => 1; @@ -44,11 +47,7 @@ sub new { } $self->{db} = new OESS::DB(config => $self->{config}->filename); $self->{nodes} = {}; - - $self->{www} = new LWP::UserAgent; - my $host = $self->{config}->nso_host; - $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string - $self->{www}->credentials($host, "restconf", $self->{config}->nso_username, $self->{config}->nso_password); + $self->{nso} = new OESS::NSO::Client(config => $self->{config}); # When this process receives sigterm send an event to notify all # children to exit cleanly. @@ -282,7 +281,18 @@ sub addVlan { my $success = $method->{success_callback}; my $error = $method->{error_callback}; - return &$success({ status => 1 }); + my $conn = new OESS::L2Circuit( + db => $self->{db}, + circuit_id => $params->{circuit_id}{value} + ); + $conn->load_endpoints; + + my $err = $self->{nso}->create_l2connection($conn); + if (defined $err) { + $self->{logger}->error($err); + return &$error($err); + } + return &$success({ status => FWDCTL_SUCCESS }); } =head2 deleteVlan @@ -296,7 +306,12 @@ sub deleteVlan { my $success = $method->{success_callback}; my $error = $method->{error_callback}; - return &$success({ status => 1 }); + my $err = $self->{nso}->delete_l2connection($params->{circuit_id}{value}); + if (defined $err) { + $self->{logger}->error($err); + return &$error($err); + } + return &$success({ status => FWDCTL_SUCCESS }); } =head2 modifyVlan From 597c0ce970c177c5d1906eaf59490d30174fed95 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Wed, 30 Dec 2020 14:04:03 -0500 Subject: [PATCH 020/156] Add initial edit_l2connection logic L2Circuit->_process_circuit_details seemed to rely too much on the db. It also loaded Entities in place of its Endpoints for some reason. I've updated the logic to load an Endpoint object to be more like VRF->from_hash. --- perl-lib/OESS/lib/OESS/L2Circuit.pm | 43 +++++++++++----------- perl-lib/OESS/lib/OESS/NSO/Client.pm | 53 ++++++++++++++++++++++++++++ perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 9 +++++ 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/L2Circuit.pm b/perl-lib/OESS/lib/OESS/L2Circuit.pm index 75cfa54b5..8d3d7df8e 100644 --- a/perl-lib/OESS/lib/OESS/L2Circuit.pm +++ b/perl-lib/OESS/lib/OESS/L2Circuit.pm @@ -517,32 +517,33 @@ sub _process_circuit_details{ $self->{'has_tertiary_path'} = (defined $hash->{'tertiary_links'} && @{$hash->{'tertiary_links'}} > 0) ? 1 : 0; # TODO Load endpoints - - foreach my $endpoint (@{$hash->{'endpoints'}}){ - if (!defined $self->{endpoints}) { - $self->{endpoints} = []; + if (defined $hash->{endpoints}) { + $self->{endpoints} = []; + foreach my $ep (@{$hash->{endpoints}}) { + push(@{$self->{endpoints}}, new OESS::Endpoint(db => $self->{db}, model => $ep)); } + } - if ($endpoint->{'local'} == 0) { - $self->{'interdomain'} = 1; - } + # foreach my $endpoint (@{$hash->{'endpoints'}}){ + # if (!defined $self->{endpoints}) { + # $self->{endpoints} = []; + # } - my $entity = OESS::Entity->new( - db => $self->{'db'}, - interface_id => $endpoint->{'interface_id'}, - vlan => $endpoint->{'tag'} - ); - if (!defined $entity) { - next; - } + # if ($endpoint->{'local'} == 0) { + # $self->{'interdomain'} = 1; + # } - push @{$self->{endpoints}}, $entity; - # $endpoint->{'entity'} = $entity->to_hash(); - } + # my $entity = OESS::Entity->new( + # db => $self->{'db'}, + # interface_id => $endpoint->{'interface_id'}, + # vlan => $endpoint->{'tag'} + # ); + # if (!defined $entity) { + # next; + # } - # warn Dumper($self); - # if (!$self->{'just_display'}) { - # $self->_create_graph(); + # push @{$self->{endpoints}}, $entity; + # # $endpoint->{'entity'} = $entity->to_hash(); # } } diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 2f2bdd258..8d5a38b07 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -118,6 +118,59 @@ sub delete_l2connection { return; } +=head2 edit_l2connection + + my $err = edit_l2connection($l2connection); + +=cut +sub edit_l2connection { + my $self = shift; + my $conn = shift; # OESS::L2Circuit + warn Dumper($conn->to_hash); + + my $eps = []; + foreach my $ep (@{$conn->endpoints}) { + my $obj = { + endpoint_id => $ep->circuit_ep_id, + bandwidth => $ep->bandwidth, + device => $ep->node, + interface => $ep->interface, + tag => $ep->tag + }; + if (defined $ep->inner_tag) { + $obj->{inner_tag} = $ep->inner_tag; + } + push(@$eps, $obj); + } + + my $conn_id = $conn->circuit_id; + my $payload = { + "internet2-l2connection:internet2-l2connection" => [ + { + "connection_id" => $conn_id, + "endpoint" => $eps + } + ] + }; + + eval { + my $res = $self->{www}->put( + $self->{config}->nso_host . "/restconf/data/internet2-l2connection:internet2-l2connection=$conn_id", + 'Content-type' => 'application/yang-data+json', + 'Content' => encode_json($payload) + ); + return if ($res->content eq ''); # Empty payload indicates success + + my $result = decode_json($res->content); + die $self->get_json_errors($result->{errors}) if (defined $result->{errors}); + }; + if ($@) { + my $err = $@; + warn $err; + return $err; + } + return; +} =head2 get_json_errors diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 902e2ed97..7a45c1684 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -325,6 +325,15 @@ sub modifyVlan { my $success = $method->{success_callback}; my $error = $method->{error_callback}; + my $pending_hash = decode($params->{pending}{value}); + my $pending_conn = new OESS::L2Circuit(db => $self->{db}, model => $pending_hash); + + my $err = $self->{nso}->edit_l2connection($pending_conn); + if (defined $err) { + $self->{logger}->error($err); + return &$error($err); + } + return &$success({ status => 1 }); } From 8795fdc758aae188e716ee44dd9bf418b2e1d9b6 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 31 Dec 2020 11:28:26 -0500 Subject: [PATCH 021/156] update web api calls for fresh installs --- frontend/www/js_templates/api/entity.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/www/js_templates/api/entity.js b/frontend/www/js_templates/api/entity.js index 2d89d355b..2bbfaf05f 100644 --- a/frontend/www/js_templates/api/entity.js +++ b/frontend/www/js_templates/api/entity.js @@ -59,7 +59,7 @@ async function getEntitiesAll(workgroupID, queryString=null) { */ async function edit_entity(entityID, entity_name, description, logo_url, entity_url){ - let url = `[% path %]services/entity.cgi?action=update_entity&entity_id=${entityID}`; + let url = `[% path %]services/entity.cgi?method=update_entity&entity_id=${entityID}`; if (entity_name.value) { url += `&name=${entity_name.value}`; @@ -101,7 +101,7 @@ async function edit_entity(entityID, entity_name, description, logo_url, entity_ async function add_entity(entityID, entity_name, desctiption, logo_url, entity_url){ let user = await getCurrentUser(); - let url = `[% path %]services/entity.cgi?action=add_child_entity¤t_entity_id=${entityID}&user_id=${user.user_id}`; + let url = `[% path %]services/entity.cgi?method=add_child_entity¤t_entity_id=${entityID}&user_id=${user.user_id}`; if (entity_name.value) { url += `&name=${entity_name.value}`; @@ -137,7 +137,7 @@ async function add_entity(entityID, entity_name, desctiption, logo_url, entity_u * @params {integer} entityID - Identifier of the currnt entity */ async function add_user(user_id, entityID){ - const url = `[% path %]services/entity.cgi?action=add_user&entity_id=${entityID}&user_id=${user_id}`; + const url = `[% path %]services/entity.cgi?method=add_user&entity_id=${entityID}&user_id=${user_id}`; try { const resp = await fetch(url, {method: 'get', credentials: 'include'}); const data = await resp.json(); @@ -157,7 +157,7 @@ async function add_user(user_id, entityID){ */ async function remove_user(user_id, entityID){ console.log("remove user"); - const url = `[% path %]services/entity.cgi?action=remove_user&entity_id=${entityID}&user_id=${user_id}`; + const url = `[% path %]services/entity.cgi?method=remove_user&entity_id=${entityID}&user_id=${user_id}`; try { const resp = await fetch(url, {method: 'get', credentials: 'include'}); const data = await resp.json(); From e35121530b4428778aa45143fc0e2ecdc66a4d22 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 31 Dec 2020 13:45:03 -0500 Subject: [PATCH 022/156] Fix interface form under edit endpoint modal This interface form only allows for modification of the endpoint's details: VLAN, bandwidth, etc. The interface itself may not be changed at this time. --- .../l2/endpoint_selection_modal.js | 38 ++++++++++--------- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 2 +- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/frontend/www/js_templates/l2/endpoint_selection_modal.js b/frontend/www/js_templates/l2/endpoint_selection_modal.js index 974c82a1e..67aec60db 100644 --- a/frontend/www/js_templates/l2/endpoint_selection_modal.js +++ b/frontend/www/js_templates/l2/endpoint_selection_modal.js @@ -3,6 +3,7 @@ class EndpointSelectionModal2 { let template = document.querySelector('#endpoint-selection-modal2'); this.element = document.importNode(template.content, true); + this.endpoint = null; this.searchTimeout = null; this.parent = document.querySelector(query); @@ -10,6 +11,8 @@ class EndpointSelectionModal2 { } display(endpoint) { + this.endpoint = endpoint; + if (endpoint !== undefined && endpoint !== null && endpoint.interface === 'TBD') { this.populateEntityForm(endpoint); } else { @@ -174,23 +177,24 @@ class EndpointSelectionModal2 { interconnectType = null; } - let endpoint = { - index: index, - bandwidth: this.parent.querySelector('.endpoint-bandwidth').value, - interface: interfaceSelector.options[interfaceSelector.selectedIndex].dataset.name, - interface_id: interfaceSelector.options[interfaceSelector.selectedIndex].value, - description: interfaceSelector.options[interfaceSelector.selectedIndex].dataset.description, - node: interfaceSelector.options[interfaceSelector.selectedIndex].dataset.node, - entity: null, // entity, - entity_id: null, // entity_id, - peerings: [], - cloud_account_id: '', - tag: vlanSelector.options[vlanSelector.selectedIndex].value, - jumbo: this.parent.querySelector('.endpoint-jumbo-frames').checked, - cloud_interconnect_type: interconnectType - }; - - state.updateEndpoint(endpoint); + if (this.endpoint === null) { + this.endpoint = { index: index }; + } + this.endpoint.bandwidth = this.parent.querySelector('.endpoint-bandwidth').value; + this.endpoint.name = interfaceSelector.options[interfaceSelector.selectedIndex].dataset.name; + this.endpoint.interface = interfaceSelector.options[interfaceSelector.selectedIndex].dataset.name; + this.endpoint.interface_id = interfaceSelector.options[interfaceSelector.selectedIndex].value; + this.endpoint.description = interfaceSelector.options[interfaceSelector.selectedIndex].dataset.description; + this.endpoint.node = interfaceSelector.options[interfaceSelector.selectedIndex].dataset.node; + this.endpoint.entity = null; + this.endpoint.entity_id = null; + this.endpoint.peerings = []; + this.endpoint.cloud_account_id = ''; + this.endpoint.tag = vlanSelector.options[vlanSelector.selectedIndex].value; + this.endpoint.jumbo = this.parent.querySelector('.endpoint-jumbo-frames').checked; + this.endpoint.cloud_interconnect_type = interconnectType; + + state.updateEndpoint(this.endpoint); $('#add-endpoint-modal2').modal('hide'); }.bind(this); diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 7a45c1684..1a8f8d19a 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -325,7 +325,7 @@ sub modifyVlan { my $success = $method->{success_callback}; my $error = $method->{error_callback}; - my $pending_hash = decode($params->{pending}{value}); + my $pending_hash = decode_json($params->{pending}{value}); my $pending_conn = new OESS::L2Circuit(db => $self->{db}, model => $pending_hash); my $err = $self->{nso}->edit_l2connection($pending_conn); From 2c082fa08b386a7bb93d8d3685a8c8e5f8083d19 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Wed, 6 Jan 2021 14:02:22 -0500 Subject: [PATCH 023/156] add dpid to update func as req by db schema --- perl-lib/OESS/lib/OESS/DB/Node.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/DB/Node.pm b/perl-lib/OESS/lib/OESS/DB/Node.pm index f999c7315..b52b6d9d3 100644 --- a/perl-lib/OESS/lib/OESS/DB/Node.pm +++ b/perl-lib/OESS/lib/OESS/DB/Node.pm @@ -292,8 +292,8 @@ sub update { INSERT INTO node_instantiation ( admin_state, vendor, model, sw_version, mgmt_addr, loopback_address, tcp_port, node_id, openflow, mpls, - start_epoch, end_epoch - ) VALUES (?,?,?,?,?,?,?,?,?,?,UNIX_TIMESTAMP(NOW()),-1) + dpid, start_epoch, end_epoch + ) VALUES (?,?,?,?,?,?,?,?,?,?,?,UNIX_TIMESTAMP(NOW()),-1) ", [ $node->{admin_state}, @@ -305,7 +305,8 @@ sub update { $node->{tcp_port}, $args->{node}->{node_id}, $args->{node}->{openflow} || 0, - $args->{node}->{mpls} || 1 + $args->{node}->{mpls} || 1, + $node->{dpid} ] ); if (!defined $inst_ok) { From 2f9e9514760b89509ba8a68d6f126bcae2692321 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 8 Jan 2021 13:40:34 -0500 Subject: [PATCH 024/156] add initial diff method to align nso with oess db --- perl-lib/OESS/lib/OESS/NSO/Client.pm | 52 +++++- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 264 ++++++++++++++++++++++++++- 2 files changed, 312 insertions(+), 4 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 8d5a38b07..98cfd07c2 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -46,7 +46,6 @@ sub new { sub create_l2connection { my $self = shift; my $conn = shift; # OESS::L2Circuit - warn Dumper($conn->to_hash); my $eps = []; foreach my $ep (@{$conn->endpoints}) { @@ -172,11 +171,62 @@ sub edit_l2connection { return; } +=head2 get_l2connections + + my ($connections, $err) = get_l2connections(); + +=cut +sub get_l2connections { + my $self = shift; + + my $connections; + eval { + my $res = $self->{www}->get( + $self->{config}->nso_host . "/restconf/data/internet2-l2connection:internet2-l2connection", + 'Content-type' => 'application/yang-data+json' + ); + if ($res->content eq '') { # Empty payload indicates success + $connections = []; + } else { + my $result = decode_json($res->content); + die $self->get_json_errors($result->{errors}) if (defined $result->{errors}); + $connections = $result->{"internet2-l2connection:internet2-l2connection"}; + } + }; + if ($@) { + my $err = $@; + warn $err; + return (undef, $err); + } + return ($connections, undef); +} + =head2 get_json_errors get_json_errors is a helper method to extract errors returned from nso's rest api. +Response code: +- 201 on success +- 204 on success no content +- 400 on error +- 409 on error conflict + +Response body: + + { + "errors": { + "error": [ + { + "error-message": "object already exists: /internet2-l2connection:internet2-l2connection[internet2-l2connection:connection_id='124']", + "error-path": "", + "error-tag": "data-exists", + "error-type": "application" + } + ] + } + } + =cut sub get_json_errors { my $self = shift; diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 1a8f8d19a..541b55fbe 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -45,6 +45,7 @@ sub new { if (!defined $self->{config}) { $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); } + $self->{cache} = {}; $self->{db} = new OESS::DB(config => $self->{config}->filename); $self->{nodes} = {}; $self->{nso} = new OESS::NSO::Client(config => $self->{config}); @@ -60,6 +61,9 @@ sub new { =head2 start +start configures polling timers, loads in-memory cache of l2 and l3 +connections, and sets up a rabbitmq dispatcher for RCP calls into FWDCTL. + =cut sub start { my $self = shift; @@ -74,16 +78,16 @@ sub start { $self->{nodes}->{$node->{node_id}} = $node; } + $self->_update_cache; + # Setup polling subroutines $self->{connection_timer} = AnyEvent->timer( after => 5, - interval => 300, + interval => 30, cb => sub { $self->diff(@_); } ); $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( - # queue => 'oess-fwdctl', - # topic => 'oess.fwdctl.rpc' queue => 'MPLS-FWDCTL', topic => 'MPLS.FWDCTL.RPC' ); @@ -292,6 +296,8 @@ sub addVlan { $self->{logger}->error($err); return &$error($err); } + + $self->_add_connection_to_cache($conn); return &$success({ status => FWDCTL_SUCCESS }); } @@ -306,11 +312,19 @@ sub deleteVlan { my $success = $method->{success_callback}; my $error = $method->{error_callback}; + my $conn = new OESS::L2Circuit( + db => $self->{db}, + circuit_id => $params->{circuit_id}{value} + ); + $conn->load_endpoints; + my $err = $self->{nso}->delete_l2connection($params->{circuit_id}{value}); if (defined $err) { $self->{logger}->error($err); return &$error($err); } + + $self->_del_connection_from_cache($conn); return &$success({ status => FWDCTL_SUCCESS }); } @@ -334,6 +348,7 @@ sub modifyVlan { return &$error($err); } + $self->_add_connection_to_cache($conn); return &$success({ status => 1 }); } @@ -385,10 +400,69 @@ diff reads all connections from cache, loads all connections from nso, determines if a configuration change within nso is required, and if so, make the change. +In the case of a large change (effects > N connections), the diff is put +into a pending state. Diff states are tracked on a per-node basis. + =cut sub diff { my $self = shift; + my ($connections, $err) = $self->{nso}->get_l2connections(); + if (defined $err) { + $self->{logger}->error($err); + return; + } + + # After a connection has been sync'd to NSO we remove it from our hash of + # nso connections. Any connections left in this hash after syncing are not + # known by OESS and should be removed. + my $nso_l2connections = {}; + foreach my $conn (@{$connections}) { + $nso_l2connections->{$conn->{connection_id}} = $conn; + } + + # Connections are stored in-memory multiple times under each node they're + # associed with. Keep a record of connections as they're sync'd to prevent a + # connection from being sync'd more than once. + my $syncd_connections = {}; + + foreach my $node_id (keys %{$self->{cache}}) { + foreach my $conn_id (keys %{$self->{cache}->{$node_id}}) { + + # Skip connections if they're already sync'd. + next if defined $syncd_connections->{$conn_id}; + $syncd_connections->{$conn_id} = 1; + + # Compare cached connection against NSO connection. If no difference + # continue with next connection, otherwise update NSO to align with + # cache. + my $conn = $self->{cache}->{$node_id}->{$conn_id}; + if (!$self->_nso_connection_equal_to_cached($conn, $nso_l2connections->{$conn_id})) { + my $err = $self->{nso}->edit_l2connection($conn); + if (defined $err) { + $self->{logger}->error($err); + warn $err; + } + } + + delete $nso_l2connections->{$conn_id}; + } + } + + foreach my $conn_id (keys %{$nso_l2connections}) { + my $err = $self->{nso}->delete_l2connection($conn_id); + if (defined $err) { + $self->{logger}->error($err); + } + } + + # TODO Queue up all required changes into an array. If size greater than + # auto-diff cutoff set diff to pending. Changes should be tracked on a per + # device basis. + # Ex. [ {type => 'add-l2', value => OESS::L2Circuit } ] + + warn 'Syncd Connections: ' . Dumper($syncd_connections); + return 1; } @@ -441,6 +515,8 @@ sub new_switch { =head2 update_cache +update_cache is a rabbitmq proxy method to _update_cache. + =cut sub update_cache { my $self = shift; @@ -450,7 +526,189 @@ sub update_cache { my $success = $method->{success_callback}; my $error = $method->{error_callback}; + my $err = $self->_update_cache; + if (defined $err) { + $self->{logger}->error($err); + return &$error($err); + } return &$success({ status => 1 }); } +=head2 _update_cache + +_update_cache reads all connections from the database and loads them into an +in-memory cache. + +In memory connection cache: +{ + "node-id-1": { + "conn-id-1": { "eps" : [ "node-id-1", "node-id-2" ] } + }, + "node-id-2": { + "conn-id-1": { "eps" : [ "node-id-1", "node-id-2" ] } + } +} + +This implies that a connection is stored under each node. This allows us to +query all connections associated with a single node. Additionally this helps us +track large changes that may effect multiple connections on a single node. + +=cut +sub _update_cache { + my $self = shift; + + my $l2connections = OESS::DB::Circuit::fetch_circuits( + db => $self->{db} + ); + if (!defined $l2connections) { + return "Couldn't load l2connections in update_cache."; + } + + foreach my $conn (@$l2connections) { + my $conn_obj = new OESS::L2Circuit(db => $self->{db}, model => $conn); + $conn_obj->load_endpoints; + $self->_add_connection_to_cache($conn_obj); + } + + return; +} + +=head2 _del_connection_from_cache + +_del_connection_in_cache is a simple helper to correctly remove a connection +object from memory. + +=cut +sub _del_connection_from_cache { + my $self = shift; + my $conn = shift; + + foreach my $ep (@{$conn->endpoints}) { + if (!defined $self->{cache}->{$ep->node_id}) { + next; + } + delete $self->{cache}->{$ep->node_id}->{$conn->circuit_id}; + } + + return 1; +} + +=head2 _add_connection_to_cache + +_add_connection_to_cache is a simple helper to correctly place a connection +object into memory. + +=cut +sub _add_connection_to_cache { + my $self = shift; + my $conn = shift; + + foreach my $ep (@{$conn->endpoints}) { + if (!defined $self->{cache}->{$ep->node_id}) { + $self->{cache}->{$ep->node_id} = {}; + } + $self->{cache}->{$ep->node_id}->{$conn->circuit_id} = $conn; + } + + return 1; +} + +=head2 _nso_connection_equal_to_cached + +_nso_connection_equal_to_cached compares the NSO provided data structure against +the cached connection object. If there is no difference return 1, otherwise +return 0. + +NSO L2Connection: + + { + 'connection_id' => 3000, + 'directly-modified' => { + 'services' => [ + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'0\'][sdp:name=\'3000\']', + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'1\'][sdp:name=\'3000\']' + ], + 'devices' => [ + 'xr0' + ] + }, + 'endpoint' => [ + { + 'bandwidth' => 0, + 'endpoint_id' => 1, + 'interface' => 'GigabitEthernet0/0', + 'tag' => 1, + 'device' => 'xr0' + }, + { + 'bandwidth' => 0, + 'endpoint_id' => 2, + 'interface' => 'GigabitEthernet0/1', + 'tag' => 1, + 'device' => 'xr0' + } + ], + 'device-list' => [ + 'xr0' + ], + 'modified' => { + 'services' => [ + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'1\'][sdp:name=\'3000\']', + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'0\'][sdp:name=\'3000\']' + ], + 'devices' => [ + 'xr0' + ] + } + } + +=cut +sub _nso_connection_equal_to_cached { + my $self = shift; + my $conn = shift; + my $nsoc = shift; # NSOConnection + + my $conn_ep_count = @{$conn->endpoints}; + my $nsoc_ep_count = @{$nsoc->{endpoint}}; + if (@{$conn->endpoints} != @{$nsoc->{endpoint}}) { + warn "ep count wrong"; + return 0; + } + + my $ep_index = {}; + foreach my $ep (@{$conn->endpoints}) { + if (!defined $ep_index->{$ep->node}) { + $ep_index->{$ep->node} = {}; + } + $ep_index->{$ep->node}->{$ep->interface} = $ep; + } + + foreach my $ep (@{$nsoc->{endpoint}}) { + if (!defined $ep_index->{$ep->{device}}->{$ep->{interface}}) { + warn "ep not in cache"; + return 0; + } + my $ref_ep = $ep_index->{$ep->{device}}->{$ep->{interface}}; + + warn "band" if $ep->{bandwidth} != $ref_ep->bandwidth; + warn "tag" if $ep->{tag} != $ref_ep->tag; + warn "inner_tag" if $ep->{inner_tag} != $ref_ep->inner_tag; + + # Compare endpoints + return 0 if $ep->{bandwidth} != $ref_ep->bandwidth; + return 0 if $ep->{tag} != $ref_ep->tag; + return 0 if $ep->{inner_tag} != $ref_ep->inner_tag; + + delete $ep_index->{$ep->{device}}->{$ep->{interface}}; + } + + foreach my $key (keys %{$ep_index}) { + my @leftovers = keys %{$ep_index->{$key}}; + warn "leftover eps: ".Dumper(@leftovers) if @leftovers > 0; + return 0 if @leftovers > 0; + } + + return 1; +} + 1; From 03321a3a26d6150ad80cbccf06d622381173f189 Mon Sep 17 00:00:00 2001 From: Andrew Ragusa Date: Tue, 12 Jan 2021 16:32:19 +0000 Subject: [PATCH 025/156] OESS# 1228 allowing 4x overprovisioning on Azure interfaces --- perl-lib/OESS/lib/OESS/Interface.pm | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/lib/OESS/Interface.pm b/perl-lib/OESS/lib/OESS/Interface.pm index 467e33c75..ba580282f 100644 --- a/perl-lib/OESS/lib/OESS/Interface.pm +++ b/perl-lib/OESS/lib/OESS/Interface.pm @@ -201,7 +201,12 @@ sub bandwidth{ if (defined $bandwidth) { $self->{bandwidth} = $bandwidth; } - return $self->{'bandwidth'}; + + if($self->{'cloud_interconnect_type'} eq 'azure-express-route'){ + return $self->{'bandwidth'} * 4; + }else{ + return $self->{'bandwidth'}; + } } =head2 mtu From 31049a35c5b0c2eaf3aa11183f8a257bd7407813 Mon Sep 17 00:00:00 2001 From: Andrew Ragusa Date: Tue, 12 Jan 2021 17:14:22 +0000 Subject: [PATCH 026/156] fixing the unit test --- perl-lib/OESS/t/z-Object/BandwidthValidator.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl-lib/OESS/t/z-Object/BandwidthValidator.t b/perl-lib/OESS/t/z-Object/BandwidthValidator.t index 61f832c7d..a761b6e5b 100644 --- a/perl-lib/OESS/t/z-Object/BandwidthValidator.t +++ b/perl-lib/OESS/t/z-Object/BandwidthValidator.t @@ -68,5 +68,5 @@ $validator2->load; my $ok3 = $validator2->is_bandwidth_valid(bandwidth => 10000, is_admin => 0); ok(!$ok3, "Got an invalid bandwidth"); -my $ok4 = $validator2->is_bandwidth_valid(bandwidth => 10000, is_admin => 1); +my $ok4 = $validator2->is_bandwidth_valid(bandwidth => 40000, is_admin => 1); ok($ok4, "Got a valid bandwidth"); From cd01394f71853ab1c60654451192ab57e6bd10ae Mon Sep 17 00:00:00 2001 From: Andrew Ragusa Date: Tue, 12 Jan 2021 20:31:14 +0000 Subject: [PATCH 027/156] fixing the tests --- perl-lib/OESS/lib/OESS/Cloud/BandwidthValidator.pm | 2 +- perl-lib/OESS/t/conf/interface-speed-config.xml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/lib/OESS/Cloud/BandwidthValidator.pm b/perl-lib/OESS/lib/OESS/Cloud/BandwidthValidator.pm index aad408967..b58c9e1d0 100644 --- a/perl-lib/OESS/lib/OESS/Cloud/BandwidthValidator.pm +++ b/perl-lib/OESS/lib/OESS/Cloud/BandwidthValidator.pm @@ -82,7 +82,7 @@ sub is_bandwidth_valid { } # 1. Matched on interface interconnect type - if ($self->{interface}->bandwidth > $selector->{max_bandwidth} || $self->{interface}->bandwidth < $selector->{min_bandwidth}) { + if ($self->{interface}->{'bandwidth'} > $selector->{max_bandwidth} || $self->{interface}->{'bandwidth'} < $selector->{min_bandwidth}) { next; } # 2. Matched on interface speed diff --git a/perl-lib/OESS/t/conf/interface-speed-config.xml b/perl-lib/OESS/t/conf/interface-speed-config.xml index f7520c86d..c4c43d902 100644 --- a/perl-lib/OESS/t/conf/interface-speed-config.xml +++ b/perl-lib/OESS/t/conf/interface-speed-config.xml @@ -43,4 +43,16 @@ + + + + + + + + + + + + From 06bd1ed793f16af113ae6e903a8c697b28866e01 Mon Sep 17 00:00:00 2001 From: Andrew Ragusa Date: Tue, 12 Jan 2021 20:45:09 +0000 Subject: [PATCH 028/156] still fixing the unit tests --- perl-lib/OESS/t/z-Object/BandwidthValidator.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl-lib/OESS/t/z-Object/BandwidthValidator.t b/perl-lib/OESS/t/z-Object/BandwidthValidator.t index a761b6e5b..61f832c7d 100644 --- a/perl-lib/OESS/t/z-Object/BandwidthValidator.t +++ b/perl-lib/OESS/t/z-Object/BandwidthValidator.t @@ -68,5 +68,5 @@ $validator2->load; my $ok3 = $validator2->is_bandwidth_valid(bandwidth => 10000, is_admin => 0); ok(!$ok3, "Got an invalid bandwidth"); -my $ok4 = $validator2->is_bandwidth_valid(bandwidth => 40000, is_admin => 1); +my $ok4 = $validator2->is_bandwidth_valid(bandwidth => 10000, is_admin => 1); ok($ok4, "Got a valid bandwidth"); From 680c70dcc716b04cba22f4a4b59f4ef5e12d1ee5 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 15 Jan 2021 11:01:55 -0500 Subject: [PATCH 029/156] add initial diffing logic --- perl-lib/OESS/lib/OESS/MPLS/FWDCTL.pm | 5 +- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 348 ++++++++++++++++++++++++-- 2 files changed, 337 insertions(+), 16 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/MPLS/FWDCTL.pm b/perl-lib/OESS/lib/OESS/MPLS/FWDCTL.pm index 85ae8385e..6fecf3900 100644 --- a/perl-lib/OESS/lib/OESS/MPLS/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/MPLS/FWDCTL.pm @@ -54,6 +54,10 @@ use constant TIMEOUT => 3600; use JSON::XS; use GRNOC::WebService::Regex; +=head1 OESS::MPLS::FWDCTL + +=cut + =head2 new create a new OESS Master process @@ -61,7 +65,6 @@ create a new OESS Master process FWDCTL->new(); =cut - sub new { my $class = shift; my %params = @_; diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 541b55fbe..4973e9822 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -25,6 +25,11 @@ use constant FWDCTL_FAILURE => 0; use constant FWDCTL_UNKNOWN => 3; use constant FWDCTL_BLOCKED => 4; +use constant PENDING_DIFF_NONE => 0; +use constant PENDING_DIFF => 1; +use constant PENDING_DIFF_ERROR => 2; +use constant PENDING_DIFF_APPROVED => 3; + =head1 OESS::NSO::FWDCTL =cut @@ -46,6 +51,8 @@ sub new { $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); } $self->{cache} = {}; + $self->{flat_cache} = {}; + $self->{pending_diff} = {}; $self->{db} = new OESS::DB(config => $self->{config}->filename); $self->{nodes} = {}; $self->{nso} = new OESS::NSO::Client(config => $self->{config}); @@ -421,11 +428,21 @@ sub diff { $nso_l2connections->{$conn->{connection_id}} = $conn; } + my $network_diff = {}; + my $changes = []; + # Connections are stored in-memory multiple times under each node they're # associed with. Keep a record of connections as they're sync'd to prevent a # connection from being sync'd more than once. my $syncd_connections = {}; + # Needed to ensure diff state may be set to pending_diff_none after approval + foreach my $node_id (keys %{$self->{cache}}) { + # TODO Cleanup this hacky lookup + my $node_obj = new OESS::Node(db => $self->{db}, node_id => $node_id); + $network_diff->{$node_obj->name} = ""; + } + foreach my $node_id (keys %{$self->{cache}}) { foreach my $conn_id (keys %{$self->{cache}->{$node_id}}) { @@ -437,31 +454,114 @@ sub diff { # continue with next connection, otherwise update NSO to align with # cache. my $conn = $self->{cache}->{$node_id}->{$conn_id}; - if (!$self->_nso_connection_equal_to_cached($conn, $nso_l2connections->{$conn_id})) { - my $err = $self->{nso}->edit_l2connection($conn); - if (defined $err) { - $self->{logger}->error($err); - warn $err; - } + my $diff_required = 0; + + my $diff = $self->_nso_connection_diff($conn, $nso_l2connections->{$conn_id}); + foreach my $node (keys %$diff) { + next if $diff->{$node} eq ""; + + $diff_required = 1; + $network_diff->{$node} .= $diff->{$node}; } + push(@$changes, { type => 'edit-l2connection', value => $conn }) if $diff_required; delete $nso_l2connections->{$conn_id}; } } foreach my $conn_id (keys %{$nso_l2connections}) { - my $err = $self->{nso}->delete_l2connection($conn_id); - if (defined $err) { - $self->{logger}->error($err); + # TODO Generate conn removal diff data and add to node diffs + my $diff = $self->_nso_connection_diff(undef, $nso_l2connections->{$conn_id}); + foreach my $node (keys %$diff) { + next if $diff->{$node} eq ""; + + $diff_required = 1; + $network_diff->{$node} .= $diff->{$node}; } + + push @$changes, { type => 'delete-l2connection', value => $nso_l2connections->{$conn_id} }; } + warn 'Network diff:' . Dumper($network_diff); + + # If the database asserts there is no diff pending but memory disagrees, + # then the pending state was modified by an admin. The pending diff may now + # proceed. + foreach my $node_name (keys %$network_diff) { + my $node = new OESS::Node(db => $self->{db}, name => $node_name); + warn "Diffing $node_name."; + + if (length $network_diff->{$node_name} < 30) { + warn "Diff approved for $node_name"; + + $self->{pending_diff}->{$node_name} = PENDING_DIFF_NONE; + $node->pending_diff(PENDING_DIFF_NONE); + $node->update; + } else { + if ($self->{pending_diff}->{$node_name} == PENDING_DIFF_NONE) { + warn "Diff requires manual approval."; + + $self->{pending_diff}->{$node_name} = PENDING_DIFF; + $node->pending_diff(PENDING_DIFF); + $node->update; + } - # TODO Queue up all required changes into an array. If size greater than - # auto-diff cutoff set diff to pending. Changes should be tracked on a per - # device basis. - # Ex. [ {type => 'add-l2', value => OESS::L2Circuit } ] + if ($self->{pending_diff}->{$node_name} == PENDING_DIFF && $node->pending_diff == PENDING_DIFF_NONE) { + warn "Diff manually approved."; + $self->{pending_diff}->{$node_name} = PENDING_DIFF_APPROVED; + } + } + } + + foreach my $change (@$changes) { + if ($change->{type} eq 'edit-l2connection') { + # my $conn = $self->{flat_cache}->{$change->{value}}; + my $conn = $change->{value}; + + # If conn endpoint on node with a blocked diff skip + my $diff_approval_required = 0; + foreach my $ep (@{$conn->endpoints}) { + if ($self->{pending_diff}->{$ep->node} == PENDING_DIFF) { + $diff_approval_required = 1; + last; + } + } + if ($diff_approval_required) { + warn "Not syncing l2connection $change->{value}."; + next; + } + + my $err = $self->{nso}->edit_l2connection($conn); + if (defined $err) { + $self->{logger}->error($err); + warn $err; + } + } + elsif ($change->{type} eq 'delete-l2connection') { + my $conn = $change->{value}; + + # If conn endpoint on node with a blocked diff skip + my $diff_approval_required = 0; + foreach my $ep (@{$conn->{endpoint}}) { + if ($self->{pending_diff}->{$ep->{device}} == PENDING_DIFF) { + $diff_approval_required = 1; + last; + } + } + if ($diff_approval_required) { + warn "Not syncing l2connection $conn->{connection_id}."; + next; + } + + my $err = $self->{nso}->delete_l2connection($conn->{connection_id}); + if (defined $err) { + $self->{logger}->error($err); + } + } + else { + warn 'no idea what happened here'; + } + } - warn 'Syncd Connections: ' . Dumper($syncd_connections); return 1; } @@ -477,7 +577,84 @@ sub get_diff_text { my $success = $method->{success_callback}; my $error = $method->{error_callback}; - return &$success({ status => 1 }); + my $node_id = $params->{node_id}{value}; + my $node_name = ""; + + my ($connections, $err) = $self->{nso}->get_l2connections(); + if (defined $err) { + $self->{logger}->error($err); + return; + } + + # After a connection has been sync'd to NSO we remove it from our hash of + # nso connections. Any connections left in this hash after syncing are not + # known by OESS and should be removed. + my $nso_l2connections = {}; + foreach my $conn (@{$connections}) { + $nso_l2connections->{$conn->{connection_id}} = $conn; + } + + my $network_diff = {}; + my $changes = []; + + # Connections are stored in-memory multiple times under each node they're + # associed with. Keep a record of connections as they're sync'd to prevent a + # connection from being sync'd more than once. + my $syncd_connections = {}; + + # Needed to ensure diff state may be set to pending_diff_none after approval + foreach my $key (keys %{$self->{cache}}) { + # TODO Cleanup this hacky lookup + my $node_obj = new OESS::Node(db => $self->{db}, node_id => $key); + $network_diff->{$node_obj->name} = ""; + if ($key == $node_id) { + $node_name = $node_obj->name; + } + } + + foreach my $node_id (keys %{$self->{cache}}) { + foreach my $conn_id (keys %{$self->{cache}->{$node_id}}) { + + # Skip connections if they're already sync'd. + next if defined $syncd_connections->{$conn_id}; + $syncd_connections->{$conn_id} = 1; + + # Compare cached connection against NSO connection. If no difference + # continue with next connection, otherwise update NSO to align with + # cache. + my $conn = $self->{cache}->{$node_id}->{$conn_id}; + my $diff_required = 0; + + my $diff = $self->_nso_connection_diff($conn, $nso_l2connections->{$conn_id}); + foreach my $node (keys %$diff) { + next if $diff->{$node} eq ""; + + $diff_required = 1; + $network_diff->{$node} .= $diff->{$node}; + } + + push(@$changes, { type => 'edit-l2connection', value => $conn_id }) if $diff_required; + delete $nso_l2connections->{$conn_id}; + } + } + + foreach my $conn_id (keys %{$nso_l2connections}) { + # TODO Generate conn removal diff data and add to node diffs + my $diff = $self->_nso_connection_diff(undef, $nso_l2connections->{$conn_id}); + foreach my $node (keys %$diff) { + next if $diff->{$node} eq ""; + + $diff_required = 1; + $network_diff->{$node} .= $diff->{$node}; + } + + push @$changes, { type => 'delete-l2connection', value => $conn_id }; + } + warn 'Network diff:' . Dumper($network_diff); + warn Dumper($network_diff->{$node_name}); + + return &$success($network_diff->{$node_name}); + } =head2 new_switch @@ -589,6 +766,7 @@ sub _del_connection_from_cache { } delete $self->{cache}->{$ep->node_id}->{$conn->circuit_id}; } + delete $self->{flat_cache}->{$conn->circuit_id}; return 1; } @@ -610,6 +788,8 @@ sub _add_connection_to_cache { $self->{cache}->{$ep->node_id}->{$conn->circuit_id} = $conn; } + $self->{flat_cache}->{$conn->circuit_id} = $conn; + return 1; } @@ -711,4 +891,142 @@ sub _nso_connection_equal_to_cached { return 1; } + + + + +=head2 _nso_connection_diff + +_nso_connection_diff compares the NSO provided data structure against the cached +connection object. Returns a hash of device-name to textual representation of +the diff. + +If $conn is undef, _nso_connection_diff will generate a diff indicating that all +endpoints of $nsoc will be removed. + +NSO L2Connection: + + { + 'connection_id' => 3000, + 'directly-modified' => { + 'services' => [ + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'0\'][sdp:name=\'3000\']', + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'1\'][sdp:name=\'3000\']' + ], + 'devices' => [ + 'xr0' + ] + }, + 'endpoint' => [ + { + 'bandwidth' => 0, + 'endpoint_id' => 1, + 'interface' => 'GigabitEthernet0/0', + 'tag' => 1, + 'device' => 'xr0' + }, + { + 'bandwidth' => 0, + 'endpoint_id' => 2, + 'interface' => 'GigabitEthernet0/1', + 'tag' => 1, + 'device' => 'xr0' + } + ], + 'device-list' => [ + 'xr0' + ], + 'modified' => { + 'services' => [ + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'1\'][sdp:name=\'3000\']', + '/i2-common:internal-services/sdp:sdp-attach[sdp:sdp=\'0\'][sdp:name=\'3000\']' + ], + 'devices' => [ + 'xr0' + ] + } + } + +=cut +sub _nso_connection_diff { + my $self = shift; + my $conn = shift; + my $nsoc = shift; # NSOConnection + + my $diff = {}; + my $ep_index = {}; + + if (!defined $conn) { + foreach my $ep (@{$nsoc->{endpoint}}) { + $diff->{$ep->{device}} = "" if !defined $diff->{$ep->{device}}; + $diff->{$ep->{device}} .= "- $ep->{interface}\n"; + $diff->{$ep->{device}} .= "- Bandwidth: $ep->{bandwidth}\n"; + $diff->{$ep->{device}} .= "- Tag: $ep->{tag}\n"; + $diff->{$ep->{device}} .= "- Inner Tag: $ep->{inner_tag}\n" if defined $ep->{inner_tag}; + next; + } + return $diff; + } + + foreach my $ep (@{$conn->endpoints}) { + if (!defined $ep_index->{$ep->node}) { + $diff->{$ep->node} = ""; + $ep_index->{$ep->node} = {}; + } + $ep_index->{$ep->node}->{$ep->interface} = $ep; + } + + foreach my $ep (@{$nsoc->{endpoint}}) { + + if (!defined $ep_index->{$ep->{device}}->{$ep->{interface}}) { + $diff->{$ep->{device}} = "" if !defined $diff->{$ep->{device}}; + $diff->{$ep->{device}} .= "- $ep->{interface}\n"; + $diff->{$ep->{device}} .= "- Bandwidth: $ep->{bandwidth}\n"; + $diff->{$ep->{device}} .= "- Tag: $ep->{tag}\n"; + $diff->{$ep->{device}} .= "- Inner Tag: $ep->{inner_tag}\n" if defined $ep->{inner_tag}; + next; + } + my $ref_ep = $ep_index->{$ep->{device}}->{$ep->{interface}}; + + # Compare endpoints + my $ok = 1; + $ok = 0 if $ep->{bandwidth} != $ref_ep->bandwidth; + $ok = 0 if $ep->{tag} != $ref_ep->tag; + $ok = 0 if $ep->{inner_tag} != $ref_ep->inner_tag; + if (!$ok) { + $diff->{$ep->{device}} = "" if !defined $diff->{$ep->{device}}; + $diff->{$ep->{device}} .= " $ep->{interface}\n"; + } + + if ($ep->{bandwidth} != $ref_ep->bandwidth) { + $diff->{$ep->{device}} .= "- Bandwidth: $ep->{bandwidth}\n"; + $diff->{$ep->{device}} .= "+ Bandwidth: $ref_ep->{bandwidth}\n"; + } + if ($ep->{tag} != $ref_ep->tag) { + $diff->{$ep->{device}} .= "- Tag: $ep->{tag}\n"; + $diff->{$ep->{device}} .= "+ Tag: $ref_ep->{tag}\n"; + } + if ($ep->{inner_tag} != $ref_ep->inner_tag) { + $diff->{$ep->{device}} .= "- Inner Tag: $ep->{inner_tag}\n" if defined $ep->{inner_tag}; + $diff->{$ep->{device}} .= "+ Inner Tag: $ref_ep->{inner_tag}\n" if defined $ref_ep->{inner_tag}; + } + + delete $ep_index->{$ep->{device}}->{$ep->{interface}}; + } + + foreach my $device_key (keys %{$ep_index}) { + foreach my $ep_key (keys %{$ep_index->{$device_key}}) { + my $ep = $ep_index->{$device_key}->{$ep_key}; + $diff->{$ep->node} = "" if !defined $diff->{$ep->node}; + + $diff->{$ep->node} .= "+ $ep->{interface}\n"; + $diff->{$ep->node} .= "+ Bandwidth: $ep->{bandwidth}\n"; + $diff->{$ep->node} .= "+ Tag: $ep->{tag}\n"; + $diff->{$ep->node} .= "+ Inner Tag: $ep->{inner_tag}\n" if defined $ep->{inner_tag}; + } + } + + return $diff; +} + 1; From a56d862981c23635ec2eb5d4504ad1129f7f1133 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 15 Jan 2021 13:56:44 -0500 Subject: [PATCH 030/156] remove unused comments --- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 4973e9822..526808b3c 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -514,7 +514,6 @@ sub diff { foreach my $change (@$changes) { if ($change->{type} eq 'edit-l2connection') { - # my $conn = $self->{flat_cache}->{$change->{value}}; my $conn = $change->{value}; # If conn endpoint on node with a blocked diff skip @@ -639,7 +638,6 @@ sub get_diff_text { } foreach my $conn_id (keys %{$nso_l2connections}) { - # TODO Generate conn removal diff data and add to node diffs my $diff = $self->_nso_connection_diff(undef, $nso_l2connections->{$conn_id}); foreach my $node (keys %$diff) { next if $diff->{$node} eq ""; From 02069a47ccda3f8e7f3480977f74963a93921953 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 22 Feb 2021 15:00:00 -0500 Subject: [PATCH 031/156] update nso package name --- app/mpls/mpls_fwdctl.pl | 12 +++++++++++- perl-lib/OESS/lib/OESS/NSO/Client.pm | 14 +++++++------- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/app/mpls/mpls_fwdctl.pl b/app/mpls/mpls_fwdctl.pl index d0d859d03..636d29bcc 100755 --- a/app/mpls/mpls_fwdctl.pl +++ b/app/mpls/mpls_fwdctl.pl @@ -43,10 +43,20 @@ sub core{ my $differ = AnyEvent->timer( after => 5, interval => get_diff_interval(), cb => sub { $FWDCTL->diff() } ); AnyEvent->condvar->recv; } + elsif ($config->network_type eq 'nso+vpn-mpls') { + my $fwdctl = OESS::NSO::FWDCTL->new(config => $config); + $fwdctl->start; + + my $FWDCTL = OESS::MPLS::FWDCTL->new(); + my $reaper = AnyEvent->timer( after => 3600, interval => 3600, cb => sub { $FWDCTL->reap_old_events() } ); + my $status = AnyEvent->timer( after => 10, interval => 60, cb => sub { $FWDCTL->save_mpls_nodes_status() } ); + my $differ = AnyEvent->timer( after => 5, interval => get_diff_interval(), cb => sub { $FWDCTL->diff() } ); + AnyEvent->condvar->recv; + } else { die "Unexpected network type configured."; } - + Log::Log4perl->get_logger('OESS.MPLS.FWDCTL.APP')->info("Starting OESS.MPLS.FWDCTL event loop."); } diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 98cfd07c2..93e633210 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -63,7 +63,7 @@ sub create_l2connection { } my $payload = { - "internet2-l2connection:internet2-l2connection" => [ + "oess-l2connection:oess-l2connection" => [ { "connection_id" => $conn->circuit_id, "endpoint" => $eps @@ -101,7 +101,7 @@ sub delete_l2connection { eval { my $res = $self->{www}->delete( - $self->{config}->nso_host . "/restconf/data/internet2-l2connection:internet2-l2connection=$conn_id", + $self->{config}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection=$conn_id", 'Content-type' => 'application/yang-data+json' ); return if ($res->content eq ''); # Empty payload indicates success @@ -144,7 +144,7 @@ sub edit_l2connection { my $conn_id = $conn->circuit_id; my $payload = { - "internet2-l2connection:internet2-l2connection" => [ + "oess-l2connection:oess-l2connection" => [ { "connection_id" => $conn_id, "endpoint" => $eps @@ -154,7 +154,7 @@ sub edit_l2connection { eval { my $res = $self->{www}->put( - $self->{config}->nso_host . "/restconf/data/internet2-l2connection:internet2-l2connection=$conn_id", + $self->{config}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection=$conn_id", 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); @@ -182,7 +182,7 @@ sub get_l2connections { my $connections; eval { my $res = $self->{www}->get( - $self->{config}->nso_host . "/restconf/data/internet2-l2connection:internet2-l2connection", + $self->{config}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection", 'Content-type' => 'application/yang-data+json' ); if ($res->content eq '') { # Empty payload indicates success @@ -190,7 +190,7 @@ sub get_l2connections { } else { my $result = decode_json($res->content); die $self->get_json_errors($result->{errors}) if (defined $result->{errors}); - $connections = $result->{"internet2-l2connection:internet2-l2connection"}; + $connections = $result->{"oess-l2connection:oess-l2connection"}; } }; if ($@) { @@ -218,7 +218,7 @@ Response body: "errors": { "error": [ { - "error-message": "object already exists: /internet2-l2connection:internet2-l2connection[internet2-l2connection:connection_id='124']", + "error-message": "object already exists: /oess-l2connection:oess-l2connection[oess-l2connection:connection_id='124']", "error-path": "", "error-tag": "data-exists", "error-type": "application" diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index da0ba990b..32b06fd5b 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -148,7 +148,7 @@ sub interface_handler { my $ports = eval { my $result = []; - my $types = ["GigabitEthernet", "TenGigE", "FortyGigE", "HundredGigE", "FourHundredGigE"]; + my $types = ["Bundle-Ether", "GigabitEthernet", "TenGigE", "FortyGigE", "HundredGigE", "FourHundredGigE"]; foreach my $type (@$types) { my @gb_ports = $dom->findnodes("/cisco-ios-xr:interface/cisco-ios-xr:$type"); From db2c49b5ae843be9fe179d161177a118bd732395 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 5 Mar 2021 19:48:36 +0000 Subject: [PATCH 032/156] add nso options to node creation --- frontend/webservice/admin/admin.cgi | 12 +++- frontend/www/js_templates/admin.js | 96 ++++++++++++++++++++++------- frontend/www/js_templates/maps.js | 5 +- perl-lib/OESS/lib/OESS/Database.pm | 14 ++--- 4 files changed, 94 insertions(+), 33 deletions(-) diff --git a/frontend/webservice/admin/admin.cgi b/frontend/webservice/admin/admin.cgi index a8324af20..97342a294 100755 --- a/frontend/webservice/admin/admin.cgi +++ b/frontend/webservice/admin/admin.cgi @@ -797,17 +797,21 @@ sub register_webservice_methods { pattern => $GRNOC::WebService::Regex::TEXT, required => 1, description => '' ); - $method->add_input_parameter( name => 'vendor', + $method->add_input_parameter( name => 'southbound', pattern => $GRNOC::WebService::Regex::TEXT, required => 1, description => '' ); + $method->add_input_parameter( name => 'vendor', + pattern => $GRNOC::WebService::Regex::TEXT, + required => 0, + description => '' ); $method->add_input_parameter( name => 'model', pattern => $GRNOC::WebService::Regex::TEXT, - required => 1, + required => 0, description => '' ); $method->add_input_parameter( name => 'sw_ver', pattern => $GRNOC::WebService::Regex::TEXT, - required => 1, + required => 0, description => '' ); $svc->register_method($method); @@ -2686,6 +2690,7 @@ sub add_mpls_switch{ my $latitude = $args->{'latitude'}{'value'}; my $longitude = $args->{'longitude'}{'value'}; my $port = $args->{'port'}{'value'}; + my $southbound = $args->{'southbound'}{'value'}; my $vendor = $args->{'vendor'}{'value'}; my $model = $args->{'model'}{'value'}; my $sw_ver = $args->{'sw_ver'}{'value'}; @@ -2705,6 +2710,7 @@ sub add_mpls_switch{ lat => $latitude, long => $longitude, port => $port, + southbound => $southbound, vendor => $vendor, model => $model, sw_ver => $sw_ver); diff --git a/frontend/www/js_templates/admin.js b/frontend/www/js_templates/admin.js index 967f50ba4..b812c5b14 100644 --- a/frontend/www/js_templates/admin.js +++ b/frontend/www/js_templates/admin.js @@ -2399,6 +2399,7 @@ function setup_network_tab(){ var vendor = args[0].vendor; var model = args[0].model; var sw_version = args[0].sw_version; + var southbound = args[0].southbound; function show_interface_acl_panel(args){ var interface_id = args.interface_id; @@ -2893,6 +2894,13 @@ function setup_network_tab(){ "Short Name:" + "" + "" + + // MPLS - Software Version + "" + + "Southbound" + + "" + + "" + + "" + + "" + " " + "" + "
Interfaces
"+ @@ -2908,6 +2916,18 @@ function setup_network_tab(){ panel.render(YAHOO.util.Dom.get("active_element_details")); + $('#southbound').on('change', function() { + if ($('#southbound').val() == 'netconf') { + $('#vendor').prop('disabled', false); + $('#model').prop('disabled', false); + $('#sw_version').prop('disabled', false); + } else { + $('#vendor').prop('disabled', true); + $('#model').prop('disabled', true); + $('#sw_version').prop('disabled', true); + } + }); + var table = make_node_intf_table(); table.subscribe("rowMouseoverEvent", table.onEventHighlightRow); @@ -2972,6 +2992,13 @@ function setup_network_tab(){ YAHOO.util.Dom.get('vendor').value = vendor; YAHOO.util.Dom.get('model').value = model; YAHOO.util.Dom.get('sw_version').value = sw_version; + YAHOO.util.Dom.get('southbound').value = southbound; + + if (southbound == 'nso') { + YAHOO.util.Dom.get('vendor').disabled = true; + YAHOO.util.Dom.get('model').disabled = true; + YAHOO.util.Dom.get('sw_version').disabled = true; + } } if(default_drop == 0){ @@ -3014,6 +3041,7 @@ function setup_network_tab(){ var vendor = YAHOO.util.Dom.get('vendor').value; var model = YAHOO.util.Dom.get('model').value; var sw_version = YAHOO.util.Dom.get('sw_version').value; + var southbound = YAHOO.util.Dom.get('southbound').value; var short_name = YAHOO.util.Dom.get('short_name').value; if (! new_name){ @@ -3059,6 +3087,7 @@ function setup_network_tab(){ "&vendor=" + encodeURIComponent(vendor) + "&model=" + encodeURIComponent(model) + "&sw_version=" + encodeURIComponent(sw_version) + + "&southbound=" + encodeURIComponent(southbound) + "&short_name=" + encodeURIComponent(short_name); var openflow_args = "&default_drop=" + encodeURIComponent(new_default_drop) + @@ -3253,40 +3282,43 @@ function setup_discovery_tab(){ this.new_mpls.setBody("" + "" + "" + - "" + "" + "" + "" + - "" + "" + "" + "" + - "" + + "" + "" + - "" + + "" + "" + "" + "" + - "" + + "" + "" + "" + - "" + + "" + "" + + ""+ + "" + + "" + ""+ - "" + + "" + "" + "" + "" + - "" + + "" + "" + "" + "" + "" + - "" + + "" + "" + "
Name:" + + "" + "" + "
Short Name:" + + "" + "" + "
Latitude:Longitude:
IP Address
Port
Southbound
Vendor
Model
Software Version
" ); @@ -3295,6 +3327,18 @@ function setup_discovery_tab(){ this.new_mpls.render(document.body); + $('#southbound').on('change', function() { + if ($('#southbound').val() == 'netconf') { + $('#new_mpls_vendor').prop('disabled', false); + $('#new_mpls_model').prop('disabled', false); + $('#new_mpls_software').prop('disabled', false); + } else { + $('#new_mpls_vendor').prop('disabled', true); + $('#new_mpls_model').prop('disabled', true); + $('#new_mpls_software').prop('disabled', true); + } + }); + $('#new_mpls_vendor').on('change', function(){ var vendor = $('#new_mpls_vendor').val(); if(vendor == 'Juniper'){ @@ -3312,9 +3356,6 @@ function setup_discovery_tab(){ var add_button = new YAHOO.widget.Button("add_node", {label: "Add Switch"}); var cancel_button = new YAHOO.widget.Button("node_add_cancel", {label: "Cancel"}); - - - //do the add event! add_button.on('click', function(){ var lat = YAHOO.util.Dom.get('new_node_lat').value; @@ -3323,10 +3364,11 @@ function setup_discovery_tab(){ var short_name = YAHOO.util.Dom.get('new_node_short_name').value; var ip = YAHOO.util.Dom.get('new_ip_address').value; var port = YAHOO.util.Dom.get('new_port').value; - var vendor= YAHOO.util.Dom.get('new_mpls_vendor').value; - var model = YAHOO.util.Dom.get('new_mpls_model').value; - var sw_ver= YAHOO.util.Dom.get('new_mpls_software').value; - + var southbound = YAHOO.util.Dom.get('southbound').value; + var vendor = $('#new_mpls_vendor').prop('disabled') ? null : YAHOO.util.Dom.get('new_mpls_vendor').value; + var model = $('#new_mpls_model').prop('disabled') ? null : YAHOO.util.Dom.get('new_mpls_model').value; + var sw_ver = $('#new_mpls_software').prop('disabled') ? null : YAHOO.util.Dom.get('new_mpls_software').value; + if (! name){ alert("You must specify a name for this device."); return; @@ -3357,18 +3399,30 @@ function setup_discovery_tab(){ return; } - if(vendor == undefined || model == undefined || sw_ver == undefined){ - alert('Hardware vendor, model and software version are required'); + if (southbound === 'netconf' && (vendor === '' || model === '' || sw_ver === '')) { + alert('Hardware vendor, model, and software version are required for NETCONF controlled devices.'); return; } add_button.set("disabled", true); add_button.set("label", "Adding device...."); - var ds = new YAHOO.util.DataSource("../services/admin/admin.cgi?method=add_mpls_switch&name=" + encodeURIComponent(name) + "&short_name=" + encodeURIComponent(short_name) + "&latitude=" + encodeURIComponent(lat) + "&longitude=" + encodeURIComponent(lon) + "&ip_address=" + encodeURIComponent(ip) + "&port=" + encodeURIComponent(port) + "&vendor=" + encodeURIComponent(vendor) + "&model=" + encodeURIComponent(model) + "&sw_ver=" + encodeURIComponent(sw_ver)); - ds.responseType = YAHOO.util.DataSource.TYPE_JSON; + var url = "../services/admin/admin.cgi?method=add_mpls_switch"; + url += `&name=${encodeURIComponent(name)}`; + url += `&short_name=${encodeURIComponent(short_name)}`; + url += `&latitude=${encodeURIComponent(lat)}`; + url += `&longitude=${encodeURIComponent(lon)}`; + url += `&ip_address=${encodeURIComponent(ip)}`; + url += `&port=${encodeURIComponent(port)}`; + url += `&southbound=${encodeURIComponent(southbound)}`; + if (vendor) url += `&vendor=${encodeURIComponent(vendor)}`; + if (model) url += `&model=${encodeURIComponent(model)}`; + if (sw_ver) url += `&sw_ver=${encodeURIComponent(sw_ver)}`; + + var ds = new YAHOO.util.DataSource(url); + ds.responseType = YAHOO.util.DataSource.TYPE_JSON; - ds.responseSchema = { + ds.responseSchema = { resultsList: "results", fields: [{key: "success"}], metaFields: { diff --git a/frontend/www/js_templates/maps.js b/frontend/www/js_templates/maps.js index f18af076c..ac5b3e715 100644 --- a/frontend/www/js_templates/maps.js +++ b/frontend/www/js_templates/maps.js @@ -285,6 +285,7 @@ function NDDIMap(div_id, interdomain_mode, options){ var vendor = node_info.vendor; var model = node_info.model; var sw_version = node_info.sw_version; + var southbound = node_info.southbound; var short_name = node_info.short_name; var avail_endpoints = node_info.number_available_endpoints; var barrier_bulk = node_info.barrier_bulk; @@ -345,6 +346,7 @@ function NDDIMap(div_id, interdomain_mode, options){ point.model = model; point.short_name = short_name; point.sw_version = sw_version; + point.southbound = southbound; point.barrier_bulk = barrier_bulk; point.max_static_mac_flows = max_static_mac_flows; point.dpid = dpid; @@ -1375,12 +1377,13 @@ function NDDIMap(div_id, interdomain_mode, options){ var vendor = geo.vendor; var model = geo.model; var sw_version = geo.sw_version; + var southbound = geo.southbound; var tx_delay_ms = geo.tx_delay_ms; var short_name = geo.short_name; var barrier_bulk = geo.barrier_bulk; var max_static_mac_flows = geo.max_static_mac_flows; var dpid = geo.dpid; - self.events['clickNode'].fire({name: node, lat: lat, lon: lon, node_id: node_id, vlan_range: range,default_forward: default_forward, default_drop: default_drop,max_flows: max_flows, tx_delay_ms: tx_delay_ms, feature: e.feature, barrier_bulk: barrier_bulk, max_static_mac_flows: max_static_mac_flows, dpid: dpid, openflow: openflow, mpls: mpls, mgmt_addr: mgmt_addr, tcp_port: tcp_port, vendor: vendor, model: model,short_name: short_name, sw_version: sw_version}); + self.events['clickNode'].fire({name: node, lat: lat, lon: lon, node_id: node_id, vlan_range: range,default_forward: default_forward, default_drop: default_drop,max_flows: max_flows, tx_delay_ms: tx_delay_ms, feature: e.feature, barrier_bulk: barrier_bulk, max_static_mac_flows: max_static_mac_flows, dpid: dpid, openflow: openflow, mpls: mpls, mgmt_addr: mgmt_addr, tcp_port: tcp_port, vendor: vendor, model: model,short_name: short_name, sw_version: sw_version, southbound: southbound}); } // otherwise we're clicking on a link else{ diff --git a/perl-lib/OESS/lib/OESS/Database.pm b/perl-lib/OESS/lib/OESS/Database.pm index 9052668d3..bda7568b7 100644 --- a/perl-lib/OESS/lib/OESS/Database.pm +++ b/perl-lib/OESS/lib/OESS/Database.pm @@ -1486,6 +1486,7 @@ sub get_map_layers { node.name as node_name, node.node_id, node.short_name, + node_instantiation.southbound, node_instantiation.openflow, node_instantiation.mpls, node_instantiation.vendor, @@ -1556,6 +1557,7 @@ HERE "openflow" => $row->{'openflow'}, "mpls" => $row->{'mpls'}, "short_name" => $row->{'short_name'}, + "southbound" => $row->{'southbound'}, "vendor" => $row->{'vendor'}, "model" => $row->{'model'}, "sw_version" => $row->{'sw_version'}, @@ -7983,6 +7985,7 @@ sub get_node_by_interface_id { lat => $lat, long => $long, port => $port, + southbound => $southbound, vendor => $vendor, model => $model, sw_ver => $sw_ver); @@ -7993,13 +7996,9 @@ sub add_mpls_node{ my $self = shift; my %args = @_; - #TODO: PARAM CHECKS - - warn Data::Dumper::Dumper(%args); - $self->_start_transaction(); - my $query = "insert into node (name, short_name, latitude, longitude, operational_state_mpls,network_id) VALUES (?,?,?,?,?,?)"; + my $query = "insert into node (name, short_name, latitude, longitude, operational_state_mpls, network_id) VALUES (?,?,?,?,?,?)"; my $res = $self->_execute_query($query, [$args{'name'},$args{'short_name'},$args{'lat'},$args{'long'},'unknown',1]); warn "New Node: " . Data::Dumper::Dumper($res); @@ -8014,6 +8013,7 @@ sub add_mpls_node{ openflow => 0, mpls => 1, admin_state => 'active', + southbound => $args{'southbound'} || 'netconf', vendor => $args{'vendor'}, model => $args{'model'}, sw_version => $args{'sw_ver'}, @@ -8131,16 +8131,14 @@ sub create_node_instance{ $args{'dpid'} = unpack('N', $data); } - my $res = $self->_execute_query("insert into node_instantiation (node_id,end_epoch,start_epoch,mgmt_addr,admin_state,dpid,vendor,model,sw_version,mpls,openflow ) VALUES (?,?,?,?,?,?,?,?,?,?,?)",[$args{'node_id'},-1,time(),$args{'mgmt_addr'},$args{'admin_state'},$args{'dpid'},$args{'vendor'},$args{'model'},$args{'sw_version'},$args{'mpls'},$args{'openflow'}]); + my $res = $self->_execute_query("insert into node_instantiation (node_id,end_epoch,start_epoch,mgmt_addr,admin_state,dpid,southbound,vendor,model,sw_version,mpls,openflow ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",[$args{'node_id'},-1,time(),$args{'mgmt_addr'},$args{'admin_state'},$args{'dpid'},$args{'southbound'},$args{'vendor'},$args{'model'},$args{'sw_version'},$args{'mpls'},$args{'openflow'}]); if(!defined($res)){ $self->_set_error("Unable to create new node instantiation"); return; } - return 1; - } =head2 update_node_operational_state From 97735f77de55c011d0f2f01cf44cec7100c05f00 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 22 Mar 2021 16:11:38 -0400 Subject: [PATCH 033/156] update config arg name --- app/nso/nso_discovery.pl | 104 ++++++++++++++++++++ app/nso/nso_fwdctl.pl | 123 ++++++++++++++++++++++++ perl-lib/OESS/lib/OESS/NSO/Client.pm | 18 ++-- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 16 +-- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 10 +- 5 files changed, 249 insertions(+), 22 deletions(-) create mode 100755 app/nso/nso_discovery.pl create mode 100755 app/nso/nso_fwdctl.pl diff --git a/app/nso/nso_discovery.pl b/app/nso/nso_discovery.pl new file mode 100755 index 000000000..2f1a5f4ba --- /dev/null +++ b/app/nso/nso_discovery.pl @@ -0,0 +1,104 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use AnyEvent; +use English; +use Getopt::Long; +use Proc::Daemon; + +use OESS::Config; +use OESS::NSO::Discovery; + +my $pid_file = "/var/run/oess/nso_discovery.pid"; +my $cnf_file = "/etc/oess/database.xml"; + +sub core{ + Log::Log4perl::init_and_watch('/etc/oess/logging.conf', 10); + + my $config = new OESS::Config(config_filename => $cnf_file); + if ($config->network_type eq 'nso') { + my $discovery = new OESS::NSO::Discovery(config_obj => $config); + $discovery->start; + AnyEvent->condvar->recv; + } else { + die "Unexpected network type configured."; + } + + Log::Log4perl->get_logger('OESS.NSO.Discovery.APP')->info("Starting OESS.NSO.Discovery event loop."); +} + +sub main{ + my $is_daemon = 0; + my $verbose; + my $username; + #remove the ready file + + #--- see if the pid file exists. if not then just continue running. + if(-e $pid_file){ + #--- read the file to get the PID + my $pid = `head -n 1 $pid_file`; + chomp $pid; + + my $run_test = `ps -p $pid | grep $pid`; + + #--- if run test is empty then the pid didn't exist. If it isn't empty then the process is already running. + if($run_test){ + print "Found $0 process: $pid already running. Aborting.\n"; + exit(0); + } + else{ + print "Pid File: $pid_file already exists but it looks like process $pid is dead. Continuing startup.\n"; + } + } + + my $result = GetOptions ( + "user|u=s" => \$username, + "verbose" => \$verbose, + "daemon|d" => \$is_daemon, + ); + + #now change username/ + if (defined $username) { + my $new_uid=getpwnam($username); + my $new_gid=getgrnam($username); + $EGID=$new_gid; + $EUID=$new_uid; + } + + if ($is_daemon != 0) { + my $daemon; + if ($verbose) { + $daemon = Proc::Daemon->new( + pid_file => $pid_file, + child_STDOUT => '/var/log/oess/nso_discovery.out', + child_STDERR => '/var/log/oess/nso_discovery.log', + ); + } else { + $daemon = Proc::Daemon->new(pid_file => $pid_file); + } + + # Init returns the PID (scalar) of the daemon to the parent, or + # the PIDs (array) of the daemons created if exec_command has + # more then one program to execute. + # + # Init returns 0 to the child (daemon). + my $kid_pid = $daemon->Init; + if ($kid_pid) { + `chmod 0644 $pid_file`; # How to wait until the child process is ready. + return; + } else { + core(); + } + } + #not a daemon, just run the core; + else { + $SIG{HUP} = sub{ exit(0); }; + core(); + } +} + +main(); + +1; diff --git a/app/nso/nso_fwdctl.pl b/app/nso/nso_fwdctl.pl new file mode 100755 index 000000000..6f6fb56c2 --- /dev/null +++ b/app/nso/nso_fwdctl.pl @@ -0,0 +1,123 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use AnyEvent; +use English; +use Getopt::Long; +use Log::Log4perl; +use Proc::Daemon; +use XML::Simple; + +use OESS::Config; +use OESS::NSO::FWDCTL; + +my $pid_file = "/var/run/oess/nso_fwdctl.pid"; +my $cnf_file = "/etc/oess/database.xml"; + +sub get_diff_interval{ + eval { + my $xml = XMLin('/etc/oess/fwdctl.xml'); + my $diff_interval = $xml->{diff}->{interval}; + die unless defined $diff_interval; + return $diff_interval; + } or do { + return 900; + } +} + +sub core{ + Log::Log4perl::init_and_watch('/etc/oess/logging.conf', 10); + + my $config = new OESS::Config(config_filename => $cnf_file); + if ($config->network_type eq 'nso') { + my $fwdctl = new OESS::NSO::FWDCTL(config_obj => $config); + $fwdctl->start; + AnyEvent->condvar->recv; + } else { + die "Unexpected network type configured."; + } + Log::Log4perl->get_logger('OESS.NSO.FWDCTL.APP')->info("Starting OESS.NSO.FWDCTL event loop."); +} + +sub main{ + my $is_daemon = 0; + my $verbose; + my $username; + #remove the ready file + + # This directory is auto-removed on reboot. Create the directory if not + # already created. This is used to store connection cache files. + if (!-d "/var/run/oess/") { + `/usr/bin/mkdir /var/run/oess`; + `/usr/bin/chown _oess:_oess /var/run/oess`; + } + + #--- see if the pid file exists. if not then just continue running. + if(-e $pid_file){ + #--- read the file to get the PID + my $pid = `head -n 1 $pid_file`; + chomp $pid; + + my $run_test = `ps -p $pid | grep $pid`; + + #--- if run test is empty then the pid didn't exist. If it isn't empty then the process is already running. + if($run_test){ + print "Found $0 process: $pid already running. Aborting.\n"; + exit(0); + } + else{ + print "Pid File: $pid_file already exists but it looks like process $pid is dead. Continuing startup.\n"; + } + } + + my $result = GetOptions ( + "user|u=s" => \$username, + "verbose" => \$verbose, + "daemon|d" => \$is_daemon, + ); + + #now change username/ + if (defined $username) { + my $new_uid=getpwnam($username); + my $new_gid=getgrnam($username); + $EGID=$new_gid; + $EUID=$new_uid; + } + + if ($is_daemon != 0) { + my $daemon; + if ($verbose) { + $daemon = Proc::Daemon->new( + pid_file => $pid_file, + child_STDOUT => '/var/log/oess/nso_fwdctl.out', + child_STDERR => '/var/log/oess/nso_fwdctl.log', + ); + } else { + $daemon = Proc::Daemon->new(pid_file => $pid_file); + } + + # Init returns the PID (scalar) of the daemon to the parent, or + # the PIDs (array) of the daemons created if exec_command has + # more then one program to execute. + # + # Init returns 0 to the child (daemon). + my $kid_pid = $daemon->Init; + if ($kid_pid) { + `chmod 0644 $pid_file`; # How to wait until the child process is ready. + return; + } else { + core(); + } + } + #not a daemon, just run the core; + else { + $SIG{HUP} = sub{ exit(0); }; + core(); + } +} + +main(); + +1; diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 93e633210..ba92a4318 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -19,21 +19,21 @@ OESS::NSO::Client provides a perl interface to the NSO web api. sub new { my $class = shift; my $args = { - config => undef, + config_obj => undef, config_filename => '/etc/oess/database.xml', logger => Log::Log4perl->get_logger('OESS.NSO.Client'), @_ }; my $self = bless $args, $class; - if (!defined $self->{config}) { - $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + if (!defined $self->{config_obj}) { + $self->{config_obj} = new OESS::Config(config_filename => $self->{config_filename}); } $self->{www} = new LWP::UserAgent; - my $host = $self->{config}->nso_host; + my $host = $self->{config_obj}->nso_host; $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string - $self->{www}->credentials($host, "restconf", $self->{config}->nso_username, $self->{config}->nso_password); + $self->{www}->credentials($host, "restconf", $self->{config_obj}->nso_username, $self->{config_obj}->nso_password); return $self; } @@ -73,7 +73,7 @@ sub create_l2connection { eval { my $res = $self->{www}->post( - $self->{config}->nso_host . "/restconf/data/", + $self->{config_obj}->nso_host . "/restconf/data/", 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); @@ -101,7 +101,7 @@ sub delete_l2connection { eval { my $res = $self->{www}->delete( - $self->{config}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection=$conn_id", + $self->{config_obj}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection=$conn_id", 'Content-type' => 'application/yang-data+json' ); return if ($res->content eq ''); # Empty payload indicates success @@ -154,7 +154,7 @@ sub edit_l2connection { eval { my $res = $self->{www}->put( - $self->{config}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection=$conn_id", + $self->{config_obj}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection=$conn_id", 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); @@ -182,7 +182,7 @@ sub get_l2connections { my $connections; eval { my $res = $self->{www}->get( - $self->{config}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection", + $self->{config_obj}->nso_host . "/restconf/data/oess-l2connection:oess-l2connection", 'Content-type' => 'application/yang-data+json' ); if ($res->content eq '') { # Empty payload indicates success diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index 32b06fd5b..a8a8394d3 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -29,23 +29,23 @@ use OESS::RabbitMQ::Dispatcher; sub new { my $class = shift; my $args = { - config => undef, + config_obj => undef, config_filename => '/etc/oess/database.xml', logger => Log::Log4perl->get_logger('OESS.NSO.Discovery'), @_ }; my $self = bless $args, $class; - if (!defined $self->{config}) { - $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + if (!defined $self->{config_obj}) { + $self->{config_obj} = new OESS::Config(config_filename => $self->{config_filename}); } - $self->{db} = new OESS::DB(config => $self->{config}->filename); + $self->{db} = new OESS::DB(config => $self->{config_obj}->filename); $self->{nodes} = {}; $self->{www} = new LWP::UserAgent; - my $host = $self->{config}->nso_host; + my $host = $self->{config_obj}->nso_host; $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string - $self->{www}->credentials($host, "restconf", $self->{config}->nso_username, $self->{config}->nso_password); + $self->{www}->credentials($host, "restconf", $self->{config_obj}->nso_username, $self->{config_obj}->nso_password); # When this process receives sigterm send an event to notify all # children to exit cleanly. @@ -80,7 +80,7 @@ sub device_handler { my $node = $self->{nodes}->{$key}; my $dom = eval { - my $res = $self->{www}->get($self->{config}->nso_host . "/restconf/data/tailf-ncs:devices/device=$node->{name}"); + my $res = $self->{www}->get($self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:devices/device=$node->{name}"); return XML::LibXML->load_xml(string => $res->content); }; if ($@) { @@ -134,7 +134,7 @@ sub interface_handler { my $node = $self->{nodes}->{$key}; my $dom = eval { - my $res = $self->{www}->get($self->{config}->nso_host . "/restconf/data/tailf-ncs:devices/device=$node->{name}/config/tailf-ned-cisco-ios-xr:interface"); + my $res = $self->{www}->get($self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:devices/device=$node->{name}/config/tailf-ned-cisco-ios-xr:interface"); return XML::LibXML->load_xml(string => $res->content); }; if ($@) { diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 526808b3c..4aba34e19 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -40,22 +40,22 @@ use constant PENDING_DIFF_APPROVED => 3; sub new { my $class = shift; my $args = { - config => undef, + config_obj => undef, config_filename => '/etc/oess/database.xml', logger => Log::Log4perl->get_logger('OESS.NSO.FWDCTL'), @_ }; my $self = bless $args, $class; - if (!defined $self->{config}) { - $self->{config} = new OESS::Config(config_filename => $self->{config_filename}); + if (!defined $self->{config_obj}) { + $self->{config_obj} = new OESS::Config(config_filename => $self->{config_filename}); } $self->{cache} = {}; $self->{flat_cache} = {}; $self->{pending_diff} = {}; - $self->{db} = new OESS::DB(config => $self->{config}->filename); + $self->{db} = new OESS::DB(config => $self->{config_obj}->filename); $self->{nodes} = {}; - $self->{nso} = new OESS::NSO::Client(config => $self->{config}); + $self->{nso} = new OESS::NSO::Client(config => $self->{config_obj}); # When this process receives sigterm send an event to notify all # children to exit cleanly. From 2af1471098f0c1476f06e0d14b4eb1fbafb09623 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 22 Mar 2021 16:14:15 -0400 Subject: [PATCH 034/156] update topic names used by nso processes --- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 4 ++-- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index a8a8394d3..52a8252d7 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -289,8 +289,8 @@ sub start { $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( # queue => 'oess-discovery', # topic => 'oess.discovery.rpc' - queue => 'MPLS-Discovery', - topic => 'MPLS.Discovery.RPC' + queue => 'NSO-Discovery', + topic => 'NSO.Discovery.RPC' ); my $new_switch = new GRNOC::RabbitMQ::Method( diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 4aba34e19..1321b7c37 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -95,8 +95,8 @@ sub start { ); $self->{dispatcher} = new OESS::RabbitMQ::Dispatcher( - queue => 'MPLS-FWDCTL', - topic => 'MPLS.FWDCTL.RPC' + queue => 'NSO-FWDCTL', + topic => 'NSO.FWDCTL.RPC' ); my $add_vlan = GRNOC::RabbitMQ::Method->new( From daca34128ad4908a8be1673a0c3e9aedb962ac32 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 22 Mar 2021 20:42:43 -0400 Subject: [PATCH 035/156] add southbound selector for node creation --- frontend/webservice/admin/admin.cgi | 67 +++++++------ frontend/www/js_templates/admin.js | 40 ++++---- frontend/www/js_templates/maps.js | 22 ++--- perl-lib/OESS/lib/OESS/DB/Node.pm | 6 +- perl-lib/OESS/lib/OESS/Database.pm | 22 ++--- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 8 +- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 4 +- perl-lib/OESS/share/nddi.sql | 1 + .../OESS/share/upgrade/oess-2.0.11-2.0.12 | 97 +++++++++++++++++++ 9 files changed, 189 insertions(+), 78 deletions(-) create mode 100644 perl-lib/OESS/share/upgrade/oess-2.0.11-2.0.12 diff --git a/frontend/webservice/admin/admin.cgi b/frontend/webservice/admin/admin.cgi index 97342a294..4d0053afb 100755 --- a/frontend/webservice/admin/admin.cgi +++ b/frontend/webservice/admin/admin.cgi @@ -797,7 +797,7 @@ sub register_webservice_methods { pattern => $GRNOC::WebService::Regex::TEXT, required => 1, description => '' ); - $method->add_input_parameter( name => 'southbound', + $method->add_input_parameter( name => 'controller', pattern => $GRNOC::WebService::Regex::TEXT, required => 1, description => '' ); @@ -2690,7 +2690,7 @@ sub add_mpls_switch{ my $latitude = $args->{'latitude'}{'value'}; my $longitude = $args->{'longitude'}{'value'}; my $port = $args->{'port'}{'value'}; - my $southbound = $args->{'southbound'}{'value'}; + my $controller = $args->{'controller'}{'value'}; my $vendor = $args->{'vendor'}{'value'}; my $model = $args->{'model'}{'value'}; my $sw_ver = $args->{'sw_ver'}{'value'}; @@ -2700,43 +2700,52 @@ sub add_mpls_switch{ return; } + my $node = $db->add_mpls_node( + name => $name, + short_name => $short_name, + ip => $ip_address, + lat => $latitude, + long => $longitude, + port => $port, + controller => $controller, + vendor => $vendor, + model => $model, + sw_ver => $sw_ver + ); + if (!defined $node) { + $method->set_error($db->get_error); + return; + } + require OESS::RabbitMQ::Client; - my $mq = OESS::RabbitMQ::Client->new( topic => 'OF.FWDCTL.RPC', - timeout => 60 ); + my $mq = OESS::RabbitMQ::Client->new( + topic => 'NSO.FWDCTL.RPC', + timeout => 60 + ); + if (!defined $mq) { + $method->set_error("Internal server error occurred. Message queue connection failed."); + return; + } - my $node = $db->add_mpls_node( name => $name, - short_name => $short_name, - ip => $ip_address, - lat => $latitude, - long => $longitude, - port => $port, - southbound => $southbound, - vendor => $vendor, - model => $model, - sw_ver => $sw_ver); + my $fwdctl_topic; + my $discovery_topic; - if(!defined($node)){ - return $db->get_error(); + if ($controller eq 'netconf') { + $fwdctl_topic = 'MPLS.FWDCTL.RPC'; + $discovery_topic = 'MPLS.Discovery.RPC'; } - - if (!defined $mq) { - my $results = {}; - $results->{'results'} = [ { - "error" => "Internal server error occurred. Message queue connection failed.", - "success" => 0 - } ]; - return $results; - } else { - $mq->{'topic'} = 'MPLS.FWDCTL.RPC'; + if ($controller eq 'nso') { + $fwdctl_topic = 'NSO.FWDCTL.RPC'; + $discovery_topic = 'NSO.Discovery.RPC'; } my $cv = AnyEvent->condvar; + $mq->{'topic'} = $fwdctl_topic; $mq->new_switch( node_id => $node->{'node_id'}, async_callback => sub { my $resultM = shift; - - $mq->{'topic'} = 'MPLS.Discovery.RPC'; + $mq->{'topic'} = $discovery_topic; $mq->new_switch( node_id => $node->{'node_id'}, async_callback => sub { @@ -2746,7 +2755,7 @@ sub add_mpls_switch{ ); } ); - my $res = $cv->recv(); + $cv->recv; return {results => [{success => 1, node_id => $node->{'node_id'}}]}; diff --git a/frontend/www/js_templates/admin.js b/frontend/www/js_templates/admin.js index b812c5b14..bb1375f14 100644 --- a/frontend/www/js_templates/admin.js +++ b/frontend/www/js_templates/admin.js @@ -2399,7 +2399,7 @@ function setup_network_tab(){ var vendor = args[0].vendor; var model = args[0].model; var sw_version = args[0].sw_version; - var southbound = args[0].southbound; + var controller = args[0].controller; function show_interface_acl_panel(args){ var interface_id = args.interface_id; @@ -2897,7 +2897,7 @@ function setup_network_tab(){ // MPLS - Software Version "" + "Southbound" + - "" + + "" + "" + "" + "" + @@ -2916,8 +2916,8 @@ function setup_network_tab(){ panel.render(YAHOO.util.Dom.get("active_element_details")); - $('#southbound').on('change', function() { - if ($('#southbound').val() == 'netconf') { + $('#controller').on('change', function() { + if ($('#controller').val() == 'netconf') { $('#vendor').prop('disabled', false); $('#model').prop('disabled', false); $('#sw_version').prop('disabled', false); @@ -2992,9 +2992,9 @@ function setup_network_tab(){ YAHOO.util.Dom.get('vendor').value = vendor; YAHOO.util.Dom.get('model').value = model; YAHOO.util.Dom.get('sw_version').value = sw_version; - YAHOO.util.Dom.get('southbound').value = southbound; + YAHOO.util.Dom.get('controller').value = controller; - if (southbound == 'nso') { + if (controller == 'nso') { YAHOO.util.Dom.get('vendor').disabled = true; YAHOO.util.Dom.get('model').disabled = true; YAHOO.util.Dom.get('sw_version').disabled = true; @@ -3036,13 +3036,13 @@ function setup_network_tab(){ var new_max_static_mac_flows = YAHOO.util.Dom.get('active_max_static_mac_flows').value; var openflow = YAHOO.util.Dom.get('openflow_enabled').value; var mpls = YAHOO.util.Dom.get('mpls_enabled').checked; - var mgmt_addr = YAHOO.util.Dom.get('mgmt_addr').value; - var tcp_port = YAHOO.util.Dom.get('tcp_port').value; - var vendor = YAHOO.util.Dom.get('vendor').value; - var model = YAHOO.util.Dom.get('model').value; - var sw_version = YAHOO.util.Dom.get('sw_version').value; - var southbound = YAHOO.util.Dom.get('southbound').value; - var short_name = YAHOO.util.Dom.get('short_name').value; + var mgmt_addr = YAHOO.util.Dom.get('mgmt_addr').value; + var tcp_port = YAHOO.util.Dom.get('tcp_port').value; + var vendor = YAHOO.util.Dom.get('vendor').value; + var model = YAHOO.util.Dom.get('model').value; + var sw_version = YAHOO.util.Dom.get('sw_version').value; + var controller = YAHOO.util.Dom.get('controller').value; + var short_name = YAHOO.util.Dom.get('short_name').value; if (! new_name){ alert("You must specify a name for this device."); @@ -3087,7 +3087,7 @@ function setup_network_tab(){ "&vendor=" + encodeURIComponent(vendor) + "&model=" + encodeURIComponent(model) + "&sw_version=" + encodeURIComponent(sw_version) + - "&southbound=" + encodeURIComponent(southbound) + + "&controller=" + encodeURIComponent(controller) + "&short_name=" + encodeURIComponent(short_name); var openflow_args = "&default_drop=" + encodeURIComponent(new_default_drop) + @@ -3306,7 +3306,7 @@ function setup_discovery_tab(){ "" + "" + "Southbound"+ - "" + + "" + "" + "Vendor"+ "" + @@ -3327,8 +3327,8 @@ function setup_discovery_tab(){ this.new_mpls.render(document.body); - $('#southbound').on('change', function() { - if ($('#southbound').val() == 'netconf') { + $('#controller').on('change', function() { + if ($('#controller').val() == 'netconf') { $('#new_mpls_vendor').prop('disabled', false); $('#new_mpls_model').prop('disabled', false); $('#new_mpls_software').prop('disabled', false); @@ -3364,7 +3364,7 @@ function setup_discovery_tab(){ var short_name = YAHOO.util.Dom.get('new_node_short_name').value; var ip = YAHOO.util.Dom.get('new_ip_address').value; var port = YAHOO.util.Dom.get('new_port').value; - var southbound = YAHOO.util.Dom.get('southbound').value; + var controller = YAHOO.util.Dom.get('controller').value; var vendor = $('#new_mpls_vendor').prop('disabled') ? null : YAHOO.util.Dom.get('new_mpls_vendor').value; var model = $('#new_mpls_model').prop('disabled') ? null : YAHOO.util.Dom.get('new_mpls_model').value; var sw_ver = $('#new_mpls_software').prop('disabled') ? null : YAHOO.util.Dom.get('new_mpls_software').value; @@ -3399,7 +3399,7 @@ function setup_discovery_tab(){ return; } - if (southbound === 'netconf' && (vendor === '' || model === '' || sw_ver === '')) { + if (controller === 'netconf' && (vendor === '' || model === '' || sw_ver === '')) { alert('Hardware vendor, model, and software version are required for NETCONF controlled devices.'); return; } @@ -3414,7 +3414,7 @@ function setup_discovery_tab(){ url += `&longitude=${encodeURIComponent(lon)}`; url += `&ip_address=${encodeURIComponent(ip)}`; url += `&port=${encodeURIComponent(port)}`; - url += `&southbound=${encodeURIComponent(southbound)}`; + url += `&controller=${encodeURIComponent(controller)}`; if (vendor) url += `&vendor=${encodeURIComponent(vendor)}`; if (model) url += `&model=${encodeURIComponent(model)}`; if (sw_ver) url += `&sw_ver=${encodeURIComponent(sw_ver)}`; diff --git a/frontend/www/js_templates/maps.js b/frontend/www/js_templates/maps.js index ac5b3e715..5cc58ffb4 100644 --- a/frontend/www/js_templates/maps.js +++ b/frontend/www/js_templates/maps.js @@ -285,7 +285,7 @@ function NDDIMap(div_id, interdomain_mode, options){ var vendor = node_info.vendor; var model = node_info.model; var sw_version = node_info.sw_version; - var southbound = node_info.southbound; + var controller = node_info.controller; var short_name = node_info.short_name; var avail_endpoints = node_info.number_available_endpoints; var barrier_bulk = node_info.barrier_bulk; @@ -346,7 +346,7 @@ function NDDIMap(div_id, interdomain_mode, options){ point.model = model; point.short_name = short_name; point.sw_version = sw_version; - point.southbound = southbound; + point.controller = controller; point.barrier_bulk = barrier_bulk; point.max_static_mac_flows = max_static_mac_flows; point.dpid = dpid; @@ -1372,18 +1372,18 @@ function NDDIMap(div_id, interdomain_mode, options){ var max_flows = geo.max_flows; var openflow = geo.openflow; var mpls = geo.mpls; - var mgmt_addr = geo.mgmt_addr; - var tcp_port = geo.tcp_port; - var vendor = geo.vendor; - var model = geo.model; - var sw_version = geo.sw_version; - var southbound = geo.southbound; - var tx_delay_ms = geo.tx_delay_ms; - var short_name = geo.short_name; + var mgmt_addr = geo.mgmt_addr; + var tcp_port = geo.tcp_port; + var vendor = geo.vendor; + var model = geo.model; + var sw_version = geo.sw_version; + var controller = geo.controller; + var tx_delay_ms = geo.tx_delay_ms; + var short_name = geo.short_name; var barrier_bulk = geo.barrier_bulk; var max_static_mac_flows = geo.max_static_mac_flows; var dpid = geo.dpid; - self.events['clickNode'].fire({name: node, lat: lat, lon: lon, node_id: node_id, vlan_range: range,default_forward: default_forward, default_drop: default_drop,max_flows: max_flows, tx_delay_ms: tx_delay_ms, feature: e.feature, barrier_bulk: barrier_bulk, max_static_mac_flows: max_static_mac_flows, dpid: dpid, openflow: openflow, mpls: mpls, mgmt_addr: mgmt_addr, tcp_port: tcp_port, vendor: vendor, model: model,short_name: short_name, sw_version: sw_version, southbound: southbound}); + self.events['clickNode'].fire({name: node, lat: lat, lon: lon, node_id: node_id, vlan_range: range,default_forward: default_forward, default_drop: default_drop,max_flows: max_flows, tx_delay_ms: tx_delay_ms, feature: e.feature, barrier_bulk: barrier_bulk, max_static_mac_flows: max_static_mac_flows, dpid: dpid, openflow: openflow, mpls: mpls, mgmt_addr: mgmt_addr, tcp_port: tcp_port, vendor: vendor, model: model,short_name: short_name, sw_version: sw_version, controller: controller}); } // otherwise we're clicking on a link else{ diff --git a/perl-lib/OESS/lib/OESS/DB/Node.pm b/perl-lib/OESS/lib/OESS/DB/Node.pm index b52b6d9d3..aab1fe76a 100644 --- a/perl-lib/OESS/lib/OESS/DB/Node.pm +++ b/perl-lib/OESS/lib/OESS/DB/Node.pm @@ -40,8 +40,12 @@ sub fetch{ sub fetch_all { my %params = @_; my $db = $params{'db'}; + my $controller = $params{'controller'} || 'netconf'; - return $db->execute_query("select * from node join node_instantiation on node.node_id=node_instantiation.node_id where node_instantiation.end_epoch=-1", []); + return $db->execute_query( + "select * from node join node_instantiation on node.node_id=node_instantiation.node_id where node_instantiation.end_epoch=-1 and node_instantiation.controller=?", + [$controller] + ); } =head2 get_node_interfaces diff --git a/perl-lib/OESS/lib/OESS/Database.pm b/perl-lib/OESS/lib/OESS/Database.pm index bda7568b7..0dad21083 100644 --- a/perl-lib/OESS/lib/OESS/Database.pm +++ b/perl-lib/OESS/lib/OESS/Database.pm @@ -1486,7 +1486,7 @@ sub get_map_layers { node.name as node_name, node.node_id, node.short_name, - node_instantiation.southbound, + node_instantiation.controller, node_instantiation.openflow, node_instantiation.mpls, node_instantiation.vendor, @@ -1556,13 +1556,13 @@ HERE "node_id" => $row->{'node_id'}, "openflow" => $row->{'openflow'}, "mpls" => $row->{'mpls'}, - "short_name" => $row->{'short_name'}, - "southbound" => $row->{'southbound'}, - "vendor" => $row->{'vendor'}, - "model" => $row->{'model'}, - "sw_version" => $row->{'sw_version'}, - "mgmt_addr" => $row->{'mgmt_addr'}, - "tcp_port" => $row->{'tcp_port'}, + "short_name" => $row->{'short_name'}, + "controller" => $row->{'controller'}, + "vendor" => $row->{'vendor'}, + "model" => $row->{'model'}, + "sw_version" => $row->{'sw_version'}, + "mgmt_addr" => $row->{'mgmt_addr'}, + "tcp_port" => $row->{'tcp_port'}, "vlan_range" => $row->{'vlan_tag_range'}, "default_drop" => $row->{'default_drop'}, "default_forward" => $row->{'default_forward'}, @@ -7985,7 +7985,7 @@ sub get_node_by_interface_id { lat => $lat, long => $long, port => $port, - southbound => $southbound, + controller => $controller, vendor => $vendor, model => $model, sw_ver => $sw_ver); @@ -8013,7 +8013,7 @@ sub add_mpls_node{ openflow => 0, mpls => 1, admin_state => 'active', - southbound => $args{'southbound'} || 'netconf', + controller => $args{'controller'} || 'netconf', vendor => $args{'vendor'}, model => $args{'model'}, sw_version => $args{'sw_ver'}, @@ -8131,7 +8131,7 @@ sub create_node_instance{ $args{'dpid'} = unpack('N', $data); } - my $res = $self->_execute_query("insert into node_instantiation (node_id,end_epoch,start_epoch,mgmt_addr,admin_state,dpid,southbound,vendor,model,sw_version,mpls,openflow ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",[$args{'node_id'},-1,time(),$args{'mgmt_addr'},$args{'admin_state'},$args{'dpid'},$args{'southbound'},$args{'vendor'},$args{'model'},$args{'sw_version'},$args{'mpls'},$args{'openflow'}]); + my $res = $self->_execute_query("insert into node_instantiation (node_id,end_epoch,start_epoch,mgmt_addr,admin_state,dpid,controller,vendor,model,sw_version,mpls,openflow ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",[$args{'node_id'},-1,time(),$args{'mgmt_addr'},$args{'admin_state'},$args{'dpid'},$args{'controller'},$args{'vendor'},$args{'model'},$args{'sw_version'},$args{'mpls'},$args{'openflow'}]); if(!defined($res)){ $self->_set_error("Unable to create new node instantiation"); diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index 52a8252d7..732740ed4 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -234,12 +234,12 @@ sub new_switch { if (!defined $node) { my $err = "Couldn't lookup node $params->{node_id}{value}. Discovery will not properly complete on this node."; $self->{logger}->error($err); - &$error($err); + return &$error($err); } $self->{nodes}->{$params->{node_id}{value}} = $node; - warn "Switch $node->{name} registered with Discovery."; - $self->{logger}->info("Switch $node->{name} registered with Discovery."); + warn "Switch $node->{name} registered with NSO.Discovery."; + $self->{logger}->info("Switch $node->{name} registered with NSO.Discovery."); # Make first invocation of polling subroutines $self->device_handler; @@ -255,7 +255,7 @@ sub start { my $self = shift; # Load devices from database - my $nodes = OESS::DB::Node::fetch_all(db => $self->{db}); + my $nodes = OESS::DB::Node::fetch_all(db => $self->{db}, controller => 'nso'); if (!defined $nodes) { warn "Couldn't lookup nodes. Discovery will not collect data on any existing nodes."; $self->{logger}->error("Couldn't lookup nodes. Discovery will not collect data on any existing nodes."); diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index 1321b7c37..dc9ec1bc7 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -76,7 +76,7 @@ sub start { my $self = shift; # Load devices from database - my $nodes = OESS::DB::Node::fetch_all(db => $self->{db}); + my $nodes = OESS::DB::Node::fetch_all(db => $self->{db}, controller => 'nso'); if (!defined $nodes) { warn "Couldn't lookup nodes. FWDCTL will not provision on any existing nodes."; $self->{logger}->error("Couldn't lookup nodes. Discovery will not provision on any existing nodes."); @@ -675,7 +675,7 @@ sub new_switch { if (!defined $node) { my $err = "Couldn't lookup node $params->{node_id}{value}. FWDCTL will not properly provision on this node."; $self->{logger}->error($err); - &$error($err); + return &$error($err); } $self->{nodes}->{$params->{node_id}{value}} = $node; diff --git a/perl-lib/OESS/share/nddi.sql b/perl-lib/OESS/share/nddi.sql index 719b10689..eaf8cf3ba 100755 --- a/perl-lib/OESS/share/nddi.sql +++ b/perl-lib/OESS/share/nddi.sql @@ -608,6 +608,7 @@ CREATE TABLE `node_instantiation` ( `mgmt_addr` varchar(255) DEFAULT NULL, `loopback_address` varchar(255) DEFAULT NULL, `tcp_port` int(6) DEFAULT '830', + `controller` enum('openflow','netconf','nso') NOT NULL DEFAULT 'nso', PRIMARY KEY (`node_id`,`end_epoch`), UNIQUE KEY `node_instantiation_idx` (`end_epoch`,`dpid`), CONSTRAINT `node_node_instantiation_fk` FOREIGN KEY (`node_id`) REFERENCES `node` (`node_id`) ON DELETE NO ACTION ON UPDATE NO ACTION diff --git a/perl-lib/OESS/share/upgrade/oess-2.0.11-2.0.12 b/perl-lib/OESS/share/upgrade/oess-2.0.11-2.0.12 new file mode 100644 index 000000000..745441055 --- /dev/null +++ b/perl-lib/OESS/share/upgrade/oess-2.0.11-2.0.12 @@ -0,0 +1,97 @@ +#!/usr/bin/perl +#------------------------------------------------------------------- +#----- OESS 2.0.11 - 2.0.12 upgrade module +#----- +#----- Copyright(C) 2010 The Trustees of Indiana University +#-------------------------------------------------------------------- +#----- $HeadURL: $ +#----- $Id: $ +#----- +#----- This is run when upgrading the database from +#----- version 2.0.11 to version 2.0.12 +#-------------------------------------------------------------------- + +use strict; +use warnings; +use OESS::Database; + +my $prev_version = "2.0.11"; +my $version = "2.0.12"; + +sub main{ + print "*******************************************************************\n"; + print "********* OESS DB UPGRADE ************\n"; + print "*******************************************************************\n"; + print "********* This will upgrade from $prev_version to $version **********\n"; + print "********* of the OESS DB any other version will not work ************\n"; + + continue_parameter("Do you wish to continue"); + + my $dbq = new OESS::Database(); + my $current_version = $dbq->get_oess_schema_version(); + if($current_version eq $prev_version){ + eval { + $dbq->{'dbh'}->begin_work(); + upgrade($dbq); + $dbq->{'dbh'}->commit(); + }; + if ($@) { + $dbq->{'dbh'}->rollback(); + print "$@\n"; + exit; + } + } else{ + print "Wrong version of OESS DB\n"; + print "This script only upgrades from version $prev_version to $version\n"; + exit; + } + + print STDERR "Upgrade Successful!!\n"; +} + +sub upgrade{ + my $dbq = shift; + my $term = shift; + my $dbh = $dbq->{'dbh'}; + my $str; + my $sth; + + $sth = $dbh->prepare("ALTER TABLE node_instantiation ADD controller enum('openflow','netconf','nso') NOT NULL DEFAULT 'nso'"); + $sth->execute() or die "Unable to add 'controller' to Table"; + + $sth = $dbh->prepare("UPDATE node_instantiation SET controller='openflow' WHERE node_instantiation.openflow=1"); + $sth->execute() or die "Unable to update column 'controller' in node_instantiation table"; + + $sth = $dbh->prepare("UPDATE node_instantiation SET controller='netconf' WHERE node_instantiation.mpls=1"); + $sth->execute() or die "Unable to update column 'controller' in node_instantiation table"; + + $str = "update oess_version set version = '$version'"; + $sth = $dbh->prepare($str) or die "Unable to prepare version update \n"; + $sth->execute() or die "Unable to update version\n"; +} + +main(); + +sub continue_parameter { + my $name = shift; + + print "$name [y/n]: "; + my $yes_or_no = <>; + chomp($yes_or_no); + + exit(0) if ($yes_or_no !~ /y/i || $yes_or_no =~ /n/i); +} + +sub required_parameter { + my $name = shift; + + while (1) { + print "$name (required): "; + my $response = <>; + chomp($response); + + return $response if ($response); + + print "\nThis option is required!\n\n"; + } +} From 113cc8de7902b9b904c3e158abbe557c302df9b1 Mon Sep 17 00:00:00 2001 From: Andrew Ragusa Date: Mon, 29 Mar 2021 17:23:46 +0000 Subject: [PATCH 036/156] starting to work on the NSI modules to make them work with the new API --- perl-lib/OESS/lib/OESS/NSI/Provisioning.pm | 40 +++++++++-------- perl-lib/OESS/lib/OESS/NSI/Query.pm | 8 ++-- perl-lib/OESS/lib/OESS/NSI/Reservation.pm | 50 ++++++---------------- 3 files changed, 38 insertions(+), 60 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSI/Provisioning.pm b/perl-lib/OESS/lib/OESS/NSI/Provisioning.pm index ed50448ff..5b52b636e 100644 --- a/perl-lib/OESS/lib/OESS/NSI/Provisioning.pm +++ b/perl-lib/OESS/lib/OESS/NSI/Provisioning.pm @@ -149,10 +149,13 @@ sub _do_provisioning{ return; } - my $url = $self->{'websvc_location'} . "provisioning.cgi"; + my $url = $self->{'websvc_location'} . "circuit.cgi"; $self->{'websvc'}->set_url($url); - my $res = $self->{'websvc'}->provision_circuit( + my $endpointa = {node => $ckt->{'node'}->[0], interface => $ckt->{'interface'}->[0], tag => $ckt->{'tag'}->[0]}; + my $endpointb = {node => $ckt->{'node'}->[1], interface => $ckt->{'interface'}->[1], tag => $ckt->{'tag'}->[1]}; + + my $res = $self->{'websvc'}->provision( state => 'provisioned', circuit_id => $connection_id, workgroup_id => $self->{'workgroup_id'}, @@ -163,10 +166,7 @@ sub _do_provisioning{ bandwidth => $ckt->{'bandwidth'}, provision_time => -1, remove_time => 1, - node => $ckt->{'node'}, - interface => $ckt->{'interface'}, - tag => $ckt->{'tag'} - ); + endpoint => [$endpointa, $endpointb]); if (!defined $res) { log_error("Couldn't call provision_circuit using $url: Fatal webservice error occurred."); push(@{$self->{'provisioning_queue'}}, {type => OESS::NSI::Constant::PROVISIONING_FAILED, args => $args}); @@ -194,15 +194,16 @@ sub _get_circuit_details{ my $self = shift; my $circuit_id = shift; - $self->{'websvc'}->set_url($self->{'websvc_location'} . "/data.cgi"); + $self->{'websvc'}->set_url($self->{'websvc_location'} . "/circuit.cgi"); log_debug("fetching circuit details"); - my $circuit = $self->{'websvc'}->get_circuit_details( - circuit_id => $circuit_id); - + my $circuit = $self->{'websvc'}->get( + circuit_id => $circuit_id, workgroup_id => $self->{'workgroup_id'}); + log_debug("Circuit Details: " . Data::Dumper::Dumper($circuit)); + #ugh how to get this... my $scheduled_actions = $self->{'websvc'}->get_circuit_scheduled_events( circuit_id => $circuit_id); @@ -281,14 +282,14 @@ sub _do_terminate{ my ($self, $args) = @_; my $connection_id = $args->{'connectionId'}; - my $cgi = $self->{'websvc_location'} . "provisioning.cgi"; + my $cgi = $self->{'websvc_location'} . "circuit.cgi"; log_debug("_do_terminate - url: $cgi circuit: $connection_id"); $self->{'websvc'}->set_url($cgi); - my $res = $self->{'websvc'}->remove_circuit(circuit_id => $connection_id, - workgroup_id => $self->{'workgroup_id'}, - remove_time => -1); + my $res = $self->{'websvc'}->remove(circuit_id => $connection_id, + workgroup_id => $self->{'workgroup_id'}, + remove_time => -1); log_debug("Results of remove_circuit: " . Data::Dumper::Dumper($res)); @@ -462,9 +463,9 @@ sub _do_release{ $ckt->{'remove_time'} = -1; } - $self->{'websvc'}->set_url($self->{'websvc_location'} . "/provisioning.cgi"); + $self->{'websvc'}->set_url($self->{'websvc_location'} . "/circuit.cgi"); - my $res = $self->{'websvc'}->provision_circuit( + my $res = $self->{'websvc'}->provision( state => 'reserved', circuit_id => $connection_id, workgroup_id => $self->{'workgroup_id'}, @@ -475,9 +476,10 @@ sub _do_release{ bandwidth => $ckt->{'bandwidth'}, provision_time => $ckt->{'provision_time'}, remove_time => $ckt->{'remove_time'}, - node => $ckt->{'node'}, - interface => $ckt->{'interface'}, - tag => $ckt->{'tag'}); + endpoint => [$endpointa, $endpointb]); + #node => $ckt->{'node'}, + #interface => $ckt->{'interface'}, + #tag => $ckt->{'tag'}); if(defined($res) && defined($res->{'results'})){ log_info("Release connectionId: " . $args->{'connectionId'} . " success!"); diff --git a/perl-lib/OESS/lib/OESS/NSI/Query.pm b/perl-lib/OESS/lib/OESS/NSI/Query.pm index 5549f39d1..0e2b9bfdf 100644 --- a/perl-lib/OESS/lib/OESS/NSI/Query.pm +++ b/perl-lib/OESS/lib/OESS/NSI/Query.pm @@ -135,8 +135,8 @@ sub query_summary{ sub get_current_circuits{ my $self = shift; - $self->{'websvc'}->set_url($self->{'websvc_location'} . "data.cgi"); - my $current_circuits = $self->{'websvc'}->get_existing_circuits(workgroup_id => $self->{'workgroup_id'}); + $self->{'websvc'}->set_url($self->{'websvc_location'} . "circuit.cgi"); + my $current_circuits = $self->{'websvc'}->get(workgroup_id => $self->{'workgroup_id'}); if(defined($current_circuits) && defined($current_circuits->{'results'})){ return $current_circuits->{'results'}; @@ -155,8 +155,8 @@ sub do_query_summarysync{ my $self = shift; my $args = shift; - $self->{'websvc'}->set_url($self->{'websvc_location'} . "data.cgi"); - my $current_circuits = $self->{'websvc'}->get_existing_circuits(workgroup_id => $self->{'workgroup_id'}); + $self->{'websvc'}->set_url($self->{'websvc_location'} . "circuit.cgi"); + my $current_circuits = $self->{'websvc'}->get(workgroup_id => $self->{'workgroup_id'}); my @ckts = (); diff --git a/perl-lib/OESS/lib/OESS/NSI/Reservation.pm b/perl-lib/OESS/lib/OESS/NSI/Reservation.pm index b0941105d..9730b3fa6 100644 --- a/perl-lib/OESS/lib/OESS/NSI/Reservation.pm +++ b/perl-lib/OESS/lib/OESS/NSI/Reservation.pm @@ -183,9 +183,11 @@ sub reserve { # # my $backup_path = $self->get_shortest_path($ep1, $ep2, $primary_path); - $self->{'websvc'}->set_url($self->{'websvc_location'} . "provisioning.cgi"); + $self->{'websvc'}->set_url($self->{'websvc_location'} . "circuit.cgi"); - my $res = $self->{'websvc'}->provision_circuit( + my $endpoint1 = {'node' => $ep1->{'node'}, interface => $ep1->{'port'}, tag => $ep1->{'vlan'}}; + my $endpoint2 = {'node' => $ep2->{'node'}, interface => $ep2->{'port'}, tag => $ep2->{'vlan'}}; + my $res = $self->{'websvc'}->provision( state => 'reserved', workgroup_id => $self->{'workgroup_id'}, external_identifier => $gri, @@ -197,9 +199,10 @@ sub reserve { # backup_link => $backup_path, remote_url => $args->{'header'}->{'replyTo'}, remote_requester => $args->{'header'}->{'requesterNSA'}, - node => [$ep1->{'node'}, $ep2->{'node'}], - interface => [$ep1->{'port'}, $ep2->{'port'}], - tag => [$ep1->{'vlan'}, $ep2->{'vlan'}]); + endpoint => [$endpoint1, $endpoint2]); + #node => [$ep1->{'node'}, $ep2->{'node'}], + #interface => [$ep1->{'port'}, $ep2->{'port'}], + #tag => [$ep1->{'vlan'}, $ep2->{'vlan'}]); log_debug("Results of provision: " . Data::Dumper::Dumper($res)); @@ -243,8 +246,8 @@ sub _do_reserve_abort{ log_info("reservationAbort: connectionId: " . $connection_id); - $self->{'websvc'}->set_url($self->{'websvc_location'} . "/provisioning.cgi"); - my $res = $self->{'websvc'}->remove_circuit( + $self->{'websvc'}->set_url($self->{'websvc_location'} . "/circuit.cgi"); + my $res = $self->{'websvc'}->remove( circuit_id => $connection_id, workgroup_id => $self->{'workgroup_id'}, remove_time => -1); @@ -352,7 +355,7 @@ sub validate_endpoint{ #need to verify this is part of our network and actually exists and that we have permission! $self->{'log'}->info("Checking validity of port $ep->{'port'} on $ep->{'node'}."); - my $url = $self->{'websvc_location'} . "data.cgi"; + my $url = $self->{'websvc_location'} . "circuit.cgi"; $self->{'websvc'}->set_url($url); $self->{'log'}->debug("Requesting all resources for NSI workgroup from $url"); @@ -416,33 +419,6 @@ sub validate_endpoint{ return 0; } -=head2 get_shortest_path - -=cut - -sub get_shortest_path{ - my $self = shift; - my $ep1 = shift; - my $ep2 = shift; - my $links = shift; - - $self->{'websvc'}->set_url($self->{'websvc_location'} . "data.cgi"); - my $shortest_path = $self->{'websvc'}->get_shortest_path( - node => [$ep1->{'node'},$ep2->{'node'}], - link => $links); - - log_debug("Shortest path: " . Data::Dumper::Dumper($shortest_path)); - if(defined($shortest_path) && defined($shortest_path->{'results'})){ - my @links = (); - foreach my $link (@{$shortest_path->{'results'}}){ - push(@links,$link->{'link'}); - } - return \@links; - } - log_error("unable to find path"); - return; -} - =head2 reserveCommit =cut @@ -750,9 +726,9 @@ sub _release_confirmed{ sub _reserve_timeout{ my ($self, $data) = @_; - $self->{'websvc'}->set_url($self->{'websvc_location'} . "provisioning.cgi"); + $self->{'websvc'}->set_url($self->{'websvc_location'} . "circuit.cgi"); - my $res = $self->{'websvc'}->remove_circuit( + my $res = $self->{'websvc'}->remove( circuit_id => $data->{'connection_id'}, workgroup_id => $self->{'workgroup_id'}); From 372874d42bc795589631a1723557d09cf71b84b9 Mon Sep 17 00:00:00 2001 From: Andrew Ragusa Date: Mon, 29 Mar 2021 18:25:51 +0000 Subject: [PATCH 037/156] adding backend support for circuit state... --- frontend/webservice/circuit.cgi | 12 ++++++++++-- perl-lib/OESS/lib/OESS/DB/Circuit.pm | 13 ++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/frontend/webservice/circuit.cgi b/frontend/webservice/circuit.cgi index c65fbd5b9..80198f5c8 100755 --- a/frontend/webservice/circuit.cgi +++ b/frontend/webservice/circuit.cgi @@ -125,6 +125,13 @@ my $provision = GRNOC::WebService::Method->new( callback => \&provision, description => 'Creates and provisions a new Circuit.' ); +$provision->add_input_parameter( + name => 'status' + pattern => '(reserved|confirmed|provisioned|released|decom)', + required => 1, + default => 'active' + description => 'Status of the Circuit (note mostly used for NSI integration)' + ); $provision->add_input_parameter( name => 'circuit_id', pattern => $GRNOC::WebService::Regex::INTEGER, @@ -246,7 +253,8 @@ sub provision { my $circuit = new OESS::L2Circuit( db => $db, model => { - name => $args->{description}->{value}, + status => $args->{status}->{value}, + name => $args->{description}->{value}, description => $args->{description}->{value}, remote_url => $args->{remote_url}->{value}, remote_requester => $args->{remote_requester}->{value}, @@ -463,7 +471,7 @@ sub update { $circuit->load_paths; my $previous = $circuit->to_hash; - + $circuit->status($args->{status}->{value}); $circuit->description($args->{description}->{value}); $circuit->remote_url($args->{remote_url}->{value}); $circuit->remote_requester($args->{remote_requester}->{value}); diff --git a/perl-lib/OESS/lib/OESS/DB/Circuit.pm b/perl-lib/OESS/lib/OESS/DB/Circuit.pm index 811d55c27..8abb702da 100644 --- a/perl-lib/OESS/lib/OESS/DB/Circuit.pm +++ b/perl-lib/OESS/lib/OESS/DB/Circuit.pm @@ -16,6 +16,7 @@ use Data::Dumper; my $id = OESS::DB::Circuit::create( db => $db, model => { + status => $status, name => $name, description => $description, user_id => $user_id, @@ -38,11 +39,13 @@ sub create { return (undef, 'Required argument `db` is missing.') if !defined $args->{db}; return (undef, 'Required argument `model` is missing.') if !defined $args->{model}; - - my $circuit_state = 'active'; - if (defined $args->{model}->{provision_time}) { - # TODO add provision event - $circuit_state = 'scheduled'; + my $circuit_state = $args->{model}->{status}; + if(!defined($circuit_state)){ + $circuit_state = 'active'; + if (defined $args->{model}->{provision_time}) { + # TODO add provision event + $circuit_state = 'scheduled'; + } } if (defined $args->{model}->{remove_time}) { From 931e00fce335fc03b5e0577b1049cb434f8992ad Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Thu, 1 Apr 2021 21:56:31 +0000 Subject: [PATCH 038/156] #1223 - fixed issue with start and end vlan params when adding new ACL --- frontend/www/js_utilities/interface_acl_panel.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/www/js_utilities/interface_acl_panel.js b/frontend/www/js_utilities/interface_acl_panel.js index eb48ad9b6..6166d07fe 100644 --- a/frontend/www/js_utilities/interface_acl_panel.js +++ b/frontend/www/js_utilities/interface_acl_panel.js @@ -143,14 +143,14 @@ var get_interface_acl_panel = function(container_id, interface_id, options){ } //required url += "&allow_deny="+allow_deny; - url += "&start="+vlan_start; + url += "&vlan_start="+vlan_start; url += "&interface_id="+interface_id; //optional if(workgroup_id) {url += "&workgroup_id="+workgroup_id;} if(entity_id) {url += "&entity_id="+entity_id;} if(notes) {url += "¬es="+notes;} - if(vlan_end) {url += "&end="+vlan_end;} + if(vlan_end) {url += "&vlan_end="+vlan_end;} var ds = new YAHOO.util.DataSource(url); ds.responseType = YAHOO.util.DataSource.TYPE_JSON; From 205afec4d868f77f263671527bb95b69b33c644b Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Tue, 6 Apr 2021 13:42:12 +0000 Subject: [PATCH 039/156] 1225 - l2 edit name done --- frontend/www/html_templates/modify_l2vpn.html | 6 +++ frontend/www/js_templates/l2/circuit.js | 2 +- frontend/www/js_templates/modify_l2vpn.js | 38 ++++++++++++++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/frontend/www/html_templates/modify_l2vpn.html b/frontend/www/html_templates/modify_l2vpn.html index ea5a934ee..b1f9ee0ed 100644 --- a/frontend/www/html_templates/modify_l2vpn.html +++ b/frontend/www/html_templates/modify_l2vpn.html @@ -1,5 +1,11 @@
+ +
diff --git a/frontend/www/js_templates/l2/circuit.js b/frontend/www/js_templates/l2/circuit.js index 1136755f8..acd5ada53 100644 --- a/frontend/www/js_templates/l2/circuit.js +++ b/frontend/www/js_templates/l2/circuit.js @@ -105,7 +105,7 @@ class CircuitHeader extends Component { return `
-

${props.description} ${props.connectionId}

+

${props.description}

${props.connectionId}
diff --git a/frontend/www/js_templates/modify_l2vpn.js b/frontend/www/js_templates/modify_l2vpn.js index 577cfb423..1956e9d58 100644 --- a/frontend/www/js_templates/modify_l2vpn.js +++ b/frontend/www/js_templates/modify_l2vpn.js @@ -54,10 +54,9 @@ class GlobalState extends Component { let provisionModal = $('#modify-loading'); provisionModal.find('p').text("Give us a few seconds. We're modifying your connection now."); provisionModal.modal('show'); - provisionCircuit( session.data.workgroup_id, - this.circuit.description, + document.querySelector('#header-description').textContent, this.circuit.endpoints, this.circuit.provision_time, this.circuit.remove_time, @@ -110,6 +109,41 @@ document.querySelector('.l2vpn-new-endpoint-button').addEventListener('click', f modal.display(); }); +function doneEditingName(){ + let name = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + const newName = document.getElementById("description-input").value == "" ? state.circuit.description : document.getElementById("description-input").value + name.innerHTML = "

" + newName + "

" + document.getElementById("change-description-button").hidden = true + button.textContent = "Edit Name" +} + +document.querySelector('.change-description-button').addEventListener('click', function(e) { + doneEditingName() +}) + +document.querySelector('.edit-description-button').addEventListener('click', function(e) { + let name = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + + if(button.textContent.trim() == "Edit Name"){ + name.innerHTML = "" + name.addEventListener("keyup", function(event) { + // Number 13 is the "Enter" key on the keyboard + if (event.keyCode === 13) { + event.preventDefault(); + doneEditingName() + } + }); + button.textContent = "Revert" + document.getElementById("change-description-button").hidden = false + }else{ + name.innerHTML = "

" + state.circuit.description + "

" + button.textContent = "Edit Name" + document.getElementById("change-description-button").hidden = true + } +}); + let circuitHeader = null; let details = null; let history = null; From 57c0039a9dabbf6e223f1e6b18492500ee36e7b1 Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Wed, 7 Apr 2021 13:50:25 +0000 Subject: [PATCH 040/156] 1225 - added edit name to l3 circuit --- frontend/webservice/vrf.cgi | 5 ++- frontend/www/html_templates/modify_cloud.html | 8 ++++ frontend/www/js_templates/modify_cloud.js | 39 ++++++++++++++++++- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/frontend/webservice/vrf.cgi b/frontend/webservice/vrf.cgi index 6d7fcb647..19e63bf20 100755 --- a/frontend/webservice/vrf.cgi +++ b/frontend/webservice/vrf.cgi @@ -322,7 +322,10 @@ sub provision_vrf{ my $previous_vrf = undef; if (defined $model->{'vrf_id'} && $model->{'vrf_id'} != -1) { - $vrf = OESS::VRF->new(db => $db, vrf_id => $model->{'vrf_id'}); + $model = {}; + $model->{'description'} = $params->{'description'}{'value'}; + $model->{'vrf_id'} = $params->{'vrf_id'}{'value'} || undef; + $vrf = OESS::VRF->new(db => $db, model => $model); if (!defined $vrf) { $method->set_error("Couldn't load VRF"); $db->rollback; diff --git a/frontend/www/html_templates/modify_cloud.html b/frontend/www/html_templates/modify_cloud.html index 38c2434f2..1d7f0bf2a 100644 --- a/frontend/www/html_templates/modify_cloud.html +++ b/frontend/www/html_templates/modify_cloud.html @@ -31,6 +31,13 @@
+
+ +
@@ -94,6 +101,7 @@ [% INCLUDE js_templates/l3/peering_modal.js %] [% INCLUDE js_templates/modify_cloud.js %] +
diff --git a/frontend/www/js_templates/modify_cloud.js b/frontend/www/js_templates/modify_cloud.js index 30f104288..27a4f1367 100644 --- a/frontend/www/js_templates/modify_cloud.js +++ b/frontend/www/js_templates/modify_cloud.js @@ -130,12 +130,11 @@ class GlobalState extends Component { let addNetworkLoadingModal = $('#add-connection-loading'); addNetworkLoadingModal.modal('show'); - try { let vrfID = await provisionVRF( session.data.workgroup_id, this.connection.name, - this.connection.description, + document.querySelector('#header-description').textContent, this.connection.endpoints, -1, -1, @@ -197,6 +196,42 @@ document.addEventListener('DOMContentLoaded', async function() { modal.display(null); }); + + function doneEditingName(){ + let name = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + const newName = document.getElementById("description-input").value == "" ? state.connection.description : document.getElementById("description-input").value + name.innerHTML = "

" + newName + "

" + document.getElementById("change-description-button").hidden = true + button.textContent = "Edit Name" + } + + document.querySelector('.change-description-button').addEventListener('click', function(e) { + doneEditingName() + }) + + document.querySelector('.edit-description-button').addEventListener('click', function(e) { + let name = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + + if(button.textContent.trim() == "Edit Name"){ + name.innerHTML = "" + name.addEventListener("keyup", function(event) { + // Number 13 is the "Enter" key on the keyboard + if (event.keyCode === 13) { + event.preventDefault(); + doneEditingName() + } + }); + button.textContent = "Revert" + document.getElementById("change-description-button").hidden = false + }else{ + name.innerHTML = "

" + state.connection.description + "

" + button.textContent = "Edit Name" + document.getElementById("change-description-button").hidden = true + } + }); + let map = new NDDIMap('map'); map.on("loaded", function(){ this.updateMapFromSession(session); From f7a2277b0fbb259f0f0fc38a141b4d888cde6096 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 9 Nov 2020 18:53:19 +0000 Subject: [PATCH 041/156] limit modification of remote_auth entires when editing users preivously edit_user would create a new entriy in the remote_auth table for each username associated with a user, even if there was no change to those usernames. This change modifies this behavior to only make changes to the remote_auth entries when requried. --- perl-lib/OESS/lib/OESS/DB/User.pm | 57 ++++++---- perl-lib/OESS/t/z-DB/User.edit_user.01.t | 137 +++++++++++++++++++++++ 2 files changed, 172 insertions(+), 22 deletions(-) create mode 100644 perl-lib/OESS/t/z-DB/User.edit_user.01.t diff --git a/perl-lib/OESS/lib/OESS/DB/User.pm b/perl-lib/OESS/lib/OESS/DB/User.pm index 1f7fef100..a2a3999c6 100644 --- a/perl-lib/OESS/lib/OESS/DB/User.pm +++ b/perl-lib/OESS/lib/OESS/DB/User.pm @@ -302,7 +302,7 @@ sub delete_user { sub edit_user { my %params = @_; my $db = $params{'db'}; - + my $user_id = $params{'user_id'}; my $given_name = $params{'given_name'}; my $family_name = $params{'family_name'}; @@ -317,37 +317,50 @@ sub edit_user { return (undef, 'Required argument `email` is missing.') if !defined $email; return (undef, 'Required argument `auth_names` is missing.') if !defined $auth_names; return (undef, 'Required argument `status` is missing.') if !defined $status; - + if ($given_name =~ /^system$/ || $family_name =~ /^system$/) { return(undef, "User 'system' is reserved."); } + my $query = "UPDATE user SET email = ?, given_names = ?, family_name = ?, status = ? WHERE user_id = ?"; - my $query = "UPDATE user SET email = ?, given_names = ?, family_name = ?, status = ? WHERE user_id = $user_id"; - - my $results = $db->execute_query($query, [$email, $given_name, $family_name, $status]); - + my $results = $db->execute_query($query, [$email, $given_name, $family_name, $status, $user_id]); if (!defined $user_id || $results == 0) { return (undef, "Unable to edit user - does this user actually exist?"); } - # TODO Blindly removing and adding remote_auth entries for a user - # causes a lot of churn in database ids. Modify this logic to only - # update the remote_auth table when required. - $db->execute_query("DELETE FROM remote_auth WHERE user_id = ?", [$user_id]); - - if (ref($auth_names) eq 'ARRAY') { - foreach my $name (@$auth_names){ - if (length $name >=1) { - $query = "INSERT INTO remote_auth (auth_name, user_id) VALUES (?, ?)"; - $db->execute_query($query, [$name,$user_id]); - } + if (ref($auth_names) ne 'ARRAY') { + $auth_names = [$auth_names]; + } + + my $uindex = {}; + my $usernames = $db->execute_query("select * from remote_auth where user_id=?", [$user_id]); + if (!defined $usernames) { + warn "edit_user called on user without any known usernames."; + $usernames = []; + } + foreach my $u (@$usernames) { + $uindex->{$u->{auth_name}} = $u; + } + + foreach my $name (@$auth_names) { + if (defined $uindex->{$name}) { + delete $uindex->{$name}; + next; } - } else { - return (undef, 'Auth_Names is required to be at least 1 character.') if length $auth_names <1; - $query = "INSERT INTO remote_auth (auth_name, user_id) VALUES (?, ?)"; - $db->execute_query($query, [$auth_names, $user_id]); - } + + return (undef, 'auth_names is required to be at least 1 character.') if length $auth_names < 1; + + my $ok = $db->execute_query("INSERT INTO remote_auth (auth_name, user_id) VALUES (?, ?)", [$name, $user_id]); + return (undef, $db->get_error) if !defined $ok; + + delete $uindex->{$name}; + } + + foreach my $name (keys %$uindex) { + my $ok = $db->execute_query("delete from remote_auth where auth_name=?", [$name]); + return (undef, $db->get_error) if !defined $ok; + } return (1,undef) } diff --git a/perl-lib/OESS/t/z-DB/User.edit_user.01.t b/perl-lib/OESS/t/z-DB/User.edit_user.01.t new file mode 100644 index 000000000..a61a29923 --- /dev/null +++ b/perl-lib/OESS/t/z-DB/User.edit_user.01.t @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use FindBin; +my $path; + +BEGIN { + if ($FindBin::Bin =~ /(.*)/) { + $path = $1; + } +} +use lib "$path/.."; + +use Data::Dumper; +use Test::More tests => 28; + +use OESSDatabaseTester; + +use OESS::DB; +use OESS::DB::User; + +# Purpose: +# +# Verify user creation errors when bad type specified. + +OESSDatabaseTester::resetOESSDB( + config => "$path/../conf/database.xml", + dbdump => "$path/../conf/oess_known_state.sql" +); + +my $db = new OESS::DB( + config => "$path/../conf/database.xml" +); +my $model = { + given_names => 'Testerfield', + family_name => 'Testerton', + email => 'ttesterton@testertonestates.com', + username => 'ttesterton', + status => 'active' +}; + +my ($id, $err) = OESS::DB::User::add_user( + db => $db, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => $model->{username} +); +ok(defined $id, "User entry was created"); + +my $user = OESS::DB::User::fetch(db => $db, user_id => $id); +foreach my $key (keys %$model) { + ok($user->{$key} eq $model->{$key}, "got expected initial $key from db"); +} +$model->{family_name} = 'Please-Ignore'; + + +my ($res, $err2) = OESS::DB::User::edit_user( + db => $db, + user_id => $id, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => [ $model->{username}, 'ttesterton2', 'ttesterton3' ], + status => $model->{status} +); +print ("ERR = $err2") if defined $err2; +ok($res == 1, "Editing User was successful"); + + +my $index = { 'ttesterton' => 962, 'ttesterton2' => 963, 'ttesterton3' => 964 }; +my $users = $db->execute_query("select * from remote_auth where user_id=?", [$id]); + + +foreach my $user (@$users) { + my $name = $user->{auth_name}; + my $uid = $user->{auth_id}; + ok(defined $index->{$name}, "found expected username $name"); + ok($index->{$name} == $uid, "found expected user id $uid"); + delete $index->{$name}; +} + +ok(keys %$index == 0, "all expected usernames accounted for"); + + +my ($res3, $err3) = OESS::DB::User::edit_user( + db => $db, + user_id => $id, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => [ $model->{username}, 'ttesterton3' ], + status => $model->{status} +); +print ("ERR = $err3") if defined $err3; +ok($res3 == 1, "Editing User was successful"); + +$index = { 'ttesterton' => 962, 'ttesterton3' => 964 }; +$users = $db->execute_query("select * from remote_auth where user_id=?", [$id]); + +foreach my $user (@$users) { + my $name = $user->{auth_name}; + my $uid = $user->{auth_id}; + ok(defined $index->{$name}, "found expected username $name"); + ok($index->{$name} == $uid, "found expected user id $uid"); + delete $index->{$name}; +} + +ok(keys %$index == 0, "all expected usernames accounted for"); + + +my ($res4, $err4) = OESS::DB::User::edit_user( + db => $db, + user_id => $id, + given_name => $model->{given_names}, + family_name => $model->{family_name}, + email => $model->{email}, + auth_names => [ $model->{username}, 'ttesterton3', 'ttesterton4' ], + status => $model->{status} +); +print ("ERR = $err4") if defined $err4; +ok($res4 == 1, "Editing User was successful"); + +$index = { 'ttesterton' => 962, 'ttesterton3' => 964, 'ttesterton4' => 965 }; +$users = $db->execute_query("select * from remote_auth where user_id=?", [$id]); + +foreach my $user (@$users) { + my $name = $user->{auth_name}; + my $uid = $user->{auth_id}; + ok(defined $index->{$name}, "found expected username $name"); + ok($index->{$name} == $uid, "found expected user id $uid"); + delete $index->{$name}; +} + +ok(keys %$index == 0, "all expected usernames accounted for"); From b608e521e9a7a55c2f44303e70dfba41cf78ca92 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 23 Oct 2020 20:41:21 +0000 Subject: [PATCH 042/156] add ability to modify peer md5_key --- perl-lib/OESS/lib/OESS/DB/Peer.pm | 4 ++++ perl-lib/OESS/lib/OESS/Peer.pm | 6 +++++- perl-lib/OESS/t/z-DB/Peer.update.00.t | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/lib/OESS/DB/Peer.pm b/perl-lib/OESS/lib/OESS/DB/Peer.pm index 918030ff3..5416e6e0a 100644 --- a/perl-lib/OESS/lib/OESS/DB/Peer.pm +++ b/perl-lib/OESS/lib/OESS/DB/Peer.pm @@ -200,6 +200,10 @@ sub update { push @$params, 'bfd=?'; push @$values, $args->{peer}->{bfd}; } + if (defined $args->{peer}->{md5_key}) { + push @$params, 'md5_key=?'; + push @$values, $args->{peer}->{md5_key}; + } my $fields = join(', ', @$params); push @$values, $args->{peer}->{vrf_ep_peer_id}; diff --git a/perl-lib/OESS/lib/OESS/Peer.pm b/perl-lib/OESS/lib/OESS/Peer.pm index f6a65ccfc..a6a1e88b2 100644 --- a/perl-lib/OESS/lib/OESS/Peer.pm +++ b/perl-lib/OESS/lib/OESS/Peer.pm @@ -162,7 +162,11 @@ sub peer_asn{ =cut sub md5_key{ my $self = shift; - return $self->{'md5_key'}; + my $md5_key = shift; + if (defined $md5_key) { + $self->{md5_key} = $md5_key; + } + return $self->{md5_key}; } =head2 vrf_ep_id diff --git a/perl-lib/OESS/t/z-DB/Peer.update.00.t b/perl-lib/OESS/t/z-DB/Peer.update.00.t index 710725571..a0e913b31 100644 --- a/perl-lib/OESS/t/z-DB/Peer.update.00.t +++ b/perl-lib/OESS/t/z-DB/Peer.update.00.t @@ -55,6 +55,7 @@ warn $err if defined $err; $model->{vrf_ep_peer_id} = $id; $model->{local_ip} = '192.168.3.2/31'; $model->{peer_ip} = '192.168.3.3/31'; +$model->{md5_key} = 'a new key'; my $err3 = OESS::DB::Peer::update(db => $db, peer => $model); ok(!defined $err3, "no error on standard update"); From 8485c2f0072feac4163a3cf9221aa89b622fd2df Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Mon, 12 Apr 2021 17:25:42 +0000 Subject: [PATCH 043/156] #1098 - OESS now detects changes made via Azure portal and updates peering, adjusts OESS to match Azure. Assumes peering will always be on a /30 subnet and OESS will be the first free address and Azure will be the last free address (i.e. does not take network or boradcast address). logs changes to journalctl --follow Uses could_account_id, and cloud_interconnect_id to match peering. --- app/oess_pull_azure_interfaces.pl | 95 ++++++++++++++++++- .../OESS/lib/OESS/MPLS/Discovery/Paths.pm | 3 + 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/app/oess_pull_azure_interfaces.pl b/app/oess_pull_azure_interfaces.pl index 7f97e2981..ccdd42266 100755 --- a/app/oess_pull_azure_interfaces.pl +++ b/app/oess_pull_azure_interfaces.pl @@ -16,6 +16,8 @@ use OESS::DB::Endpoint; use OESS::Endpoint; +Log::Log4perl::init_and_watch('/etc/oess/logging.conf',10); + my $logger; sub main{ @@ -31,10 +33,17 @@ sub main{ cloud_interconnect_type => 'azure-express-route' ); + foreach my $cloud (@azure_cloud_accounts_config) { - my $azure_connections = ($azure->expressRouteCrossConnections($cloud->{interconnect_id})); - reconcile_oess_endpoints($db, $endpoints, $azure_connections); + my $connectionsWithNoPeering = ($azure->expressRouteCrossConnections($cloud->{interconnect_id})); + my $azure_connections = []; + foreach my $conn (@$connectionsWithNoPeering) { + my $connWithPeering = $azure->expressRouteCrossConnection($cloud->{interconnect_id}, $conn->{name}); + push($azure_connections, $connWithPeering); + } + reconcile_oess_endpoints($db, $endpoints, $azure_connections, $cloud->{interconnect_id}); } + } =head2 get_connection_by_id @@ -65,6 +74,8 @@ sub reconcile_oess_endpoints { my $db = shift; my $endpoints = shift; my $azure_connections = shift; + my $cloud_interconnect_id = shift; + my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); foreach my $endpoint (@$endpoints) { my $azure_connection = get_connection_by_id( @@ -73,12 +84,26 @@ sub reconcile_oess_endpoints { ); next if (!defined $azure_connection); + my $ep = new OESS::Endpoint(db => $db, model => $endpoint); + $ep->load_peers(); + next if(! $cloud_interconnect_id eq $ep->cloud_interconnect_id()); + + my $could_account_id = $azure_connection->{name}; + my $azure_subnet = find_matching_azure_connection($azure_connection, $cloud_interconnect_id); + my $endpoint_peer = get_endpoint_peer($ep, $cloud_interconnect_id, $could_account_id); + + next if(!defined $azure_subnet || !defined $endpoint_peer); + + if(!(increment_ip($endpoint_peer->{local_ip}, -1) eq $azure_subnet)){ + $logger->info("MISMATCH: on could_account_id: $could_account_id cloud_interconnect_id: $cloud_interconnect_id azure_subnet: $azure_subnet but OESS is peering on $endpoint_peer->{local_ip}"); + update_endpoint_peer_ips($db, $azure_subnet, $endpoint_peer); + } + my $cloud_bandwidth = $azure_connection->{properties}->{bandwidthInMbps}; if (!$cloud_bandwidth || $endpoint->{bandwidth} eq $cloud_bandwidth) { next; } - my $ep = new OESS::Endpoint(db => $db, model => $endpoint); $ep->bandwidth($cloud_bandwidth); my $error = $ep->update_db; @@ -88,6 +113,70 @@ sub reconcile_oess_endpoints { } } +sub find_matching_azure_connection{ + my $azure_connection_info = shift; + my $cloud_interconnect_id = shift; + my $azure_peering_subnet; + my $peering = $azure_connection_info->{properties}->{peerings}; + foreach my $ip (@$peering){ + if($cloud_interconnect_id =~ m/-SEC-/){ + $azure_peering_subnet = $ip->{properties}->{secondaryPeerAddresssubnet}; + }elsif($cloud_interconnect_id =~ m/-PRI-/){ + $azure_peering_subnet = $ip->{properties}->{primaryPeerAddresssubnet}; + }else{ + $azure_peering_subnet = ""; + } + } + return $azure_peering_subnet; +} + +sub get_endpoint_peer{ + my $endpoint = shift; + my $cloud_interconnect_id = shift; + my $cloud_account_id = shift; + my $peers = $endpoint->peers(); + my $endpoint_ips =[]; + if($endpoint->cloud_interconnect_id() eq $cloud_interconnect_id && $endpoint->cloud_account_id() eq $cloud_account_id){ + foreach my $peer (@$peers){ + my $cloud_connetions = $peer->{db}->{configuration}->{cloud}->{connection}; + foreach my $conn (@$cloud_connetions){ + if( $conn->{interconnect_id} eq $cloud_interconnect_id){ + return $peer; + } + } + } + } + return undef; +} + +sub update_endpoint_peer_ips{ + my $db = shift; + my $azure_subnet = shift; + my $peer = shift; + my $new_oess_ip = increment_ip($azure_subnet, 1); + my $new_azure_ip = increment_ip($azure_subnet, 2); + my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); + $logger->info("UPDATEING AZURE PEERING: changing $peer->{local_ip} to $new_oess_ip, and changing $peer->{peer_ip} to $new_azure_ip"); + $peer->{local_ip} = $new_oess_ip; + $peer->{peer_ip} = $new_azure_ip; + $peer->update($db, $peer->to_hash()); + +} + +sub increment_ip{ + my $ip = shift; + my $increment = shift; + $ip =~ m/^(\d\d?\d?)\.(\d\d?\d?)\.(\d\d?\d?)\.(\d\d?\d?)/; + my $firstOctet = $1; + my $secondOctet = $2; + my $thirdOctet = $3; + my $lastOctet = $4; + $ip =~ m/\/(\d\d?)$/; + my $subnet = $1; + $lastOctet = int($lastOctet) + $increment; + return "$firstOctet.$secondOctet.$thirdOctet.$lastOctet/$subnet"; +} + sub fetch_azure_cloud_account_configs{ my $config = shift; my @results = (); diff --git a/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm b/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm index a1c3855ea..8dd0f9b51 100644 --- a/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm +++ b/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm @@ -120,6 +120,9 @@ sub _process_paths{ my @ckt_path = map { $links->{$_} } keys(%$links); my $ckt = new OESS::L2Circuit(db => $self->{db2}, circuit_id => $circuit_id); + if(!defined $ckt){ + next; + } $ckt->load_paths; my $pri = $ckt->path(type => 'primary'); From cfd62c22b1ba4016944d2c34b16a720473083dd4 Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Mon, 12 Apr 2021 17:25:42 +0000 Subject: [PATCH 044/156] #1098 - OESS now detects changes made via Azure portal and updates peering, adjusts OESS to match Azure. Assumes peering will always be on a /30 subnet and OESS will be the first free address and Azure will be the last free address (i.e. does not take network or boradcast address). logs changes to journalctl --follow Uses could_account_id, and cloud_interconnect_id to match peering. --- app/oess_pull_azure_interfaces.pl | 95 ++++++++++++++++++- .../OESS/lib/OESS/MPLS/Discovery/Paths.pm | 3 + 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/app/oess_pull_azure_interfaces.pl b/app/oess_pull_azure_interfaces.pl index 7f97e2981..ccdd42266 100755 --- a/app/oess_pull_azure_interfaces.pl +++ b/app/oess_pull_azure_interfaces.pl @@ -16,6 +16,8 @@ use OESS::DB::Endpoint; use OESS::Endpoint; +Log::Log4perl::init_and_watch('/etc/oess/logging.conf',10); + my $logger; sub main{ @@ -31,10 +33,17 @@ sub main{ cloud_interconnect_type => 'azure-express-route' ); + foreach my $cloud (@azure_cloud_accounts_config) { - my $azure_connections = ($azure->expressRouteCrossConnections($cloud->{interconnect_id})); - reconcile_oess_endpoints($db, $endpoints, $azure_connections); + my $connectionsWithNoPeering = ($azure->expressRouteCrossConnections($cloud->{interconnect_id})); + my $azure_connections = []; + foreach my $conn (@$connectionsWithNoPeering) { + my $connWithPeering = $azure->expressRouteCrossConnection($cloud->{interconnect_id}, $conn->{name}); + push($azure_connections, $connWithPeering); + } + reconcile_oess_endpoints($db, $endpoints, $azure_connections, $cloud->{interconnect_id}); } + } =head2 get_connection_by_id @@ -65,6 +74,8 @@ sub reconcile_oess_endpoints { my $db = shift; my $endpoints = shift; my $azure_connections = shift; + my $cloud_interconnect_id = shift; + my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); foreach my $endpoint (@$endpoints) { my $azure_connection = get_connection_by_id( @@ -73,12 +84,26 @@ sub reconcile_oess_endpoints { ); next if (!defined $azure_connection); + my $ep = new OESS::Endpoint(db => $db, model => $endpoint); + $ep->load_peers(); + next if(! $cloud_interconnect_id eq $ep->cloud_interconnect_id()); + + my $could_account_id = $azure_connection->{name}; + my $azure_subnet = find_matching_azure_connection($azure_connection, $cloud_interconnect_id); + my $endpoint_peer = get_endpoint_peer($ep, $cloud_interconnect_id, $could_account_id); + + next if(!defined $azure_subnet || !defined $endpoint_peer); + + if(!(increment_ip($endpoint_peer->{local_ip}, -1) eq $azure_subnet)){ + $logger->info("MISMATCH: on could_account_id: $could_account_id cloud_interconnect_id: $cloud_interconnect_id azure_subnet: $azure_subnet but OESS is peering on $endpoint_peer->{local_ip}"); + update_endpoint_peer_ips($db, $azure_subnet, $endpoint_peer); + } + my $cloud_bandwidth = $azure_connection->{properties}->{bandwidthInMbps}; if (!$cloud_bandwidth || $endpoint->{bandwidth} eq $cloud_bandwidth) { next; } - my $ep = new OESS::Endpoint(db => $db, model => $endpoint); $ep->bandwidth($cloud_bandwidth); my $error = $ep->update_db; @@ -88,6 +113,70 @@ sub reconcile_oess_endpoints { } } +sub find_matching_azure_connection{ + my $azure_connection_info = shift; + my $cloud_interconnect_id = shift; + my $azure_peering_subnet; + my $peering = $azure_connection_info->{properties}->{peerings}; + foreach my $ip (@$peering){ + if($cloud_interconnect_id =~ m/-SEC-/){ + $azure_peering_subnet = $ip->{properties}->{secondaryPeerAddresssubnet}; + }elsif($cloud_interconnect_id =~ m/-PRI-/){ + $azure_peering_subnet = $ip->{properties}->{primaryPeerAddresssubnet}; + }else{ + $azure_peering_subnet = ""; + } + } + return $azure_peering_subnet; +} + +sub get_endpoint_peer{ + my $endpoint = shift; + my $cloud_interconnect_id = shift; + my $cloud_account_id = shift; + my $peers = $endpoint->peers(); + my $endpoint_ips =[]; + if($endpoint->cloud_interconnect_id() eq $cloud_interconnect_id && $endpoint->cloud_account_id() eq $cloud_account_id){ + foreach my $peer (@$peers){ + my $cloud_connetions = $peer->{db}->{configuration}->{cloud}->{connection}; + foreach my $conn (@$cloud_connetions){ + if( $conn->{interconnect_id} eq $cloud_interconnect_id){ + return $peer; + } + } + } + } + return undef; +} + +sub update_endpoint_peer_ips{ + my $db = shift; + my $azure_subnet = shift; + my $peer = shift; + my $new_oess_ip = increment_ip($azure_subnet, 1); + my $new_azure_ip = increment_ip($azure_subnet, 2); + my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); + $logger->info("UPDATEING AZURE PEERING: changing $peer->{local_ip} to $new_oess_ip, and changing $peer->{peer_ip} to $new_azure_ip"); + $peer->{local_ip} = $new_oess_ip; + $peer->{peer_ip} = $new_azure_ip; + $peer->update($db, $peer->to_hash()); + +} + +sub increment_ip{ + my $ip = shift; + my $increment = shift; + $ip =~ m/^(\d\d?\d?)\.(\d\d?\d?)\.(\d\d?\d?)\.(\d\d?\d?)/; + my $firstOctet = $1; + my $secondOctet = $2; + my $thirdOctet = $3; + my $lastOctet = $4; + $ip =~ m/\/(\d\d?)$/; + my $subnet = $1; + $lastOctet = int($lastOctet) + $increment; + return "$firstOctet.$secondOctet.$thirdOctet.$lastOctet/$subnet"; +} + sub fetch_azure_cloud_account_configs{ my $config = shift; my @results = (); diff --git a/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm b/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm index a1c3855ea..8dd0f9b51 100644 --- a/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm +++ b/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm @@ -120,6 +120,9 @@ sub _process_paths{ my @ckt_path = map { $links->{$_} } keys(%$links); my $ckt = new OESS::L2Circuit(db => $self->{db2}, circuit_id => $circuit_id); + if(!defined $ckt){ + next; + } $ckt->load_paths; my $pri = $ckt->path(type => 'primary'); From 0e8f540eb006e93a24731b1fdfaec5d0b15cc36a Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Wed, 14 Apr 2021 22:00:57 +0000 Subject: [PATCH 045/156] 1098 - general code quality improvment --- app/oess_pull_azure_interfaces.pl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/app/oess_pull_azure_interfaces.pl b/app/oess_pull_azure_interfaces.pl index ccdd42266..86857b513 100755 --- a/app/oess_pull_azure_interfaces.pl +++ b/app/oess_pull_azure_interfaces.pl @@ -18,10 +18,9 @@ Log::Log4perl::init_and_watch('/etc/oess/logging.conf',10); -my $logger; +my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer');; sub main{ - my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); my $config = OESS::Config->new(); my $db = OESS::DB->new(); @@ -75,7 +74,6 @@ sub reconcile_oess_endpoints { my $endpoints = shift; my $azure_connections = shift; my $cloud_interconnect_id = shift; - my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); foreach my $endpoint (@$endpoints) { my $azure_connection = get_connection_by_id( @@ -89,12 +87,12 @@ sub reconcile_oess_endpoints { next if(! $cloud_interconnect_id eq $ep->cloud_interconnect_id()); my $could_account_id = $azure_connection->{name}; - my $azure_subnet = find_matching_azure_connection($azure_connection, $cloud_interconnect_id); + my $azure_subnet = find_matching_azure_subnet($azure_connection, $cloud_interconnect_id); my $endpoint_peer = get_endpoint_peer($ep, $cloud_interconnect_id, $could_account_id); next if(!defined $azure_subnet || !defined $endpoint_peer); - - if(!(increment_ip($endpoint_peer->{local_ip}, -1) eq $azure_subnet)){ + + if(increment_ip($endpoint_peer->{local_ip}, -1) ne $azure_subnet){ $logger->info("MISMATCH: on could_account_id: $could_account_id cloud_interconnect_id: $cloud_interconnect_id azure_subnet: $azure_subnet but OESS is peering on $endpoint_peer->{local_ip}"); update_endpoint_peer_ips($db, $azure_subnet, $endpoint_peer); } @@ -113,7 +111,7 @@ sub reconcile_oess_endpoints { } } -sub find_matching_azure_connection{ +sub find_matching_azure_subnet{ my $azure_connection_info = shift; my $cloud_interconnect_id = shift; my $azure_peering_subnet; @@ -124,7 +122,7 @@ sub find_matching_azure_connection{ }elsif($cloud_interconnect_id =~ m/-PRI-/){ $azure_peering_subnet = $ip->{properties}->{primaryPeerAddresssubnet}; }else{ - $azure_peering_subnet = ""; + $azure_peering_subnet = undef; } } return $azure_peering_subnet; @@ -155,12 +153,10 @@ sub update_endpoint_peer_ips{ my $peer = shift; my $new_oess_ip = increment_ip($azure_subnet, 1); my $new_azure_ip = increment_ip($azure_subnet, 2); - my $logger = Log::Log4perl->get_logger('OESS.Cloud.Azure.Syncer'); $logger->info("UPDATEING AZURE PEERING: changing $peer->{local_ip} to $new_oess_ip, and changing $peer->{peer_ip} to $new_azure_ip"); $peer->{local_ip} = $new_oess_ip; $peer->{peer_ip} = $new_azure_ip; - $peer->update($db, $peer->to_hash()); - + $peer->update; } sub increment_ip{ From 733098d83a56902e902569813632da7b2bd1efb0 Mon Sep 17 00:00:00 2001 From: Kendric Hood <31542693+khood5@users.noreply.github.com> Date: Wed, 14 Apr 2021 18:12:25 -0400 Subject: [PATCH 046/156] 1225- Update frontend/webservice/vrf.cgi loading the vrf from the db Co-authored-by: Jonathan Stout --- frontend/webservice/vrf.cgi | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frontend/webservice/vrf.cgi b/frontend/webservice/vrf.cgi index 19e63bf20..617ee1e2a 100755 --- a/frontend/webservice/vrf.cgi +++ b/frontend/webservice/vrf.cgi @@ -322,10 +322,8 @@ sub provision_vrf{ my $previous_vrf = undef; if (defined $model->{'vrf_id'} && $model->{'vrf_id'} != -1) { - $model = {}; - $model->{'description'} = $params->{'description'}{'value'}; - $model->{'vrf_id'} = $params->{'vrf_id'}{'value'} || undef; - $vrf = OESS::VRF->new(db => $db, model => $model); + $vrf = OESS::VRF->new(db => $db, vrf_id => $model->{vrf_id}); + $vrf->description($params->{description}{value}); if (!defined $vrf) { $method->set_error("Couldn't load VRF"); $db->rollback; From e6e48d97f305a09bf17fe676edb23f56b63b717f Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Wed, 14 Apr 2021 22:23:25 +0000 Subject: [PATCH 047/156] Revert "1225- Update frontend/webservice/vrf.cgi " returns error, needs rework. This reverts commit 733098d83a56902e902569813632da7b2bd1efb0. --- frontend/webservice/vrf.cgi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/webservice/vrf.cgi b/frontend/webservice/vrf.cgi index 617ee1e2a..19e63bf20 100755 --- a/frontend/webservice/vrf.cgi +++ b/frontend/webservice/vrf.cgi @@ -322,8 +322,10 @@ sub provision_vrf{ my $previous_vrf = undef; if (defined $model->{'vrf_id'} && $model->{'vrf_id'} != -1) { - $vrf = OESS::VRF->new(db => $db, vrf_id => $model->{vrf_id}); - $vrf->description($params->{description}{value}); + $model = {}; + $model->{'description'} = $params->{'description'}{'value'}; + $model->{'vrf_id'} = $params->{'vrf_id'}{'value'} || undef; + $vrf = OESS::VRF->new(db => $db, model => $model); if (!defined $vrf) { $method->set_error("Couldn't load VRF"); $db->rollback; From 89c15a28acaae01cf4ea349931990bae4b83f6a1 Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Thu, 15 Apr 2021 20:54:54 +0000 Subject: [PATCH 048/156] 1225 - fix styling on l2 cricut --- frontend/www/html_templates/modify_l2vpn.html | 11 ++-- frontend/www/js_templates/l2/circuit.js | 32 ++++++---- frontend/www/js_templates/modify_l2vpn.js | 62 ++++++++++--------- .../OESS/lib/OESS/MPLS/Discovery/Paths.pm | 3 + 4 files changed, 58 insertions(+), 50 deletions(-) diff --git a/frontend/www/html_templates/modify_l2vpn.html b/frontend/www/html_templates/modify_l2vpn.html index b1f9ee0ed..599a4df3e 100644 --- a/frontend/www/html_templates/modify_l2vpn.html +++ b/frontend/www/html_templates/modify_l2vpn.html @@ -1,11 +1,8 @@
-
- - +
+
+
+
diff --git a/frontend/www/js_templates/l2/circuit.js b/frontend/www/js_templates/l2/circuit.js index acd5ada53..61bf49322 100644 --- a/frontend/www/js_templates/l2/circuit.js +++ b/frontend/www/js_templates/l2/circuit.js @@ -101,21 +101,27 @@ class CircuitHeader extends Component { } async render(props) { - let displayEdits = (props.editable) ? 'block' : 'none'; + let displayEdits = (props.editable) ? 'inline-block' : 'none'; return ` -
-

${props.description}

${props.connectionId} -
+
+

${props.description}
${props.connectionId}

+ + +
-
- - -
-`; +
+ + +
+ `; } } diff --git a/frontend/www/js_templates/modify_l2vpn.js b/frontend/www/js_templates/modify_l2vpn.js index 1956e9d58..bcda95960 100644 --- a/frontend/www/js_templates/modify_l2vpn.js +++ b/frontend/www/js_templates/modify_l2vpn.js @@ -109,40 +109,40 @@ document.querySelector('.l2vpn-new-endpoint-button').addEventListener('click', f modal.display(); }); -function doneEditingName(){ - let name = document.querySelector('#header-description'); +function doneEditingName(name){ + let description = document.querySelector('#header-description'); let button = document.getElementById("edit-description-button") - const newName = document.getElementById("description-input").value == "" ? state.circuit.description : document.getElementById("description-input").value - name.innerHTML = "

" + newName + "

" + description.innerHTML = "
" + name + "
" document.getElementById("change-description-button").hidden = true button.textContent = "Edit Name" } - -document.querySelector('.change-description-button').addEventListener('click', function(e) { - doneEditingName() -}) - -document.querySelector('.edit-description-button').addEventListener('click', function(e) { - let name = document.querySelector('#header-description'); - let button = document.getElementById("edit-description-button") - - if(button.textContent.trim() == "Edit Name"){ - name.innerHTML = "" - name.addEventListener("keyup", function(event) { - // Number 13 is the "Enter" key on the keyboard - if (event.keyCode === 13) { - event.preventDefault(); - doneEditingName() - } - }); - button.textContent = "Revert" - document.getElementById("change-description-button").hidden = false - }else{ - name.innerHTML = "

" + state.circuit.description + "

" - button.textContent = "Edit Name" - document.getElementById("change-description-button").hidden = true - } -}); +function addEditNameEvents(){ + document.querySelector('.change-description-button').addEventListener('click', function(e) { + const newName = document.getElementById("description-input").value == "" ? state.circuit.description : document.getElementById("description-input").value + doneEditingName(newName) + }) + + document.querySelector('.edit-description-button').addEventListener('click', function(e) { + let name = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + + if(button.textContent.trim() == "Edit Name"){ + name.innerHTML = "" + name.addEventListener("keyup", function(event) { + // Number 13 is the "Enter" key on the keyboard + if (event.keyCode === 13) { + event.preventDefault(); + const newName = document.getElementById("description-input").value == "" ? state.circuit.description : document.getElementById("description-input").value + doneEditingName(newName) + } + }); + button.textContent = "Revert" + document.getElementById("change-description-button").hidden = false + }else{ + doneEditingName(state.circuit.description) + } + }); +} let circuitHeader = null; let details = null; @@ -181,6 +181,8 @@ async function update(props) { let elem = NewEndpoint(e); list.appendChild(elem); }); + + addEditNameEvents() } document.addEventListener('DOMContentLoaded', async function() { diff --git a/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm b/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm index a1c3855ea..8dd0f9b51 100644 --- a/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm +++ b/perl-lib/OESS/lib/OESS/MPLS/Discovery/Paths.pm @@ -120,6 +120,9 @@ sub _process_paths{ my @ckt_path = map { $links->{$_} } keys(%$links); my $ckt = new OESS::L2Circuit(db => $self->{db2}, circuit_id => $circuit_id); + if(!defined $ckt){ + next; + } $ckt->load_paths; my $pri = $ckt->path(type => 'primary'); From 7c3007259f58ddc1afa0728854389cc1404fc198 Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Thu, 15 Apr 2021 21:20:44 +0000 Subject: [PATCH 049/156] 1225 - tighten up divs on header controls --- frontend/www/js_templates/l2/circuit.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/www/js_templates/l2/circuit.js b/frontend/www/js_templates/l2/circuit.js index 61bf49322..55e90a34b 100644 --- a/frontend/www/js_templates/l2/circuit.js +++ b/frontend/www/js_templates/l2/circuit.js @@ -104,7 +104,7 @@ class CircuitHeader extends Component { let displayEdits = (props.editable) ? 'inline-block' : 'none'; return ` -
+

${props.description}
${props.connectionId}

-
+
From 4adddefc629109f47e8e7c72a640550e1e3c513a Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Thu, 15 Apr 2021 22:10:02 +0000 Subject: [PATCH 050/156] 1225 - added edit name to L3 circuit --- frontend/www/html_templates/modify_cloud.html | 13 +++---- frontend/www/html_templates/modify_l2vpn.html | 1 + .../www/js_templates/edit_circuit_name.js | 34 ++++++++++++++++ frontend/www/js_templates/modify_cloud.js | 39 +------------------ frontend/www/js_templates/modify_l2vpn.js | 37 +----------------- 5 files changed, 42 insertions(+), 82 deletions(-) create mode 100644 frontend/www/js_templates/edit_circuit_name.js diff --git a/frontend/www/html_templates/modify_cloud.html b/frontend/www/html_templates/modify_cloud.html index 1d7f0bf2a..7cf8b9cfa 100644 --- a/frontend/www/html_templates/modify_cloud.html +++ b/frontend/www/html_templates/modify_cloud.html @@ -30,14 +30,10 @@
-
-
- - +
+
+
+

@@ -101,6 +97,7 @@ [% INCLUDE js_templates/l3/peering_modal.js %] [% INCLUDE js_templates/modify_cloud.js %] + [% INCLUDE js_templates/edit_circuit_name.js %] diff --git a/frontend/www/html_templates/modify_l2vpn.html b/frontend/www/html_templates/modify_l2vpn.html index 599a4df3e..c73f7d7f9 100644 --- a/frontend/www/html_templates/modify_l2vpn.html +++ b/frontend/www/html_templates/modify_l2vpn.html @@ -91,5 +91,6 @@ [% INCLUDE js_templates/l2/circuit.js %] [% INCLUDE js_templates/modify_l2vpn.js %] + [% INCLUDE js_templates/edit_circuit_name.js %]
diff --git a/frontend/www/js_templates/edit_circuit_name.js b/frontend/www/js_templates/edit_circuit_name.js new file mode 100644 index 000000000..7d688def9 --- /dev/null +++ b/frontend/www/js_templates/edit_circuit_name.js @@ -0,0 +1,34 @@ +function doneEditingName(name){ + let description = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + description.innerHTML = "
" + name + "
" + document.getElementById("change-description-button").hidden = true + button.textContent = "Edit Name" +} +function addEditNameEvents(default_name){ + document.querySelector('.change-description-button').addEventListener('click', function(e) { + const newName = document.getElementById("description-input").value == "" ? default_name : document.getElementById("description-input").value + doneEditingName(newName) + }) + + document.querySelector('.edit-description-button').addEventListener('click', function(e) { + let name = document.querySelector('#header-description'); + let button = document.getElementById("edit-description-button") + + if(button.textContent.trim() == "Edit Name"){ + name.innerHTML = "" + name.addEventListener("keyup", function(event) { + // Number 13 is the "Enter" key on the keyboard + if (event.keyCode === 13) { + event.preventDefault(); + const newName = document.getElementById("description-input").value == "" ? default_name: document.getElementById("description-input").value + doneEditingName(newName) + } + }); + button.textContent = "Revert" + document.getElementById("change-description-button").hidden = false + }else{ + doneEditingName(default_name) + } + }); +} \ No newline at end of file diff --git a/frontend/www/js_templates/modify_cloud.js b/frontend/www/js_templates/modify_cloud.js index 27a4f1367..fb98b692f 100644 --- a/frontend/www/js_templates/modify_cloud.js +++ b/frontend/www/js_templates/modify_cloud.js @@ -188,7 +188,7 @@ document.addEventListener('DOMContentLoaded', async function() { description: state.connection.description, editable: editable }); - + addEditNameEvents(state.connection.description); }); let addNetworkEndpoint = document.querySelector('#new-endpoint-button'); @@ -196,47 +196,10 @@ document.addEventListener('DOMContentLoaded', async function() { modal.display(null); }); - - function doneEditingName(){ - let name = document.querySelector('#header-description'); - let button = document.getElementById("edit-description-button") - const newName = document.getElementById("description-input").value == "" ? state.connection.description : document.getElementById("description-input").value - name.innerHTML = "

" + newName + "

" - document.getElementById("change-description-button").hidden = true - button.textContent = "Edit Name" - } - - document.querySelector('.change-description-button').addEventListener('click', function(e) { - doneEditingName() - }) - - document.querySelector('.edit-description-button').addEventListener('click', function(e) { - let name = document.querySelector('#header-description'); - let button = document.getElementById("edit-description-button") - - if(button.textContent.trim() == "Edit Name"){ - name.innerHTML = "" - name.addEventListener("keyup", function(event) { - // Number 13 is the "Enter" key on the keyboard - if (event.keyCode === 13) { - event.preventDefault(); - doneEditingName() - } - }); - button.textContent = "Revert" - document.getElementById("change-description-button").hidden = false - }else{ - name.innerHTML = "

" + state.connection.description + "

" - button.textContent = "Edit Name" - document.getElementById("change-description-button").hidden = true - } - }); - let map = new NDDIMap('map'); map.on("loaded", function(){ this.updateMapFromSession(session); }); - }); async function update() { diff --git a/frontend/www/js_templates/modify_l2vpn.js b/frontend/www/js_templates/modify_l2vpn.js index bcda95960..36620912b 100644 --- a/frontend/www/js_templates/modify_l2vpn.js +++ b/frontend/www/js_templates/modify_l2vpn.js @@ -109,41 +109,6 @@ document.querySelector('.l2vpn-new-endpoint-button').addEventListener('click', f modal.display(); }); -function doneEditingName(name){ - let description = document.querySelector('#header-description'); - let button = document.getElementById("edit-description-button") - description.innerHTML = "
" + name + "
" - document.getElementById("change-description-button").hidden = true - button.textContent = "Edit Name" -} -function addEditNameEvents(){ - document.querySelector('.change-description-button').addEventListener('click', function(e) { - const newName = document.getElementById("description-input").value == "" ? state.circuit.description : document.getElementById("description-input").value - doneEditingName(newName) - }) - - document.querySelector('.edit-description-button').addEventListener('click', function(e) { - let name = document.querySelector('#header-description'); - let button = document.getElementById("edit-description-button") - - if(button.textContent.trim() == "Edit Name"){ - name.innerHTML = "" - name.addEventListener("keyup", function(event) { - // Number 13 is the "Enter" key on the keyboard - if (event.keyCode === 13) { - event.preventDefault(); - const newName = document.getElementById("description-input").value == "" ? state.circuit.description : document.getElementById("description-input").value - doneEditingName(newName) - } - }); - button.textContent = "Revert" - document.getElementById("change-description-button").hidden = false - }else{ - doneEditingName(state.circuit.description) - } - }); -} - let circuitHeader = null; let details = null; let history = null; @@ -182,7 +147,7 @@ async function update(props) { list.appendChild(elem); }); - addEditNameEvents() + addEditNameEvents(state.circuit.description); } document.addEventListener('DOMContentLoaded', async function() { From 1af280fea372e758b5a71ef22921ce72d1fbd9f0 Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Fri, 16 Apr 2021 18:17:05 +0000 Subject: [PATCH 051/156] 1225 - added label fix input feild so it is inline --- frontend/www/js_templates/edit_circuit_name.js | 5 ++++- frontend/www/js_templates/l2/circuit.js | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/www/js_templates/edit_circuit_name.js b/frontend/www/js_templates/edit_circuit_name.js index 7d688def9..2c5bf348a 100644 --- a/frontend/www/js_templates/edit_circuit_name.js +++ b/frontend/www/js_templates/edit_circuit_name.js @@ -16,7 +16,10 @@ function addEditNameEvents(default_name){ let button = document.getElementById("edit-description-button") if(button.textContent.trim() == "Edit Name"){ - name.innerHTML = "" + name.innerHTML = ` + + + `; name.addEventListener("keyup", function(event) { // Number 13 is the "Enter" key on the keyboard if (event.keyCode === 13) { diff --git a/frontend/www/js_templates/l2/circuit.js b/frontend/www/js_templates/l2/circuit.js index 55e90a34b..39a312f86 100644 --- a/frontend/www/js_templates/l2/circuit.js +++ b/frontend/www/js_templates/l2/circuit.js @@ -105,7 +105,12 @@ class CircuitHeader extends Component { return `
-

${props.description}
${props.connectionId}

+

+
${props.description}
+ + ${props.connectionId} +

+ From 96accdbce0274e985b2d1d6243dd20a16147f782 Mon Sep 17 00:00:00 2001 From: Kendric Hood Date: Fri, 16 Apr 2021 19:21:34 +0000 Subject: [PATCH 052/156] 1225 - fix for attribute in label to match id --- frontend/www/js_templates/edit_circuit_name.js | 4 ++-- frontend/www/js_templates/l2/circuit.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/www/js_templates/edit_circuit_name.js b/frontend/www/js_templates/edit_circuit_name.js index 2c5bf348a..6385649e0 100644 --- a/frontend/www/js_templates/edit_circuit_name.js +++ b/frontend/www/js_templates/edit_circuit_name.js @@ -17,8 +17,8 @@ function addEditNameEvents(default_name){ if(button.textContent.trim() == "Edit Name"){ name.innerHTML = ` - - + + `; name.addEventListener("keyup", function(event) { // Number 13 is the "Enter" key on the keyboard diff --git a/frontend/www/js_templates/l2/circuit.js b/frontend/www/js_templates/l2/circuit.js index 39a312f86..76608e0c7 100644 --- a/frontend/www/js_templates/l2/circuit.js +++ b/frontend/www/js_templates/l2/circuit.js @@ -108,7 +108,7 @@ class CircuitHeader extends Component {

${props.description}
- ${props.connectionId} + ${props.connectionId}

- + Cloud Connect @@ -44,15 +44,15 @@ -
  • Explore
    • -
  • Workgroup
    • +
  • Explore
    • +
  • Workgroup
      • diff --git a/frontend/www/html_templates/workgroups.html b/frontend/www/html_templates/workgroups.html index 540f88a68..43d48d866 100644 --- a/frontend/www/html_templates/workgroups.html +++ b/frontend/www/html_templates/workgroups.html @@ -5,7 +5,6 @@
      -

      Available Workgroups

      Select a workgroup to use

      diff --git a/frontend/www/index.cgi b/frontend/www/index.cgi index eacd2c3d0..74c0954bf 100755 --- a/frontend/www/index.cgi +++ b/frontend/www/index.cgi @@ -43,49 +43,21 @@ use Log::Log4perl; Log::Log4perl::init('/etc/oess/logging.conf'); -my $config = new OESS::Config(); -my $db= OESS::Database->new(); - - -my $ADD_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"}, - {title => "Home", url => "?action=index"}, - {title => "Endpoints", url => "?action=endpoints"}, - {title => "Options", url => "?action=options"}, - {title => "Primary Path", url => "?action=primary_path"}, - {title => "Backup Path", url => "?action=backup_path"}, - {title => "Scheduling", url => "?action=scheduling"}, - {title => "Provisioning", url => "?action=provisioning"}, - ]; - -my $REMOVE_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"}, - {title => "Home", url => "?action=index"}, - {title => "Scheduling", url => "?action=remove_scheduling"}, - {title => "Provisioning", url => "?action=remove_provisioning"}, - ]; - -my $HOME_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"}, - {title => "Home", url => "?action=index"} - ]; - -my $DETAILS_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"},, - {title => "Home", url => "?action=index"}, - {title => "Circuit Details", url => "?action=view_details"}, - ]; sub main{ - + my $config = new OESS::Config(); + my $db = OESS::Database->new(); my $cgi = new CGI; - my $tt = Template->new(INCLUDE_PATH => "$FindBin::Bin") || die $Template::ERROR; my $is_valid_user = $db->get_user_id_by_auth_name( auth_name => $ENV{'REMOTE_USER'}); if(!defined($is_valid_user)){ - #-- What to pass to the TT and what http headers to send + #-- What to pass to the TT and what http headers to send my ($vars, $output, $filename, $title, $breadcrumbs, $current_breadcrumb); - $filename = "html_templates/denied.html"; - $title = "Access Denied"; + $filename = "html_templates/denied2.html"; + $title = "Access Denied"; $vars->{'admin_email'} = $db->get_admin_email(); $vars->{'page'} = $filename; @@ -93,146 +65,141 @@ sub main{ $vars->{'breadcrumbs'} = $breadcrumbs; $vars->{'current_breadcrumb'} = $current_breadcrumb; $vars->{'is_admin'} = 0; + $vars->{'path'} = "./"; $vars->{'is_read_only'} = 1; $vars->{'version'} = OESS::Database::VERSION; + $vars->{'network_type'} = $config->network_type; - $tt->process("html_templates/page_base.html", $vars, \$output) or warn $tt->error(); - print "Content-type: text/html\n\n" . $output; - return; + $tt->process("html_templates/base.html", $vars, \$output) or warn $tt->error(); + print "Content-type: text/html\n\n" . $output; + return; } - + my $is_admin = $db->get_user_admin_status(username=>$ENV{'REMOTE_USER'})->[0]{'is_admin'}; - if(!defined($is_admin)){ + if (!defined $is_admin) { $is_admin = 0; } - my $is_read_only =0; + my $is_read_only = 0; my $user = $db->get_user_by_id( user_id => $db->get_user_id_by_auth_name( auth_name => $ENV{'REMOTE_USER'}))->[0]; - + #-- What to pass to the TT and what http headers to send my ($vars, $output, $filename, $title, $breadcrumbs, $current_breadcrumb); - #-- Figure out what we're trying to templatize here or default to workgroups page. - my $action = "workgroups"; - + #-- Figure out what we're trying to templatize here or default to welcome page. + my $action = "welcome"; + if ($cgi->param('action') =~ /^(\w+)$/){ $action = $1; } - + if ($user->{'status'} eq 'decom') { - $action = "decom"; + $action = "decom"; } - switch ($action) { - - case "workgroups" { $filename = "html_templates/workgroups.html"; - $title = "Workgroups"; - $breadcrumbs = [{title => "Workgroups", url => "?action=workgroups"}]; - $current_breadcrumb = "Workgroups"; - } - case "index" { $filename = "html_templates/index.html"; - $title = "Home"; - $breadcrumbs = $HOME_BREADCRUMBS; - $current_breadcrumb = "Home"; - } - case "edit_details" { $filename = "html_templates/edit_details.html"; - $title = "Details"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Details"; - } - - case "loop_circuit" { $filename = "html_templates/loop_circuit.html"; - $title = "Loop Circuit"; - $breadcrumbs = $DETAILS_BREADCRUMBS; - $current_breadcrumb = "Loop Circuit"; - } - case "view_details" { $filename = "html_templates/view_details.html"; - $title = "Circuit Details"; - $breadcrumbs = $DETAILS_BREADCRUMBS; - $current_breadcrumb = "Circuit Details"; - } - case "interdomain" { $filename = "html_templates/interdomain.html"; - $title = "Interdomain Endpoints"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Endpoints"; - } - case "endpoints" { $filename = "html_templates/endpoints.html"; - $title = "Endpoints"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Endpoints"; - } - case "options" { $filename = "html_templates/options.html"; - $title = "Options"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Options"; - } - case "primary_path" { $filename = "html_templates/primary_path.html"; - $title = "Primary Path"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Primary Path"; - } - case "backup_path" { $filename = "html_templates/backup_path.html"; - $title = "Backup Path"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Backup Path"; - } - case "scheduling" { $filename = "html_templates/scheduling.html"; - $title = "Scheduling"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Scheduling"; - } - - case "provisioning" { - $filename = "html_templates/provisioning.html"; - $title = "Provisioning"; - $breadcrumbs = $ADD_BREADCRUMBS; - $current_breadcrumb = "Provisioning"; - } - case "remove_scheduling" { - $filename = "html_templates/remove_scheduling.html"; - $title = "Removal Scheduling"; - $breadcrumbs = $REMOVE_BREADCRUMBS; - $current_breadcrumb = "Scheduling"; - } - case "remove_provisioning" { - $filename = "html_templates/remove_provisioning.html"; - $title = "Removal Provisioning"; - $breadcrumbs = $REMOVE_BREADCRUMBS; - $current_breadcrumb = "Provisioning"; - } - case "decom" { - $filename = "html_templates/denied.html"; - $title = "Access Denied"; - } - case "about" { - $filename = "html_templates/splash.html"; - $title = "About"; - } + case "modify_l2vpn" { + $title = "Layer 2 Connection"; + $filename = "html_templates/modify_l2vpn.html"; + $current_breadcrumb = "Layer 2 Connection"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "Layer 2 Connection", url => "#"} + ]; + } + case "provision_l2vpn" { + $title = "New Layer 2 Connection"; + $filename = "html_templates/provision_l2vpn.html"; + $current_breadcrumb = "New Layer 2 Connection"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "New Layer 2 Connection", url => "#"} + ]; + } + case "modify_cloud" { + $title = "Layer 3 Connection"; + $filename = "html_templates/modify_cloud.html"; + $current_breadcrumb = "Layer 3 Connection"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "Layer 3 Connection", url => "#"} + ]; + } + case "provision_cloud" { + $title = "New Layer 3 Connection"; + $filename = "html_templates/provision_cloud.html"; + $current_breadcrumb = "New Layer 3 Connection"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "New Layer 3 Connection", url => "#"} + ]; + } + case "phonebook" { + $title = "Phonebook"; + $filename = "html_templates/phonebook.html"; + $current_breadcrumb = "Phonebook"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "Phonebook", url => "#"} + ]; + } + case "welcome" { + $title = "Welcome"; + $filename = "html_templates/welcome.html"; + $current_breadcrumb = "Welcome"; + $breadcrumbs = [ + {title => "Welcome", url => "#"} + ]; + } + case "acl" { + $filename = "html_templates/acl.html"; + $title = "Edit ACL"; + } + case "decom" { + $filename = "html_templates/denied.html"; + $title = "Access Denied"; + } + case "edit_entity" { + $filename = "html_templates/edit_entity.html"; + $current_breadcrumb = "Edit Entity"; + $title = "Edit Entity"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "Edit Entity", url => "#"} + ]; + } + case "add_entity" { + $filename = "html_templates/add_entity.html"; + $current_breadcrumb = "Add Entity"; + $title = "Add Entity"; + $breadcrumbs = [ + {title => "Welcome", url => "?action=welcome"}, + {title => "Add Entity", url => "#"} + ]; + } else { - $filename = "html_templates/error.html"; - $title = "Error"; - } - + $filename = "html_templates/error.html"; + $title = "Error"; + } } + + $vars->{'g_port'} = $db->{grafana}->{'oess-interface'}; + $vars->{'g_l2_port'} = $db->{grafana}->{'oess-l2-interface'}; + $vars->{'g_peer'} = $db->{grafana}->{'oess-bgp-peer'}; + $vars->{'g_route'} = $db->{grafana}->{'oess-routing-table'}; + $vars->{'admin_email'} = $db->get_admin_email(); - $vars->{'page'} = $filename; $vars->{'title'} = $title; $vars->{'breadcrumbs'} = $breadcrumbs; $vars->{'current_breadcrumb'} = $current_breadcrumb; - $vars->{'is_admin'} = $is_admin; + $vars->{'path'} = "./"; + $vars->{'is_admin'} = $is_admin; $vars->{'is_read_only'} = $is_read_only; $vars->{'version'} = OESS::Database::VERSION; $vars->{'network_type'} = $config->network_type; - - #print STDERR Dumper($vars); - if ($action eq 'view_l3vpn' || $action eq 'provision_cloud' || $action eq 'modify_cloud' || $action eq 'phonebook' || $action eq 'welcome') { - $tt->process("html_templates/base.html", $vars, \$output) or warn $tt->error(); - } else { - $tt->process("html_templates/page_base.html", $vars, \$output) or warn $tt->error(); - } + + $tt->process("html_templates/base.html", $vars, \$output) or warn $tt->error(); print "Content-type: text/html\n\n" . $output; } - main(); diff --git a/frontend/www/js_templates/welcome.js b/frontend/www/js_templates/welcome.js index 75dc91fbb..3c7f41412 100644 --- a/frontend/www/js_templates/welcome.js +++ b/frontend/www/js_templates/welcome.js @@ -68,7 +68,7 @@ async function loadEntityList() { let ok = true; if (entities.length === 0) { - html = '

      There are no Layer 3 Connections currently provisioned. Click here to create one.

      '; + html = '

      There are no Layer 3 Connections currently provisioned. Click here to create one.

      '; } entities.forEach(function(entity, index) { @@ -189,7 +189,7 @@ async function loadL2VPNs() { let ok = true; if (circuits.length === 0) { - html = '

      There are no Layer 2 Connections currently provisioned. Click here to create one.

      '; + html = '

      There are no Layer 2 Connections currently provisioned. Click here to create one.

      '; } circuits.forEach(function(circuit, index) { diff --git a/frontend/www/new/index.cgi b/frontend/www/new/index.cgi deleted file mode 100755 index d3f960b9a..000000000 --- a/frontend/www/new/index.cgi +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/perl - -#--------- -# This is the main workhorse service for serving up all the template toolkit -# pages that serve the OE-SS frontend. There is nothing particularly special in here, -# it just accepts what page the user wants to see and returns it templatized. -#--------- -##----- -##----- $HeadURL: svn+ssh://svn.grnoc.iu.edu/grnoc/oe-ss/frontend/trunk/www/index.cgi $ -##----- $Id$ -##----- $Date$ -##----- $LastChangedBy$ -##----- -##----- Provides object oriented methods to interact with the DBus test -## -##------------------------------------------------------------------------- -## -## -## Copyright 2011 Trustees of Indiana University -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -# -use strict; -use warnings; -use OESS::Config; -use OESS::Database; -use Data::Dumper; -use CGI; -use Template; -use Switch; -use FindBin; -use Log::Log4perl; - -Log::Log4perl::init('/etc/oess/logging.conf'); - - - -sub main{ - my $config = new OESS::Config(); - my $db = OESS::Database->new(); - my $cgi = new CGI; - my $tt = Template->new(INCLUDE_PATH => "$FindBin::Bin/..") || die $Template::ERROR; - - my $is_valid_user = $db->get_user_id_by_auth_name( auth_name => $ENV{'REMOTE_USER'}); - if(!defined($is_valid_user)){ - - #-- What to pass to the TT and what http headers to send - my ($vars, $output, $filename, $title, $breadcrumbs, $current_breadcrumb); - $filename = "html_templates/denied2.html"; - $title = "Access Denied"; - $vars->{'admin_email'} = $db->get_admin_email(); - - $vars->{'page'} = $filename; - $vars->{'title'} = $title; - $vars->{'breadcrumbs'} = $breadcrumbs; - $vars->{'current_breadcrumb'} = $current_breadcrumb; - $vars->{'is_admin'} = 0; - $vars->{'path'} = "../"; - $vars->{'is_read_only'} = 1; - $vars->{'version'} = OESS::Database::VERSION; - $vars->{'network_type'} = $config->network_type; - - $tt->process("html_templates/base.html", $vars, \$output) or warn $tt->error(); - print "Content-type: text/html\n\n" . $output; - return; - } - - my $is_admin = $db->get_user_admin_status(username=>$ENV{'REMOTE_USER'})->[0]{'is_admin'}; - if (!defined $is_admin) { - $is_admin = 0; - } - my $is_read_only = 0; - my $user = $db->get_user_by_id( user_id => $db->get_user_id_by_auth_name( auth_name => $ENV{'REMOTE_USER'}))->[0]; - - #-- What to pass to the TT and what http headers to send - my ($vars, $output, $filename, $title, $breadcrumbs, $current_breadcrumb); - - #-- Figure out what we're trying to templatize here or default to welcome page. - my $action = "welcome"; - - if ($cgi->param('action') =~ /^(\w+)$/){ - $action = $1; - } - - if ($user->{'status'} eq 'decom') { - $action = "decom"; - } - - switch ($action) { - case "modify_l2vpn" { - $title = "Layer 2 Connection"; - $filename = "html_templates/modify_l2vpn.html"; - $current_breadcrumb = "Layer 2 Connection"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "Layer 2 Connection", url => "#"} - ]; - } - case "provision_l2vpn" { - $title = "New Layer 2 Connection"; - $filename = "html_templates/provision_l2vpn.html"; - $current_breadcrumb = "New Layer 2 Connection"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "New Layer 2 Connection", url => "#"} - ]; - } - case "modify_cloud" { - $title = "Layer 3 Connection"; - $filename = "html_templates/modify_cloud.html"; - $current_breadcrumb = "Layer 3 Connection"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "Layer 3 Connection", url => "#"} - ]; - } - case "provision_cloud" { - $title = "New Layer 3 Connection"; - $filename = "html_templates/provision_cloud.html"; - $current_breadcrumb = "New Layer 3 Connection"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "New Layer 3 Connection", url => "#"} - ]; - } - case "phonebook" { - $title = "Phonebook"; - $filename = "html_templates/phonebook.html"; - $current_breadcrumb = "Phonebook"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "Phonebook", url => "#"} - ]; - } - case "welcome" { - $title = "Welcome"; - $filename = "html_templates/welcome.html"; - $current_breadcrumb = "Welcome"; - $breadcrumbs = [ - {title => "Welcome", url => "#"} - ]; - } - case "acl" { - $filename = "html_templates/acl.html"; - $title = "Edit ACL"; - } - case "decom" { - $filename = "html_templates/denied.html"; - $title = "Access Denied"; - } - case "edit_entity" { - $filename = "html_templates/edit_entity.html"; - $current_breadcrumb = "Edit Entity"; - $title = "Edit Entity"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "Edit Entity", url => "#"} - ]; - } - case "add_entity" { - $filename = "html_templates/add_entity.html"; - $current_breadcrumb = "Add Entity"; - $title = "Add Entity"; - $breadcrumbs = [ - {title => "Welcome", url => "?action=welcome"}, - {title => "Add Entity", url => "#"} - ]; - } - else { - $filename = "html_templates/error.html"; - $title = "Error"; - } - } - - $vars->{'g_port'} = $db->{grafana}->{'oess-interface'}; - $vars->{'g_l2_port'} = $db->{grafana}->{'oess-l2-interface'}; - $vars->{'g_peer'} = $db->{grafana}->{'oess-bgp-peer'}; - $vars->{'g_route'} = $db->{grafana}->{'oess-routing-table'}; - - $vars->{'admin_email'} = $db->get_admin_email(); - $vars->{'page'} = $filename; - $vars->{'title'} = $title; - $vars->{'breadcrumbs'} = $breadcrumbs; - $vars->{'current_breadcrumb'} = $current_breadcrumb; - $vars->{'path'} = "../"; - $vars->{'is_admin'} = $is_admin; - $vars->{'is_read_only'} = $is_read_only; - $vars->{'version'} = OESS::Database::VERSION; - $vars->{'network_type'} = $config->network_type; - - $tt->process("html_templates/base.html", $vars, \$output) or warn $tt->error(); - print "Content-type: text/html\n\n" . $output; -} - -main(); diff --git a/frontend/www/old.cgi b/frontend/www/old.cgi new file mode 100755 index 000000000..eacd2c3d0 --- /dev/null +++ b/frontend/www/old.cgi @@ -0,0 +1,238 @@ +#!/usr/bin/perl + +#--------- +# This is the main workhorse service for serving up all the template toolkit +# pages that serve the OE-SS frontend. There is nothing particularly special in here, +# it just accepts what page the user wants to see and returns it templatized. +#--------- +##----- +##----- $HeadURL: svn+ssh://svn.grnoc.iu.edu/grnoc/oe-ss/frontend/trunk/www/index.cgi $ +##----- $Id$ +##----- $Date$ +##----- $LastChangedBy$ +##----- +##----- Provides object oriented methods to interact with the DBus test +## +##------------------------------------------------------------------------- +## +## +## Copyright 2011 Trustees of Indiana University +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +# +use strict; +use warnings; +use OESS::Config; +use OESS::Database; +use Data::Dumper; +use CGI; +use Template; +use Switch; +use FindBin; +use Log::Log4perl; + +Log::Log4perl::init('/etc/oess/logging.conf'); + +my $config = new OESS::Config(); +my $db= OESS::Database->new(); + + +my $ADD_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"}, + {title => "Home", url => "?action=index"}, + {title => "Endpoints", url => "?action=endpoints"}, + {title => "Options", url => "?action=options"}, + {title => "Primary Path", url => "?action=primary_path"}, + {title => "Backup Path", url => "?action=backup_path"}, + {title => "Scheduling", url => "?action=scheduling"}, + {title => "Provisioning", url => "?action=provisioning"}, + ]; + +my $REMOVE_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"}, + {title => "Home", url => "?action=index"}, + {title => "Scheduling", url => "?action=remove_scheduling"}, + {title => "Provisioning", url => "?action=remove_provisioning"}, + ]; + +my $HOME_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"}, + {title => "Home", url => "?action=index"} + ]; + +my $DETAILS_BREADCRUMBS = [{title => "Workgroups", url => "?action=workgroups"},, + {title => "Home", url => "?action=index"}, + {title => "Circuit Details", url => "?action=view_details"}, + ]; + + +sub main{ + + my $cgi = new CGI; + + my $tt = Template->new(INCLUDE_PATH => "$FindBin::Bin") || die $Template::ERROR; + + my $is_valid_user = $db->get_user_id_by_auth_name( auth_name => $ENV{'REMOTE_USER'}); + if(!defined($is_valid_user)){ + + #-- What to pass to the TT and what http headers to send + my ($vars, $output, $filename, $title, $breadcrumbs, $current_breadcrumb); + $filename = "html_templates/denied.html"; + $title = "Access Denied"; + $vars->{'admin_email'} = $db->get_admin_email(); + + $vars->{'page'} = $filename; + $vars->{'title'} = $title; + $vars->{'breadcrumbs'} = $breadcrumbs; + $vars->{'current_breadcrumb'} = $current_breadcrumb; + $vars->{'is_admin'} = 0; + $vars->{'is_read_only'} = 1; + $vars->{'version'} = OESS::Database::VERSION; + + $tt->process("html_templates/page_base.html", $vars, \$output) or warn $tt->error(); + print "Content-type: text/html\n\n" . $output; + return; + } + + my $is_admin = $db->get_user_admin_status(username=>$ENV{'REMOTE_USER'})->[0]{'is_admin'}; + if(!defined($is_admin)){ + $is_admin = 0; + } + my $is_read_only =0; + my $user = $db->get_user_by_id( user_id => $db->get_user_id_by_auth_name( auth_name => $ENV{'REMOTE_USER'}))->[0]; + + #-- What to pass to the TT and what http headers to send + my ($vars, $output, $filename, $title, $breadcrumbs, $current_breadcrumb); + + #-- Figure out what we're trying to templatize here or default to workgroups page. + my $action = "workgroups"; + + if ($cgi->param('action') =~ /^(\w+)$/){ + $action = $1; + } + + if ($user->{'status'} eq 'decom') { + $action = "decom"; + } + + + switch ($action) { + + case "workgroups" { $filename = "html_templates/workgroups.html"; + $title = "Workgroups"; + $breadcrumbs = [{title => "Workgroups", url => "?action=workgroups"}]; + $current_breadcrumb = "Workgroups"; + } + case "index" { $filename = "html_templates/index.html"; + $title = "Home"; + $breadcrumbs = $HOME_BREADCRUMBS; + $current_breadcrumb = "Home"; + } + case "edit_details" { $filename = "html_templates/edit_details.html"; + $title = "Details"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Details"; + } + + case "loop_circuit" { $filename = "html_templates/loop_circuit.html"; + $title = "Loop Circuit"; + $breadcrumbs = $DETAILS_BREADCRUMBS; + $current_breadcrumb = "Loop Circuit"; + } + case "view_details" { $filename = "html_templates/view_details.html"; + $title = "Circuit Details"; + $breadcrumbs = $DETAILS_BREADCRUMBS; + $current_breadcrumb = "Circuit Details"; + } + case "interdomain" { $filename = "html_templates/interdomain.html"; + $title = "Interdomain Endpoints"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Endpoints"; + } + case "endpoints" { $filename = "html_templates/endpoints.html"; + $title = "Endpoints"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Endpoints"; + } + case "options" { $filename = "html_templates/options.html"; + $title = "Options"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Options"; + } + case "primary_path" { $filename = "html_templates/primary_path.html"; + $title = "Primary Path"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Primary Path"; + } + case "backup_path" { $filename = "html_templates/backup_path.html"; + $title = "Backup Path"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Backup Path"; + } + case "scheduling" { $filename = "html_templates/scheduling.html"; + $title = "Scheduling"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Scheduling"; + } + + case "provisioning" { + $filename = "html_templates/provisioning.html"; + $title = "Provisioning"; + $breadcrumbs = $ADD_BREADCRUMBS; + $current_breadcrumb = "Provisioning"; + } + case "remove_scheduling" { + $filename = "html_templates/remove_scheduling.html"; + $title = "Removal Scheduling"; + $breadcrumbs = $REMOVE_BREADCRUMBS; + $current_breadcrumb = "Scheduling"; + } + case "remove_provisioning" { + $filename = "html_templates/remove_provisioning.html"; + $title = "Removal Provisioning"; + $breadcrumbs = $REMOVE_BREADCRUMBS; + $current_breadcrumb = "Provisioning"; + } + case "decom" { + $filename = "html_templates/denied.html"; + $title = "Access Denied"; + } + case "about" { + $filename = "html_templates/splash.html"; + $title = "About"; + } + else { + $filename = "html_templates/error.html"; + $title = "Error"; + } + + } + $vars->{'admin_email'} = $db->get_admin_email(); + + $vars->{'page'} = $filename; + $vars->{'title'} = $title; + $vars->{'breadcrumbs'} = $breadcrumbs; + $vars->{'current_breadcrumb'} = $current_breadcrumb; + $vars->{'is_admin'} = $is_admin; + $vars->{'is_read_only'} = $is_read_only; + $vars->{'version'} = OESS::Database::VERSION; + $vars->{'network_type'} = $config->network_type; + + #print STDERR Dumper($vars); + if ($action eq 'view_l3vpn' || $action eq 'provision_cloud' || $action eq 'modify_cloud' || $action eq 'phonebook' || $action eq 'welcome') { + $tt->process("html_templates/base.html", $vars, \$output) or warn $tt->error(); + } else { + $tt->process("html_templates/page_base.html", $vars, \$output) or warn $tt->error(); + } + print "Content-type: text/html\n\n" . $output; +} + + +main(); From fc3cb542e0432ef4df2f4943f587e16293c49721 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Tue, 27 Jul 2021 22:36:07 +0000 Subject: [PATCH 142/156] disable provisioning of multi-point l2 connections --- frontend/webservice/circuit.cgi | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frontend/webservice/circuit.cgi b/frontend/webservice/circuit.cgi index 4ce79be92..08d09d181 100755 --- a/frontend/webservice/circuit.cgi +++ b/frontend/webservice/circuit.cgi @@ -274,6 +274,11 @@ sub provision { return; } + if (@{$args->{endpoint}->{value}} > 2) { + $method->set_error("Support for Multi-Point Layer 2 Connections is currently disabled. Please contact your OESS administrator for more information."); + return; + } + # Endpoint: { entity: 'entity name', bandwidth: 0, tag: 100, inner_tag: 100, peerings: [{ version: 4 }] } foreach my $value (@{$args->{endpoint}->{value}}) { my $ep; @@ -504,6 +509,11 @@ sub update { my $add_endpoints = []; my $del_endpoints = []; + if (@{$args->{endpoint}->{value}} > 2) { + $method->set_error("Support for Multi-Point Layer 2 Connections is currently disabled. Please contact your OESS administrator for more information."); + return; + } + foreach my $value (@{$args->{endpoint}->{value}}) { my $ep; eval{ From 22bc743fea32b6c1044d081ce56634e6805a7fe6 Mon Sep 17 00:00:00 2001 From: Ben Siefers Date: Wed, 28 Jul 2021 18:12:38 +0000 Subject: [PATCH 143/156] Updated version number --- app/Makefile | 2 +- app/oess-core.spec | 4 ++-- frontend/Makefile | 2 +- frontend/oess-frontend.spec | 6 +++--- oess.spec | 6 +++--- perl-lib/OESS/MANIFEST | 2 +- perl-lib/OESS/perl-OESS.spec | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/Makefile b/app/Makefile index cc8946711..ad3702f6e 100644 --- a/app/Makefile +++ b/app/Makefile @@ -1,5 +1,5 @@ NAME= oess-core -VERSION = 2.0.11 +VERSION = 2.0.12 rpm: dist rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz diff --git a/app/oess-core.spec b/app/oess-core.spec index 4ed6a2338..8c00058e7 100644 --- a/app/oess-core.spec +++ b/app/oess-core.spec @@ -1,5 +1,5 @@ Name: oess-core -Version: 2.0.11 +Version: 2.0.12 Release: 3%{?dist} Summary: The core OESS service providers @@ -20,7 +20,7 @@ Requires: /bin/bash Requires: /usr/bin/perl Requires: perl(base), perl(constant), perl(strict), perl(warnings) -Requires: perl-OESS >= 2.0.11 +Requires: perl-OESS >= 2.0.12 Requires: perl(AnyEvent), perl(AnyEvent::DBus), perl(AnyEvent::RabbitMQ) Requires: perl(CPAN), perl(CPAN::Shell) diff --git a/frontend/Makefile b/frontend/Makefile index 4343d4b94..c5fda76c5 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -1,5 +1,5 @@ NAME= oess-frontend -VERSION = 2.0.11 +VERSION = 2.0.12 rpm: dist rpmbuild -ta dist/$(NAME)-$(VERSION).tar.gz diff --git a/frontend/oess-frontend.spec b/frontend/oess-frontend.spec index 92431403f..2e787a8ff 100644 --- a/frontend/oess-frontend.spec +++ b/frontend/oess-frontend.spec @@ -1,5 +1,5 @@ Name: oess-frontend -Version: 2.0.11 +Version: 2.0.12 Release: 3%{?dist} Summary: The OESS webservices and user interface @@ -13,14 +13,14 @@ BuildRequires: perl BuildRequires: python >= 2.6, python-libs >= 2.6 BuildRequires: python-simplejson -Requires: oess-core >= 2.0.11 +Requires: oess-core >= 2.0.12 Requires: yui Requires: httpd, mod_ssl Requires: nddi-tiles Requires: perl-Crypt-SSLeay Requires: xmlsec1, xmlsec1-openssl -Requires: perl-OESS >= 2.0.11 +Requires: perl-OESS >= 2.0.12 Requires: perl(strict), perl(warnings) Requires: perl(AnyEvent) diff --git a/oess.spec b/oess.spec index 98db9d10f..af8223c89 100644 --- a/oess.spec +++ b/oess.spec @@ -1,14 +1,14 @@ Summary: OESS Metapackage Name: oess -Version: 2.0.11 +Version: 2.0.12 Release: 3%{?dist} License: Apache Group: GRNOC #Source: URL: http://globalnoc.iu.edu Buildroot: %{_tmppath}/%{name}-root -Requires: oess-core >= 2.0.11 -Requires: oess-frontend >= 2.0.11 +Requires: oess-core >= 2.0.12 +Requires: oess-frontend >= 2.0.12 %description Package that installs all of the OESS packages diff --git a/perl-lib/OESS/MANIFEST b/perl-lib/OESS/MANIFEST index 6e860155a..5e7a3e0c3 100644 --- a/perl-lib/OESS/MANIFEST +++ b/perl-lib/OESS/MANIFEST @@ -82,7 +82,7 @@ lib/OESS/Webservice.pm lib/OESS/Workgroup.pm Makefile.PL MANIFEST This list of files -perl-OESS-2.0.11.tar.gz +perl-OESS-2.0.12.tar.gz perl-OESS.spec share/customer-templates/cisco/2900/ios124/template.txt share/customer-templates/cisco/3700/ios124/template.txt diff --git a/perl-lib/OESS/perl-OESS.spec b/perl-lib/OESS/perl-OESS.spec index 653d36f61..6870004fc 100644 --- a/perl-lib/OESS/perl-OESS.spec +++ b/perl-lib/OESS/perl-OESS.spec @@ -1,6 +1,6 @@ Summary: OESS Perl Libraries Name: perl-OESS -Version: 2.0.11 +Version: 2.0.12 Release: 3%{?dist} License: APL 2.0 Group: Network From 1b057237e65f69bacdcc69f27178942ea9f749ea Mon Sep 17 00:00:00 2001 From: Ben Siefers Date: Wed, 28 Jul 2021 18:21:58 +0000 Subject: [PATCH 144/156] release versions changed from 3 to 1 --- app/oess-core.spec | 2 +- frontend/oess-frontend.spec | 2 +- oess.spec | 2 +- perl-lib/OESS/perl-OESS.spec | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/oess-core.spec b/app/oess-core.spec index 8c00058e7..5864293b4 100644 --- a/app/oess-core.spec +++ b/app/oess-core.spec @@ -1,6 +1,6 @@ Name: oess-core Version: 2.0.12 -Release: 3%{?dist} +Release: 1%{?dist} Summary: The core OESS service providers Group: Network diff --git a/frontend/oess-frontend.spec b/frontend/oess-frontend.spec index 2e787a8ff..81e95ccde 100644 --- a/frontend/oess-frontend.spec +++ b/frontend/oess-frontend.spec @@ -1,6 +1,6 @@ Name: oess-frontend Version: 2.0.12 -Release: 3%{?dist} +Release: 1%{?dist} Summary: The OESS webservices and user interface Group: Network diff --git a/oess.spec b/oess.spec index af8223c89..01f6d6fe1 100644 --- a/oess.spec +++ b/oess.spec @@ -1,7 +1,7 @@ Summary: OESS Metapackage Name: oess Version: 2.0.12 -Release: 3%{?dist} +Release: 1%{?dist} License: Apache Group: GRNOC #Source: diff --git a/perl-lib/OESS/perl-OESS.spec b/perl-lib/OESS/perl-OESS.spec index 6870004fc..703b3f314 100644 --- a/perl-lib/OESS/perl-OESS.spec +++ b/perl-lib/OESS/perl-OESS.spec @@ -1,7 +1,7 @@ Summary: OESS Perl Libraries Name: perl-OESS Version: 2.0.12 -Release: 3%{?dist} +Release: 1%{?dist} License: APL 2.0 Group: Network URL: http://globalnoc.iu.edu From 1d55d6243fb76694add25ba040f896ffa4c12b32 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Wed, 28 Jul 2021 16:32:07 -0400 Subject: [PATCH 145/156] add OESS/RabbitMQ/Topic.pm to spec file --- perl-lib/OESS/perl-OESS.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perl-lib/OESS/perl-OESS.spec b/perl-lib/OESS/perl-OESS.spec index 703b3f314..41bd11ace 100644 --- a/perl-lib/OESS/perl-OESS.spec +++ b/perl-lib/OESS/perl-OESS.spec @@ -212,6 +212,7 @@ rm -rf $RPM_BUILD_ROOT %doc %{_mandir}/man3/OESS::NSI::Utils.3pm.gz %doc %{_mandir}/man3/OESS::RabbitMQ::Client.3pm.gz %doc %{_mandir}/man3/OESS::RabbitMQ::Dispatcher.3pm.gz +%doc %{_mandir}/man3/OESS::RabbitMQ::Topic.3pm.gz %doc %{_mandir}/man3/OESS::Topology.3pm.gz %doc %{_mandir}/man3/OESS::Traceroute.3pm.gz %doc %{_mandir}/man3/OESS::Watchdog.3pm.gz @@ -291,6 +292,7 @@ rm -rf $RPM_BUILD_ROOT %{perl_vendorlib}/OESS/NSI/Utils.pm %{perl_vendorlib}/OESS/RabbitMQ/Client.pm %{perl_vendorlib}/OESS/RabbitMQ/Dispatcher.pm +%{perl_vendorlib}/OESS/RabbitMQ/Topic.pm %{perl_vendorlib}/OESS/Topology.pm %{perl_vendorlib}/OESS/Traceroute.pm %{perl_vendorlib}/OESS/Watchdog.pm From d57edc42fc860c8e0c751b57591e36b9c25d9a05 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Wed, 28 Jul 2021 16:32:55 -0400 Subject: [PATCH 146/156] add OESS/RabbitMQ/Topic.pm to manifest --- perl-lib/OESS/MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/perl-lib/OESS/MANIFEST b/perl-lib/OESS/MANIFEST index 5e7a3e0c3..6a0d13892 100644 --- a/perl-lib/OESS/MANIFEST +++ b/perl-lib/OESS/MANIFEST @@ -73,6 +73,7 @@ lib/OESS/Path.pm lib/OESS/Peer.pm lib/OESS/RabbitMQ/Client.pm lib/OESS/RabbitMQ/Dispatcher.pm +lib/OESS/RabbitMQ/Topic.pm lib/OESS/Topology.pm lib/OESS/Traceroute.pm lib/OESS/User.pm From 92f0f9f715229f3de7a6991e995551e57335f3a8 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Thu, 29 Jul 2021 02:11:51 +0000 Subject: [PATCH 147/156] manually set header to avoid lwp issue with certs --- perl-lib/OESS/lib/OESS/NSO/Client.pm | 16 +++++++++-- perl-lib/OESS/t/00-load_base.t | 42 ++++++++++++++++------------ 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 8c89cb822..3b8aa9a7b 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -37,9 +37,19 @@ sub new { $self->{local_asn} = $self->{config_obj}->local_as; $self->{www} = new LWP::UserAgent; - my $host = $self->{config_obj}->nso_host; - $host =~ s/http(s){0,1}:\/\///g; # Strip http:// or https:// from string - $self->{www}->credentials($host, "restconf", $self->{config_obj}->nso_username, $self->{config_obj}->nso_password); + + # We replace the traditional $ua->credentials approach to basic # + # auth as it fails to work with self-signed certs. + my $username = $self->{config_obj}->nso_username; + my $password = $self->{config_obj}->nso_password; + + my $userpass = "$username:$password"; + $userpass = Encode::encode("UTF-8", "$username:$password"); + my $credentials = MIME::Base64::encode($userpass, ''); + + $self->{www}->default_header('Authorization' => "Basic $credentials"); + $self->{www}->default_header('www-authenticate' => 'Basic realm="restconf", charset="UTF-8"'); + $self->{www}->ssl_opts(verify_hostname => 0); return $self; } diff --git a/perl-lib/OESS/t/00-load_base.t b/perl-lib/OESS/t/00-load_base.t index 9f00e8958..e430d5a3f 100644 --- a/perl-lib/OESS/t/00-load_base.t +++ b/perl-lib/OESS/t/00-load_base.t @@ -1,25 +1,31 @@ #!perl -T -use Test::More tests => 17; +use Test::More tests => 23; BEGIN { - use_ok( 'OESS::Database' ); - use_ok( 'OESS::Topology' ); - use_ok( 'OESS::DBus' ); - use_ok( 'OESS::FlowRule' ); - use_ok( 'OESS::Circuit' ); - use_ok( 'OESS::Measurement' ); - use_ok( 'OESS::Notification' ); - use_ok( 'OESS::Traceroute'); - use_ok( 'OESS::MPLS::FWDCTL'); - use_ok( 'OESS::MPLS::Switch'); - use_ok( 'OESS::MPLS::Discovery'); - use_ok( 'OESS::MPLS::Device'); - use_ok( 'OESS::MPLS::Device::Juniper::MX'); - use_ok( 'OESS::MPLS::Discovery::Interface'); - use_ok( 'OESS::MPLS::Discovery::ISIS'); - use_ok( 'OESS::MPLS::Discovery::Paths'); - use_ok( 'OESS::MPLS::Topology'); + use_ok( 'OESS::Database' ); + use_ok( 'OESS::Topology' ); + use_ok( 'OESS::DBus' ); + use_ok( 'OESS::FlowRule' ); + use_ok( 'OESS::Circuit' ); + use_ok( 'OESS::Measurement' ); + use_ok( 'OESS::Notification' ); + use_ok( 'OESS::Traceroute'); + use_ok( 'OESS::MPLS::FWDCTL'); + use_ok( 'OESS::MPLS::Switch'); + use_ok( 'OESS::MPLS::Discovery'); + use_ok( 'OESS::MPLS::Device'); + use_ok( 'OESS::MPLS::Device::Juniper::MX'); + use_ok( 'OESS::MPLS::Discovery::Interface'); + use_ok( 'OESS::MPLS::Discovery::ISIS'); + use_ok( 'OESS::MPLS::Discovery::Paths'); + use_ok( 'OESS::MPLS::Topology'); + use_ok( 'OESS::NSO::Client' ); + use_ok( 'OESS::NSO::ClientStub' ); + use_ok( 'OESS::NSO::ConnectionCache' ); + use_ok( 'OESS::NSO::Discovery' ); + use_ok( 'OESS::NSO::FWDCTL' ); + use_ok( 'OESS::NSO::FWDCTLService' ); } From 530b804cb507c37352cdbe428165d2b1944599ba Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 30 Jul 2021 00:17:47 +0000 Subject: [PATCH 148/156] pass corrected arg to Link::update --- perl-lib/OESS/lib/OESS/NSO/Discovery.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm index bc4eba560..84ab35acf 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Discovery.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Discovery.pm @@ -352,8 +352,8 @@ sub link_handler { $self->{logger}->info("Decommissioning link $link->{name}."); my ($link_id, $link_err) = OESS::DB::Link::update( - db => $self->{db}, - model => { + db => $self->{db}, + link => { link_id => $link->{link_id}, link_state => 'decom', status => 'down', From 7d76d1060718012942dfc17a8217e6ed489691b2 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 30 Jul 2021 02:06:56 +0000 Subject: [PATCH 149/156] set interface role on link create and decom --- perl-lib/OESS/lib/OESS/DB/Link.pm | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/perl-lib/OESS/lib/OESS/DB/Link.pm b/perl-lib/OESS/lib/OESS/DB/Link.pm index 353835e04..02c625819 100644 --- a/perl-lib/OESS/lib/OESS/DB/Link.pm +++ b/perl-lib/OESS/lib/OESS/DB/Link.pm @@ -51,6 +51,10 @@ sub create { VALUES (?,?,?,?,?,?,?,UNIX_TIMESTAMP(NOW()),-1) "; + my $q3 = " + UPDATE interface set role='trunk' WHERE interface_id=? or interface_id=? + "; + my $link_id = $args->{db}->execute_query($q1, [ $args->{model}->{name}, $args->{model}->{remote_urn}, @@ -74,6 +78,14 @@ sub create { return (undef, $args->{db}->get_error); } + my $ok = $args->{db}->execute_query($q3, [ + $args->{model}->{interface_a_id}, + $args->{model}->{interface_z_id} + ]); + if (!defined $ok) { + return (undef, $args->{db}->get_error); + } + return ($link_id, undef); } @@ -337,7 +349,6 @@ sub update { return (undef, $args->{db}->get_error); } - my $q2 = " INSERT INTO link_instantiation ( link_id, openflow, mpls, link_state, interface_a_id, ip_a, @@ -360,6 +371,23 @@ sub update { return (undef, $args->{db}->get_error); } + # Set role of interfaces to 'unknown' whenever a link is + # decom'd. This allows the link's interfaces to be used for other + # purposes after removal. + if ($args->{link}->{link_state} eq 'decom') { + my $q3 = " + UPDATE interface set role='unknown' WHERE interface_id=? or interface_id=? + "; + + my $ok = $args->{db}->execute_query($q3, [ + $args->{model}->{interface_a_id}, + $args->{model}->{interface_z_id} + ]); + if (!defined $ok) { + return (undef, $args->{db}->get_error); + } + } + return ($link_instantiation_id, undef); } From 1ddaee3e318b0e6624e480361ce3dba9aa121267 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 30 Jul 2021 03:09:45 +0000 Subject: [PATCH 150/156] update links in entity section --- frontend/www/js_templates/entity.js | 6 +++--- frontend/www/js_templates/phonebook.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/www/js_templates/entity.js b/frontend/www/js_templates/entity.js index 85bce3ce8..ed868998b 100644 --- a/frontend/www/js_templates/entity.js +++ b/frontend/www/js_templates/entity.js @@ -21,7 +21,7 @@ async function loadEntityContent(parentEntity=null, action){ e.preventDefault(); try { let res = await add_entity(entityID, entity_name, description, logo_url, entity_url); - window.location.href = `[% path %]new/index.cgi?action=phonebook&entity_id=${entityID}`; + window.location.href = `[% path %]index.cgi?action=phonebook&entity_id=${entityID}`; } catch (error) { console.error(error); @@ -37,11 +37,11 @@ async function loadEntityContent(parentEntity=null, action){ entity_url.value = entity.url; document.querySelector('#edit-entity-btn').onclick = async function(){ await edit_entity(entityID, entity_name, description, logo_url, entity_url); - window.location.href = `[% path %]new/index.cgi?action=phonebook&entity_id=${entityID}`; + window.location.href = `[% path %]index.cgi?action=phonebook&entity_id=${entityID}`; }; } document.querySelector('#cancel').onclick = function(){ - window.location.href = `[% path %]new/index.cgi?action=phonebook&entity_id=${entityID}`; + window.location.href = `[% path %]index.cgi?action=phonebook&entity_id=${entityID}`; }; } diff --git a/frontend/www/js_templates/phonebook.js b/frontend/www/js_templates/phonebook.js index 01c61b67f..592f42e67 100644 --- a/frontend/www/js_templates/phonebook.js +++ b/frontend/www/js_templates/phonebook.js @@ -217,12 +217,12 @@ async function loadEntityList(parentEntity=null) { if ((user.is_admin == 1 && user.type != 'read-only') || valid_users.includes(user.user_id)){ edit_entity_btn.style.display = 'block'; edit_entity_btn.onclick = function(){ - window.location.href = `[% path %]new/index.cgi?action=edit_entity&entity_id=${entityID}`; + window.location.href = `[% path %]index.cgi?action=edit_entity&entity_id=${entityID}`; }; add_entity_btn.style.display = 'block'; add_entity_btn.onclick = function(){ - window.location.href = `[% path %]new/index.cgi?action=add_entity&entity_id=${entityID}`; + window.location.href = `[% path %]index.cgi?action=add_entity&entity_id=${entityID}`; }; let user_list = document.querySelector('#user-list'); From 66166cd3596f6208ff1dd9f0f0c64804a0af7069 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Fri, 30 Jul 2021 17:45:34 +0000 Subject: [PATCH 151/156] log http codes >= 400 from nso api --- perl-lib/OESS/lib/OESS/NSO/Client.pm | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 3b8aa9a7b..214fbc9d5 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -94,6 +94,9 @@ sub create_l2connection { 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } return if ($res->content eq ''); # Empty payload indicates success my $result = decode_json($res->content); @@ -122,6 +125,9 @@ sub delete_l2connection { $self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:services/oess-l2connection:oess-l2connection=$conn_id?unhide=oess", 'Content-type' => 'application/yang-data+json' ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } return if ($res->content eq ''); # Empty payload indicates success my $result = decode_json($res->content); @@ -176,6 +182,9 @@ sub edit_l2connection { 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } return if ($res->content eq ''); # Empty payload indicates success my $result = decode_json($res->content); @@ -204,6 +213,9 @@ sub get_l2connections { $self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:services/oess-l2connection:oess-l2connection/?unhide=oess", 'Content-type' => 'application/yang-data+json' ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } if ($res->content eq '') { # Empty payload indicates success $connections = []; } else { @@ -340,6 +352,9 @@ sub create_l3connection { 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } return if ($res->content eq ''); # Empty payload indicates success my $result = decode_json($res->content); @@ -368,6 +383,9 @@ sub delete_l3connection { $self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:services/oess-l3connection:oess-l3connection=$conn_id?unhide=oess", 'Content-type' => 'application/yang-data+json' ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } return if ($res->content eq ''); # Empty payload indicates success my $result = decode_json($res->content); @@ -441,6 +459,9 @@ sub edit_l3connection { 'Content-type' => 'application/yang-data+json', 'Content' => encode_json($payload) ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } return if ($res->content eq ''); # Empty payload indicates success my $result = decode_json($res->content); @@ -469,6 +490,9 @@ sub get_l3connections { $self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:services/oess-l3connection:oess-l3connection/?unhide=oess", 'Content-type' => 'application/yang-data+json' ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } if ($res->content eq '') { # Empty payload indicates success $connections = []; } else { @@ -517,6 +541,9 @@ sub get_backbones { $self->{config_obj}->nso_host . "/restconf/data/tailf-ncs:services/backbone:backbone/", 'Content-type' => 'application/yang-data+json' ); + if ($res->code >= 400) { + die "HTTP Error " . $res->code; + } if ($res->content eq '') { # Empty payload indicates success $backbones = []; } else { From eb013804e0b0fe0624ae70bd6046121c1933ce48 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Sat, 31 Jul 2021 01:15:36 +0000 Subject: [PATCH 152/156] log response body for http errors from nso --- perl-lib/OESS/lib/OESS/NSO/Client.pm | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/perl-lib/OESS/lib/OESS/NSO/Client.pm b/perl-lib/OESS/lib/OESS/NSO/Client.pm index 214fbc9d5..8df6c180e 100644 --- a/perl-lib/OESS/lib/OESS/NSO/Client.pm +++ b/perl-lib/OESS/lib/OESS/NSO/Client.pm @@ -95,7 +95,7 @@ sub create_l2connection { 'Content' => encode_json($payload) ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } return if ($res->content eq ''); # Empty payload indicates success @@ -126,7 +126,7 @@ sub delete_l2connection { 'Content-type' => 'application/yang-data+json' ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } return if ($res->content eq ''); # Empty payload indicates success @@ -183,7 +183,7 @@ sub edit_l2connection { 'Content' => encode_json($payload) ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } return if ($res->content eq ''); # Empty payload indicates success @@ -214,7 +214,7 @@ sub get_l2connections { 'Content-type' => 'application/yang-data+json' ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } if ($res->content eq '') { # Empty payload indicates success $connections = []; @@ -353,7 +353,7 @@ sub create_l3connection { 'Content' => encode_json($payload) ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } return if ($res->content eq ''); # Empty payload indicates success @@ -384,7 +384,7 @@ sub delete_l3connection { 'Content-type' => 'application/yang-data+json' ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } return if ($res->content eq ''); # Empty payload indicates success @@ -460,7 +460,7 @@ sub edit_l3connection { 'Content' => encode_json($payload) ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } return if ($res->content eq ''); # Empty payload indicates success @@ -491,7 +491,7 @@ sub get_l3connections { 'Content-type' => 'application/yang-data+json' ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } if ($res->content eq '') { # Empty payload indicates success $connections = []; @@ -542,7 +542,7 @@ sub get_backbones { 'Content-type' => 'application/yang-data+json' ); if ($res->code >= 400) { - die "HTTP Error " . $res->code; + die "HTTP Code: " . $res->code . " HTTP Content: " . $res->content; } if ($res->content eq '') { # Empty payload indicates success $backbones = []; From 11a0a06fb6e73221d0ad534fd31819ed03879a27 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Sun, 1 Aug 2021 21:12:40 +0000 Subject: [PATCH 153/156] populate entire vrf object for edits --- perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm index ec1fd2175..b1f4e1738 100644 --- a/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm +++ b/perl-lib/OESS/lib/OESS/NSO/FWDCTL.pm @@ -193,6 +193,16 @@ sub modifyVrf { my $pending_hash = decode_json($args->{pending}); my $pending_conn = new OESS::VRF(db => $self->{db}, model => $pending_hash); + $pending_conn->{endpoints} = []; + foreach my $pending_ep_hash (@{$pending_hash->{endpoints}}) { + my $pending_ep = new OESS::Endpoint(db => $self->{db}, model => $pending_ep_hash); + + $pending_ep->{peers} = []; + foreach my $pending_peer_hash (@{$pending_ep_hash->{peers}}) { + push @{$pending_ep->{peers}}, new OESS::Peer(db => $self->{db}, model => $pending_peer_hash); + } + push @{$pending_conn->{endpoints}}, $pending_ep; + } my $err = $self->{nso}->edit_l3connection($pending_conn); return $err if (defined $err); From 24d8f1b51ca8edd781b207334e0ba944220c5efb Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Sun, 1 Aug 2021 23:31:10 +0000 Subject: [PATCH 154/156] set user role when adding via old admin ui --- frontend/www/js_templates/admin.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/www/js_templates/admin.js b/frontend/www/js_templates/admin.js index 0b5f88077..d776c54a3 100644 --- a/frontend/www/js_templates/admin.js +++ b/frontend/www/js_templates/admin.js @@ -1267,7 +1267,7 @@ function setup_users_tab(){ else{ - var ds = new YAHOO.util.DataSource("../services/admin/admin.cgi?method=add_user_to_workgroup&workgroup_id=" + add_new_user_to_workgroup + "&user_id="+ user_id + "&role=normal"); + var ds = new YAHOO.util.DataSource("../services/admin/admin.cgi?method=add_user_to_workgroup&workgroup_id=" + add_new_user_to_workgroup + "&user_id="+ user_id + "&role=" + encodeURIComponent(type)); ds.responseType = YAHOO.util.DataSource.TYPE_JSON; ds.responseSchema = { resultsList: "results", From 425a3b1720f0b3018cd2036e2da292f86c9d5d23 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Mon, 2 Aug 2021 20:05:15 +0000 Subject: [PATCH 155/156] update has_circuit_access method to check for permissions vs role --- frontend/webservice/data.cgi | 12 +- frontend/webservice/measurement.cgi | 2 +- perl-lib/OESS/lib/OESS/DB/User.pm | 138 +++++++++++--- .../OESS/t/z-DB/User.has_circuit_access.00.t | 57 ------ .../t/z-DB/User.has_circuit_permission.00.t | 176 ++++++++++++++++++ 5 files changed, 293 insertions(+), 92 deletions(-) delete mode 100755 perl-lib/OESS/t/z-DB/User.has_circuit_access.00.t create mode 100755 perl-lib/OESS/t/z-DB/User.has_circuit_permission.00.t diff --git a/frontend/webservice/data.cgi b/frontend/webservice/data.cgi index ec11f4b66..7770db1d5 100755 --- a/frontend/webservice/data.cgi +++ b/frontend/webservice/data.cgi @@ -767,7 +767,7 @@ sub get_circuits_by_interface_id { else { my $circuit; foreach(@{$circuits}){ - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $_->{'circuit_id'}, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $_->{'circuit_id'}, permission => 'read'); if($ok){ push($results->{'results'}, $_); } @@ -918,7 +918,7 @@ sub get_circuit_scheduled_events { my $results; my $circuit_id = $args->{'circuit_id'}{'value'}; - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, permission => 'read'); if(!$ok){ $results->{'error'} = $err; return $results; @@ -944,7 +944,7 @@ sub get_circuit_history { my $circuit_id = $args->{'circuit_id'}{'value'}; - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, permission => 'read'); if(!$ok){ $results->{'error'} = $err; return $results; @@ -969,7 +969,7 @@ sub get_circuit_details { my ( $method, $args ) = @_ ; my $circuit_id = $args->{'circuit_id'}{'value'}; - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, permission => 'read'); if(!$ok){ $results->{'error'} = $err; return $results; @@ -1003,7 +1003,7 @@ sub get_circuit_details_by_external_identifier { return; } my $circuit_id = $info->{'circuit_id'}; - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, permission => 'read'); if(!$ok){ $results->{'error'} = $err; return $results; @@ -1222,7 +1222,7 @@ sub generate_clr { if ( !defined($circuit_id) ) { $method->set_error( "No Circuit ID Specified" ); } - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, permission => 'read'); if(!$ok){ $results->{'error'} = $err; return $results; diff --git a/frontend/webservice/measurement.cgi b/frontend/webservice/measurement.cgi index fc7207202..398d9d5d2 100755 --- a/frontend/webservice/measurement.cgi +++ b/frontend/webservice/measurement.cgi @@ -155,7 +155,7 @@ sub get_circuit_data { my $link = $args->{'link'}{'value'}; my ($ok, $err); - my ($ok, $err) = OESS::DB::User::has_circuit_access(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, role => 'read-only'); + my ($ok, $err) = OESS::DB::User::has_circuit_permission(db => $db2, username => $ENV{'REMOTE_USER'}, circuit_id => $circuit_id, permission => 'read'); if(!$ok){ $results->{'error'} = $err; return $results; diff --git a/perl-lib/OESS/lib/OESS/DB/User.pm b/perl-lib/OESS/lib/OESS/DB/User.pm index ac5a6816d..56642a206 100644 --- a/perl-lib/OESS/lib/OESS/DB/User.pm +++ b/perl-lib/OESS/lib/OESS/DB/User.pm @@ -1,12 +1,14 @@ -#!/usr/bin/perl - use strict; use warnings; -use OESS::Workgroup; -use OESS::DB::Workgroup; package OESS::DB::User; +use Data::Dumper; + +use OESS::DB::Circuit; +use OESS::DB::Endpoint; +use OESS::DB::Workgroup; + =head2 fetch =cut @@ -640,7 +642,7 @@ sub has_workgroup_access { } } -=head2 has_circuit_access +=head2 has_circuit_permission =over @@ -656,52 +658,132 @@ sub has_workgroup_access { =item circuit_id Denotes the circuit_id of the circuit we checking the users permissions in -=item role - Denotes the level of access the user needs for a particular action +=item permission + Denotes the level of access the user needs for a particular action. May be 'create', 'read', 'update', or 'delete'. =back - my $results = OESS::DB::User::has_circuit_access( - db => $db, - user_id => $user_id, - circuit_id => $circuit_id, - - - has_circuit_access checks if a specified user's role found in the circuit's associated workgroup, and has - appropriate access for that workgroup. The workgroup is accessed by getting the circuit with C - and then uses C to determine whether the user has access to the circuit. + my $results = OESS::DB::User::has_circuit_permission( + db => $db, + user_id => $user_id, + circuit_id => $circuit_id, + permission => 'update' + ); + +has_circuit_permission checks if the specified user has appropriate +permission for that circuit. Permissions are determined by looking up +circuit details, pairing those details against the user's workgroups, +and then fetching the user's role within those workgroups. + =cut -sub has_circuit_access { +sub has_circuit_permission { my %params = @_; - my $db = $params{'db'}; - my $user_id = $params{'user_id'}; - my $username = $params{'username'}; + my $db = $params{'db'}; + my $user_id = $params{'user_id'}; + my $username = $params{'username'}; my $circuit_id = $params{'circuit_id'}; - my $role = $params{'role'}; + my $permission = $params{'permission'}; + return (0, "Required argument 'db' is missing.") if !defined $db; return (0, "Required to pass either 'user_id' or 'username'.") if !defined $user_id && !defined $username; return (0, "Required argument 'circuit_id' is missing.") if !defined $circuit_id; - return (0, "Required argument 'role' is missing.") if !defined $role; + return (0, "Required argument 'permission' is missing.") if !defined $permission; + + my $permissions = { + create => undef, + read => undef, + update => undef, + delete => undef, + }; + if (!exists $permissions->{$permission}) { + return (0, "Required argument 'permission' must be 'create', 'read', 'update', or 'delete'."); + } + my $user; if (!defined $user_id) { $user = OESS::DB::User::fetch(db => $db, username => $username); } else { $user = OESS::DB::User::fetch(db => $db, user_id => $user_id); } - if (!defined $user || $user->{'status'} eq 'decom'){ return (0, "Invalid or decommissioned user specified."); } my $circuit = OESS::DB::Circuit::fetch_circuit(db => $db, circuit_id => $circuit_id)->[0]; + if (!defined $circuit || $circuit->{'state'} eq 'decom') { + return (0, "Invalid or decommissioned circuit specified."); + } - if (!defined $circuit || $circuit->{'state'} eq 'decom'){ - return (0, "Invalid or decommissioned circuit specified." ); + # Permissions for admin users + my $admin_results = $db->execute_query( + "SELECT role + FROM user_workgroup_membership JOIN workgroup + ON user_workgroup_membership.workgroup_id=workgroup.workgroup_id + WHERE workgroup.type='admin' AND user_workgroup_membership.user_id=?", + [$user->{user_id}] + ); + $admin_results = [] if !defined $admin_results; + foreach my $result (@$admin_results) { + if ($result->{role} eq 'admin' || $result->{role} eq 'normal') { + $permissions->{create} = 1; + $permissions->{read} = 1; + $permissions->{update} = 1; + $permissions->{delete} = 1; + } else { + $permissions->{read} = 1; + } } - $user_id = $user->{'user_id'}; - my $workgroup_id = $circuit->{'workgroup_id'}; - return has_workgroup_access(db => $db, user_id => $user_id, workgroup_id => $workgroup_id, role => $role); + # Permissions for users in the circuit owner's workgroup + my $workgroup_results = $db->execute_query( + "SELECT role + FROM user_workgroup_membership + WHERE user_workgroup_membership.user_id=? and user_workgroup_membership.workgroup_id=?", + [$user->{user_id}, $circuit->{workgroup_id}] + ); + $workgroup_results = [] if !defined $workgroup_results; + foreach my $result (@$workgroup_results) { + if ($result->{role} eq 'admin' || $result->{role} eq 'normal') { + $permissions->{create} = 1; + $permissions->{read} = 1; + $permissions->{update} = 1; + $permissions->{delete} = 1; + } else { + $permissions->{read} = 1; + } + } + + # Permissions for users in the circuit's interfaces' owner workgroups + my $interface_results = $db->execute_query( + "SELECT user_workgroup_membership.role + FROM user_workgroup_membership JOIN interface + ON user_workgroup_membership.workgroup_id=interface.workgroup_id + JOIN circuit_edge_interface_membership AS circuit_ep + ON circuit_ep.interface_id=interface.interface_id AND circuit_ep.end_epoch=-1 + WHERE user_workgroup_membership.user_id=? AND circuit_ep.circuit_id=? + ", + [$user->{user_id}, $circuit->{circuit_id}] + ); + $interface_results = [] if !defined $interface_results; + foreach my $result (@$interface_results) { + if ($result->{role} eq 'admin' || $result->{role} eq 'normal') { + $permissions->{read} = 1; + # TODO Ideally interface owners may remove endpoints from + # any circuit which terminate on their interfaces, + # although is not supported at this time. + # + # $permissions->{update} = 1; + } else { + $permissions->{read} = 1; + } + } + + if (!$permissions->{$permission}) { + return (0, "User $user->{username} doesn't have $permission permission for circuit $circuit_id."); + } + return (1, undef); } + + 1; diff --git a/perl-lib/OESS/t/z-DB/User.has_circuit_access.00.t b/perl-lib/OESS/t/z-DB/User.has_circuit_access.00.t deleted file mode 100755 index e3f0e6f25..000000000 --- a/perl-lib/OESS/t/z-DB/User.has_circuit_access.00.t +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; - -use FindBin; -my $path; - -BEGIN { - if ($FindBin::Bin =~ /(.*)/) { - $path = $1; - } -} -use lib "$path/.."; - -use Data::Dumper; -use Test::More tests => 7; - -use OESSDatabaseTester; - -use OESS::DB; -use OESS::DB::User; -use OESS::DB::Workgroup; -use OESS::DB::Circuit; - -OESSDatabaseTester::resetOESSDB( - config => "$path/../conf/database.xml", - dbdump => "$path/../conf/oess_known_state.sql" -); - -my $db = new OESS::DB( - config => "$path/../conf/database.xml" -); - - -my ($result, $error) = OESS::DB::User::has_circuit_access(db => $db, username => 'aragusa', circuit_id => 11, role => 'admin'); -ok($result == 1, "Got expected result '1'"); - -($result, $error) = OESS::DB::User::has_circuit_access(db => $db, user_id => 11, circuit_id => 11, role => 'admin'); -ok($result == 1, "Got expected result '1'"); - -($result, $error) = OESS::DB::User::has_circuit_access(db => $db, user_id => 11, circuit_id => 11, role => 'normal'); -ok($result == 1, "Got expected result '1'"); - -($result, $error) = OESS::DB::User::has_circuit_access(db => $db, user_id => 11, circuit_id => 11, role => 'read-only'); -ok($result == 1, "Got expected result '1'"); - -($result, $error) = OESS::DB::User::has_circuit_access(db => $db, user_id => 901, circuit_id => 11, role => 'admin'); -ok(defined $error, "Got expected error, $error"); - -($result, $error) = OESS::DB::User::has_circuit_access(db => $db, user_id => 901, circuit_id => 11, role => 'noraml'); -ok(defined $error, "Got expected error, $error"); - -($result, $error) = OESS::DB::User::has_circuit_access(db => $db, user_id => 901, circuit_id => 11, role => 'read-only'); -ok(defined $error, "Got expected error, $error"); - - diff --git a/perl-lib/OESS/t/z-DB/User.has_circuit_permission.00.t b/perl-lib/OESS/t/z-DB/User.has_circuit_permission.00.t new file mode 100755 index 000000000..e07bd5e70 --- /dev/null +++ b/perl-lib/OESS/t/z-DB/User.has_circuit_permission.00.t @@ -0,0 +1,176 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use FindBin; +my $path; + +BEGIN { + if ($FindBin::Bin =~ /(.*)/) { + $path = $1; + } +} +use lib "$path/.."; + +use Data::Dumper; +use Test::More tests => 80; + +use OESSDatabaseTester; + +use OESS::DB; +use OESS::DB::User; +use OESS::DB::Workgroup; +use OESS::DB::Circuit; + +OESSDatabaseTester::resetOESSDB( + config => "$path/../conf/database.xml", + dbdump => "$path/../conf/oess_known_state.sql" +); + +my $db = new OESS::DB( + config => "$path/../conf/database.xml" +); + +$db->execute_query("update circuit set workgroup_id=31 where circuit_id=2281", []); +$db->execute_query("update interface set workgroup_id=21 where interface_id=401 or interface_id=751", []); + +# user 501: workgroup 31 user with role admin + not in admin workgroup + not in workgroup 21 +$db->execute_query("delete from user_workgroup_membership where user_id=501 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=501 and workgroup_id=21", []); +$db->execute_query( + "insert into user_workgroup_membership (workgroup_id,user_id,role) values (?,?,?)", + [31, 501, 'admin'] +); + +# user 601: workgroup 31 user with role normal + not in admin workgroup + not in workgroup 21 +$db->execute_query("delete from user_workgroup_membership where user_id=601 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=601 and workgroup_id=21", []); +$db->execute_query( + "insert into user_workgroup_membership (workgroup_id,user_id,role) values (?,?,?)", + [31, 601, 'normal'] +); + +# user 701: workgroup 31 user with role read-only + not in admin workgroup + not in workgroup 21 +$db->execute_query("delete from user_workgroup_membership where user_id=701 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=701 and workgroup_id=21", []); +$db->execute_query( + "insert into user_workgroup_membership (workgroup_id,user_id,role) values (?,?,?)", + [31, 701, 'read-only'] +); + +# user 201: workgroup user with role admin + not in admin workgroup + not in workgroup 31 +$db->execute_query("delete from user_workgroup_membership where user_id=201 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=201 and workgroup_id=31", []); + +# user 301: workgroup user with role normal + not in admin workgroup + not in workgroup 31 +$db->execute_query("delete from user_workgroup_membership where user_id=301 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=301 and workgroup_id=31", []); +$db->execute_query( + "insert into user_workgroup_membership (workgroup_id,user_id,role) values (?,?,?)", + [21, 301, 'normal'] +); + +# user 401: workgroup user with role read-only + not in admin workgroup + not in workgroup 31 +$db->execute_query("delete from user_workgroup_membership where user_id=401 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=401 and workgroup_id=31", []); +$db->execute_query( + "insert into user_workgroup_membership (workgroup_id,user_id,role) values (?,?,?)", + [21, 401, 'read-only'] +); + +# user 11 (aragusa): admin workgroup user with role admin + not in workgroup 21 +$db->execute_query("update user_workgroup_membership set role='normal' where user_id=31 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=11 and workgroup_id=21", []); + +# user 31: admin workgroup user with role normal + not in workgroup 21 +$db->execute_query("update user_workgroup_membership set role='normal' where user_id=31 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=31 and workgroup_id=21", []); + +# user 21: admin workgroup user with role read-only + not in workgroup 21 +$db->execute_query("update user_workgroup_membership set role='read-only' where user_id=21 and workgroup_id=11", []); +$db->execute_query("delete from user_workgroup_membership where user_id=21 and workgroup_id=21", []); + + +# NOTE: circuit 2281 is owned by workgroup 31. interfaces 401 and 751 +# used by circuit 2281 are owned by workgroup 21. + +my $tests = [ + # BEGIN Circuit owners + # user 501: workgroup 31 user with role admin + not in admin workgroup + not in workgroup 21 + { args => { db => $db, user_id => 501, circuit_id => 2281, permission => 'create' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 501, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 501, circuit_id => 2281, permission => 'update' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 501, circuit_id => 2281, permission => 'delete' }, result => { value => 1, error => undef } }, + + # user 601: workgroup 31 user with role normal + not in admin workgroup + not in workgroup 21 + { args => { db => $db, user_id => 601, circuit_id => 2281, permission => 'create' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 601, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 601, circuit_id => 2281, permission => 'update' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 601, circuit_id => 2281, permission => 'delete' }, result => { value => 1, error => undef } }, + + # user 701: workgroup 31 user with role read-only + not in admin workgroup + not in workgroup 21 + { args => { db => $db, user_id => 701, circuit_id => 2281, permission => 'create' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 701, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 701, circuit_id => 2281, permission => 'update' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 701, circuit_id => 2281, permission => 'delete' }, result => { value => 0, error => 1 } }, + # END Circuit owners + + # BEGIN Interface owners + # user 201: workgroup user with role admin + not in admin workgroup + { args => { db => $db, user_id => 201, circuit_id => 2281, permission => 'create' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 201, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 201, circuit_id => 2281, permission => 'update' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 201, circuit_id => 2281, permission => 'delete' }, result => { value => 0, error => 1 } }, + + # user 301: workgroup user with role normal + not in admin workgroup + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'create' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'update' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'delete' }, result => { value => 0, error => 1 } }, + + # user 401: workgroup user with role normal + not in admin workgroup + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'create' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'update' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 301, circuit_id => 2281, permission => 'delete' }, result => { value => 0, error => 1 } }, + # END Interface owners + + # BEGIN Admins + # admin workgroup user with a role of admin. has global access to everything. + { args => { db => $db, username => 'aragusa', circuit_id => 2281, permission => 'create' }, result => { value => 1, error => undef } }, + { args => { db => $db, username => 'aragusa', circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, username => 'aragusa', circuit_id => 2281, permission => 'update' }, result => { value => 1, error => undef } }, + { args => { db => $db, username => 'aragusa', circuit_id => 2281, permission => 'delete' }, result => { value => 1, error => undef } }, + + # admin workgroup user with a role of normal. has global access to circuits. + { args => { db => $db, user_id => 31, circuit_id => 2281, permission => 'create' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 31, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 31, circuit_id => 2281, permission => 'update' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 31, circuit_id => 2281, permission => 'delete' }, result => { value => 1, error => undef } }, + + # admin workgroup user with a role of read-only. has global read access to circuits. + { args => { db => $db, user_id => 21, circuit_id => 2281, permission => 'create' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 21, circuit_id => 2281, permission => 'read' }, result => { value => 1, error => undef } }, + { args => { db => $db, user_id => 21, circuit_id => 2281, permission => 'update' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 21, circuit_id => 2281, permission => 'delete' }, result => { value => 0, error => 1 } }, + # END Admins + + # user not in admin workgroup, circuit workgroup, or circuit's interfaces' workgroup + { args => { db => $db, user_id => 371, circuit_id => 2281, permission => 'create' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 371, circuit_id => 2281, permission => 'read' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 371, circuit_id => 2281, permission => 'update' }, result => { value => 0, error => 1 } }, + { args => { db => $db, user_id => 371, circuit_id => 2281, permission => 'delete' }, result => { value => 0, error => 1 } }, +]; + +foreach my $test (@$tests) { + my ($result, $error) = OESS::DB::User::has_circuit_permission(%{$test->{args}}); + + ok($result == $test->{result}->{value}, "Got '$result'. Expected '$test->{result}->{value}'."); + if (defined $test->{result}->{error}) { + ok(defined $error, "Got error message '$error'. Expected an error message."); + } else { + my $error_str = (!defined $error) ? 'undef' : $error; + ok(!defined $error, "Got error message '$error_str'. Expected error message 'undef'."); + } +} From 6479cdb5dc002fd87e91910c522c02e30b7df7a8 Mon Sep 17 00:00:00 2001 From: Jonathan Stout Date: Tue, 3 Aug 2021 15:22:53 +0000 Subject: [PATCH 156/156] update tsds url to work with existing configs --- app/mpls/mpls_discovery.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/mpls/mpls_discovery.pl b/app/mpls/mpls_discovery.pl index be77a487c..f4b6d019b 100755 --- a/app/mpls/mpls_discovery.pl +++ b/app/mpls/mpls_discovery.pl @@ -27,7 +27,7 @@ sub core{ my $nso_discovery; my $nso_client = new OESS::NSO::Client(config_obj => $config); my $tsds_client = new GRNOC::WebService::Client( - url => $config->tsds_url . "/services/push.cgi", + url => $config->tsds_url . "/push.cgi", uid => $config->tsds_username, passwd => $config->tsds_password, realm => $config->tsds_realm,