Merge remote-tracking branch 'origin/dev'

* origin/dev:
  Added router list to ngreport vrf
  Added terminal size awareness to VLAN group printing
  Updated path output and error handling for default CIDR match
  Fixed error handling for empty VRFs
  Reverted to using netgrphdev.ini for testing
  Updated test/dev config_file locations, base directory config overrides all others
  Added -nfilter to netgrph to search on custom filters at runtime
  Refactored get_networks_on_group to get_networks_on_filter to handle stored config filters or dynamic filters. Completed ngreport on VRFs.
  VRF Checkpoint
  Skeleton code for VRF report, documentation update
  Added -dev report to reporting module
  Local commit test with VSC
  Fixed pytest-3 glitch
  Updated test suite for macos

.cache/v/cache/lastfailed

@@ -0,0 +1 @@
+{}

.gitignore

@@ -6,3 +6,4 @@ netgrphdev.ini
 netgrphtest.ini
 neo4j-queries.txt
 build-commands
+.vscode/

CONTRIBUTING.md

@@ -4,7 +4,7 @@ These are the early days of NetGrph, but I fully plan to support this program
 going forward and am looking for contributors. I am also the primary maintainer
 of the [Network Tracking Database
 (NetDB)](http://netdbtracking.sourceforge.net/), which is in production on some
-extremely large network. NetGrph is currently serving as our internal model for
+extremely large networks. NetGrph is currently serving as our internal model for
 network automation tasks, and we hope others find it useful as well. It's
 primary purpose is to do work at this time, from automation to troubleshooting
 tasks.

csv/vrfs.csv

@@ -3,3 +3,4 @@ guest,10,Guest Network
 outside,1,External Networks
 perim,25,Perimeter DMZ Network
 utility,150,Utility Network
+test,1,Test Empty VRF

cypher/constraints.cyp

@@ -1,7 +1,6 @@
 -- Create constraints on DB for consistency and performance
 --CREATE Garbage
 CREATE CONSTRAINT ON (n:Network) ASSERT n.vrfcidr IS UNIQUE
-CREATE CONSTRAINT ON (m:MgmtGroup) ASSERT m.name IS UNIQUE
 CREATE CONSTRAINT ON (s:Switch) ASSERT s.name IS UNIQUE
 CREATE CONSTRAINT ON (v:VRF) ASSERT v.name IS UNIQUE
 CREATE CONSTRAINT ON (v:VLAN) ASSERT v.name IS UNIQUE

docs/CHANGES.md

@@ -1,6 +1,11 @@
 
+### v0.7.3
+- Added vrf and dev to ngreport
+- Added nfilter to netgrph to pass in custom filters at runtime
+- Cleaned up debug netgrph.ini handling
+- VLAN Group printing follows terminal size
 
-### 7/1/16
+### v0.7.0
 - Created public repo
 - Added lots of documentation
 - Cleaned up config files

docs/INSTALL.md

@@ -51,14 +51,16 @@ sudo pip3 install -r requirements.txt
 - Browse to the database web interface and run `MATCH (s:Switch) RETURN s`
 - Try out some sample commands
 
-## Sample netgrph for use with test data
+## Sample commands for use with test data
 ```
+./netgrph.py -nf all
+./netgrph.py -nf all -o tree
+./ngreport.py -dev ".*"
 ./netgrph.py abc4mdf
-./netgrph.py abc4mdf -o csv
 ./netgrph.py abc4mdf -o json
 ./netgrph.py abc4mdf -o yaml
 ./netgrph.py -sp abc2sw1 xyz2sw1
-./netgrph.py -sp abc.* xyz.*
+./netgrph.py -sp abc.* xyz.* -o csv
 ./netgrph.py 120
 ./netgrph.py 1246
 ./netgrph.py -fp 10.1.120.50 8.8.8.8

docs/TODO

@@ -1,21 +1,21 @@
 
 
 NetDB:
-- Add CIDR usage for scope usage reporting
+- Add CIDR scope usage reporting
 - Add more extensive device reporting
 
 Reporting:
-  - VRF Report
-  - Switch Report
   - No name, no root, no router report
 
 Other:
 - Add VRF to -rpath
 - Add master -path to netgrph
+- Add tabular output option
+- built-in paging support
+- Group report takes up entire console
 
 Maintenance:
 - Add switch skip on certain neighbors
 - Rewrite nglib.net_update using Bolt as Pythonic
 - Emulate dict with ngtree
-- built-in paging support
-- Group report takes up entire console
+

docs/VERSION

@@ -1 +1 @@
-v0.7.2
+v0.7.3

netgrph.py

@@ -39,6 +39,7 @@
 import argparse
 import nglib
 
+
 # Default Config File Location
 config_file = '/etc/netgrph.ini'
 alt_config = './docs/netgrph.ini'
@@ -48,7 +49,7 @@
 if re.search(r'\/dev$', dirname):
     config_file = 'netgrphdev.ini'
 elif re.search(r'\/test$', dirname):
-    config_file = "netgrphtest.ini"
+    config_file = "netgrphdev.ini"
 
 parser = argparse.ArgumentParser()
 
@@ -70,6 +71,8 @@
                     action="store_true")
 parser.add_argument("-nlist", help="Get all networks in an alert group",
                     action="store_true")
+parser.add_argument("-nfilter", help="Get all networks on a filter (see netgrph.ini)",
+                    action="store_true")
 parser.add_argument("-dev", help="Get the Details for a Device (Switch/Router/FW)",
                     action="store_true")
 parser.add_argument("-fpath", metavar="src",
@@ -134,16 +137,19 @@
 
 if args.fpath:
     nglib.query.path.get_fw_path(args.fpath, args.search)
+
 elif args.spath:
     rtype = "TREE"
     if args.output:
         rtype = args.output
     nglib.query.path.get_switch_path(args.spath, args.search, rtype=rtype)
+
 elif args.rpath:
     rtype = "TREE"
     if args.output:
         rtype = args.output
     nglib.query.path.get_routed_path(args.rpath, args.search, rtype=rtype)
+
 elif args.dev:
     rtype = "TREE"
     if args.output:
@@ -167,7 +173,13 @@
     rtype = "CSV"
     if args.output:
         rtype = args.output
-    nglib.query.net.get_networks_on_group(args.search, rtype=rtype)
+    nglib.query.net.get_networks_on_filter(args.search, rtype=rtype)
+
+elif args.nfilter:
+    rtype = "CSV"
+    if args.output:
+        rtype = args.output
+    nglib.query.net.get_networks_on_filter(nFilter=args.search, rtype=rtype)
 
 elif args.group:
     nglib.query.vlan.get_vlans_on_group(args.search, args.vrange)

nglib/query/__init__.py

@@ -114,11 +114,11 @@ def get_net_filter(group):
         raise Exception("No Group Filter Found", group)
 
 
-def check_net_filter(netDict, group):
+def check_net_filter(netDict, group=None, nFilter=None):
     """
     Filters Networks based on vrf:role (from supernets)
 
-    Needs optimization
+    Pass in a group filter to read from config, or a custom nFilter
 
     Examples:
     nst = all (all networks)
@@ -136,7 +136,10 @@ def check_net_filter(netDict, group):
     com = default:printer
     """
 
-    vDict = get_filter_dict(group)
+    if group:
+        vDict = get_filter_dict(group=group)
+    elif nFilter:
+        vDict = get_filter_dict(nFilter=nFilter)
 
     # Check filter against vDict
     for vrf in vDict.keys():
@@ -164,13 +167,19 @@ def check_net_filter(netDict, group):
 
 
 @functools.lru_cache(maxsize=1)
-def get_filter_dict(group):
-    """Process config for group and cache it for each call"""
+def get_filter_dict(group=None, nFilter=None):
+    """Process config for group or custom filter and cache it for each call"""
 
     vDict = dict()
+    gFilter = None
 
-    gFilter = config['NetAlertFilter'][group]
-
+    if group:
+        gFilter = config['NetAlertFilter'][group]
+    elif nFilter:
+        gFilter = nFilter
+    else:
+        raise Exception("Must pass in group or filter")
+
     # Split all VRFs by spaces
     vrfFilters = gFilter.rsplit()
 
@@ -213,7 +222,7 @@ def universal_text_search(text, vrange, rtype="TREE"):
     #Look for group filter first
     try:
         get_net_filter(text)
-        nglib.query.net.get_networks_on_group(text, rtype=rtype)
+        nglib.query.net.get_networks_on_filter(text, rtype=rtype)
         found = True
     except:
         pass

nglib/query/dev.py

@@ -34,6 +34,7 @@
 """
 import logging
 import re
+import sys
 import nglib
 import nglib.ngtree
 import nglib.ngtree.export
@@ -42,7 +43,7 @@
 
 verbose = 0
 
-def get_device(dev, rtype="TREE", vrange=None):
+def get_device(dev, rtype="NGTREE", vrange=None):
     """Get Switch perspective (neighbors, vlans, routed networks)"""
 
     rtypes = ('TREE', 'JSON', 'YAML', 'NGTREE')
@@ -52,7 +53,8 @@ def get_device(dev, rtype="TREE", vrange=None):
 
     ngtree = nglib.ngtree.get_ngtree(dev, tree_type="Device")
 
-    logger.info("Query: Device %s for %s", dev, nglib.user)
+    if rtype != "NGTREE":
+        logger.info("Query: Device %s for %s", dev, nglib.user)
 
 
     switch = nglib.bolt_ses.run(
@@ -62,7 +64,12 @@ def get_device(dev, rtype="TREE", vrange=None):
 
     for sw in switch:
 
-        ngtree['Distance'] = int(sw['distance'])
+        try:
+            ngtree['Distance'] = int(sw['distance'])
+        except TypeError:
+            print("Error, device not part of the topology:", dev, file=sys.stderr)
+            return
+
         ngtree['MGMT Group'] = sw['mgmt']
         if vrange:
             ngtree['VLAN Range'] = vrange
@@ -131,7 +138,7 @@ def get_device(dev, rtype="TREE", vrange=None):
         nglib.query.exp_ngtree(ngtree, rtype)
         return ngtree
 
-    print("No results found for device:", dev)
+    print("No results found for device:", dev, file=sys.stderr)
 
 def get_neighbors(dev):
     """
@@ -252,4 +259,21 @@ def get_vlans(dev, vrange=None):
             if vlan['mcount']:
                 vt['MAC Count'] = vlan['mcount']
     return vtree
+
+
+def get_devlist_vrf(vrf):
+    """Returns a list of devices that route a VRF"""
+
+    devices = nglib.bolt_ses.run(
+            'MATCH(v:VRF)-[e:VRF_ON]-(r:Router) WHERE v.name = {vrf} '
+            + 'RETURN r.name AS name ORDER BY name',
+            {"vrf": vrf})
+
+    devlist = []
+
+    for r in devices:
+        devlist.append(r["name"])
+
+    return devlist
+
 #END