Skip to content

Commit

Permalink
Merge pull request #909 from netenglabs/path-improv
Browse files Browse the repository at this point in the history 
Path improvements to support searching even if source is not in arpnd tables
  • Loading branch information
@ddutt
ddutt authored Jul 17, 2023
2 parents 0c0d39e + 4ed54aa commit 59a2a32
Show file tree
Hide file tree
Showing 6 changed files with 527 additions and 27 deletions.
38 changes: 33 additions & 5 deletions suzieq/engines/pandas/engineobj.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,15 +105,15 @@ def _is_any_in_list(self, column: pd.Series, values: List) -> pd.Series:
return column.apply(lambda x: any(v in x for v in values))

def _is_in_subnet(self, addr: pd.Series, net: str) -> pd.Series:
"""Check if the IP addresses in a Pandas dataframe
belongs to the given subnet
"""check which of the addresses belongs to a given subnet
Used to implement the prefix filter of arpnd/address/dhcp
Args:
addr (PandasObject): the collection of ip addresses to check
net: (str): network id of the subnet
addr (pd.Series): the pandas series of ip addresses to check
net: (str): the IP network to check the addresses against
Returns:
PandasObject: A collection of bool reporting the result
pd.Series: A collection of bool reporting the result
"""
network = ip_network(net)
if isinstance(addr.iloc[0], np.ndarray):
Expand All @@ -127,6 +127,34 @@ def _is_in_subnet(self, addr: pd.Series, net: str) -> pd.Series:
False if not a else ip_address(a.split("/")[0]) in network)
)

def _in_subnet_series(self, addr: str, net: pd.Series) -> pd.Series:
"""Check if an addr is in any of series' subnets
unlike is_in_subnet which checks if a pandas series of addresses
belongs in a given subnet, this routine checks if a given address
belongs to any of the provided pandas series of subnets. THis is
currently used in path to identify an SVI for a given address
Args:
addr (str): the address we're checking for
net: (pd.Series): the pandas series of subnets
Returns:
pd.Series: A collection of bool reporting the result
"""
address = ip_address(addr)
if isinstance(net.iloc[0], np.ndarray):
return net.apply(lambda x, network:
False if not x.any()
else any(address in ip_network(a, strict=False)
for a in x if a != '0.0.0.0/0'),
args=(address,))
else:
return net.apply(lambda a: (
False if not a or a == '0.0.0.0/0'
else address in ip_network(a, strict=False))
)

def _check_ipvers(self, addr: pd.Series, version: int) -> pd.Series:
"""Check if the IP version of addresses in a Pandas dataframe
correspond to the given version
Expand Down
58 changes: 44 additions & 14 deletions suzieq/engines/pandas/path.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import Optional, List, Any, Iterable
from ipaddress import ip_network, ip_address
from collections import OrderedDict
from itertools import repeat
Expand All @@ -19,7 +20,7 @@ class PathObj(SqPandasEngine):
'''Backend class to handle manipulating virtual table, path, with pandas'''

@staticmethod
def table_name():
def table_name() -> str:
'''Table name'''
return 'path'

Expand Down Expand Up @@ -137,7 +138,34 @@ def _init_dfs(self, ns, source, dest):
self._srcnode_via_arp = True

if self._src_df.empty:
raise AttributeError(f"Invalid src {source}")
# See if we can find an mlag pair of devices that contains the SVI
# pandas has a bug that prevents us from using startswith with a
# tuple such as ("Vlan", "vlan", "irb.") directly instead of using
# the @svi_names trick
# pylint: disable=unused-variable
svi_names = tuple(["Vlan", "vlan", "irb."]) # noqa: F841
if ':' in source:
self._src_df = (
self._if_df.query(
f'@self._in_subnet_series("{source}", ip6AddressList)'
' and ifname.str.startswith(@svi_names)')
)
else:
self._src_df = (
self._if_df.query(
f'@self._in_subnet_series("{source}", ipAddressList)'
' and ifname.str.startswith(@svi_names)')
)
if not self._src_df.empty:
hosts = self._src_df.hostname.unique().tolist()
if len(hosts) > 2:
raise ValueError(
'source not in ARP and SVI on too many hosts')
self._srcnode_via_arp = True

if self._src_df.empty:
raise AttributeError(f"Unable to find starting node for {source}")

src_hostname = self._src_df.hostname.unique().tolist()[0]

if self._src_df.hostname.nunique() == 1 and len(self._src_df) > 1:
Expand Down Expand Up @@ -241,7 +269,7 @@ def _get_vrf(self, hostname: str, ifname: str, addr: str) -> str:

return vrf

def _find_fhr_df(self, device: str, ip: str) -> pd.DataFrame:
def _find_fhr_df(self, device: Optional[str], ip: str) -> pd.DataFrame:
"""Find Firstt Hop Router's iface DF for a given IP and device.
The logic in finding the next hop router is:
find the arp table entry corresponding to the IP provided;
Expand Down Expand Up @@ -317,12 +345,12 @@ def _get_if_vlan(self, device: str, ifname: str) -> int:
(self._if_df["ifname"] == ifname)]

if oif_df.empty:
return []
return -1

return oif_df.iloc[0]["vlan"]

def _get_l2_nexthop(self, device: str, vrf: str, dest: str,
macaddr: str, protocol: str) -> list:
macaddr: Optional[str], protocol: str) -> list:
"""Get the bridged/tunnel nexthops
We're passing protocol because we need to keep the return
match the other get nexthop function returns. We don't really
Expand Down Expand Up @@ -387,7 +415,7 @@ def _get_l2_nexthop(self, device: str, vrf: str, dest: str,

def _get_underlay_nexthop(self, hostname: str, vtep_list: list,
vrf_list: list,
is_overlay: bool) -> pd.DataFrame:
is_overlay: bool) -> List[Any]:
"""Return the underlay nexthop given the Vtep and VRF"""

# WARNING: This function is incomplete right now
Expand Down Expand Up @@ -486,7 +514,7 @@ def _handle_recursive_route(self, df: pd.DataFrame,
return df

def _get_nexthops(self, device: str, vrf: str, dest: str, is_l2: bool,
vtep: str, macaddr: str) -> list:
vtep: str, macaddr: str) -> Iterable:
"""Get nexthops (oif + IP + overlay) or just oif for given host/vrf.
The overlay is a bit indicating we're getting into overlay or not.
Expand Down Expand Up @@ -612,6 +640,7 @@ def _get_nh_with_peer(self, device: str, vrf: str, dest: str, is_l2: bool,
for (nhip, iface, overlay, l2hop, protocol,
timestamp) in new_nexthop_list:
df = pd.DataFrame()
arpdf = pd.DataFrame()
errormsg = ''
if l2hop and macaddr and not overlay:
if (not nhip or nhip == 'None') and iface:
Expand Down Expand Up @@ -721,7 +750,7 @@ def _get_nh_with_peer(self, device: str, vrf: str, dest: str, is_l2: bool,
'state!="failed"')
if not revarp_df.empty:
df = df.query(f'ifname == "{revarp_df.oif.iloc[0]}"')
df.apply(lambda x, nexthops:
df.apply(lambda x, nexthops: # type: ignore
nexthops.append((iface, x['hostname'],
x['ifname'], overlay,
l2hop, nhip,
Expand Down Expand Up @@ -761,8 +790,8 @@ def get(self, **kwargs) -> pd.DataFrame:
if not src or not dest:
raise AttributeError("Must specify trace source and dest")

srcvers = ip_network(src, strict=False)._version
dstvers = ip_network(dest, strict=False)._version
srcvers = ip_network(src, strict=False).version
dstvers = ip_network(dest, strict=False).version
if srcvers != dstvers:
raise AttributeError(
"Source and Dest MUST belong to same address familt")
Expand All @@ -771,7 +800,8 @@ def get(self, **kwargs) -> pd.DataFrame:
self._init_dfs(self.namespace, src, dest)

devices_iifs = OrderedDict()
src_mtu = None
src_mtu: int = MAX_MTU + 1
item = None
for i in range(len(self._src_df)):
item = self._src_df.iloc[i]
devices_iifs[f'{item.hostname}/'] = {
Expand All @@ -793,9 +823,9 @@ def get(self, **kwargs) -> pd.DataFrame:
"l3_visited_devices": set(),
"l2_visited_devices": set()
}
if src_mtu is None or (item.get('mtu', 0) < src_mtu):
src_mtu = item.get('mtu', 0)
if not dvrf:
if (src_mtu > MAX_MTU) or (item.get('mtu', 0) < src_mtu):
src_mtu = item.get('mtu', 0) # type: ignore
if not dvrf and item is not None:
dvrf = item['master']
if not dvrf:
dvrf = "default"
Expand Down
112 changes: 110 additions & 2 deletions tests/integration/sqcmds/cumulus-samples/path.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1959,9 +1959,111 @@ tests:
- command: path show --dest=172.16.2.104 --src=172.16.1.104 --namespace=dual-evpn
--format=json
data-directory: tests/data/parquet/
error:
error: '[{"error": "ERROR: Invalid src 172.16.1.104"}]'
marks: path show cumulus
output: '[{"pathid": 1, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01",
"iif": "vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false,
"mtuMatch": true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup":
"172.16.2.0/24", "vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1",
"hopError": "", "timestamp": 1616644822008}, {"pathid": 1, "hopCount": 1, "namespace":
"dual-evpn", "hostname": "spine01", "iif": "swp6", "oif": "swp3", "vrf": "default",
"isL2": true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216,
"protocol": "bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null,
"nexthopIp": "", "hopError": "", "timestamp": 1616644822008}, {"pathid": 1, "hopCount":
2, "namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 2, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 2, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine01", "iif": "swp5", "oif": "swp3", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 2, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 3, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822008}, {"pathid": 3, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine01", "iif": "swp6", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 3, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
{"pathid": 4, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 4, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine01", "iif": "swp5", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 4, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp1", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
{"pathid": 5, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822008}, {"pathid": 5, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp6", "oif": "swp3", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 5, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 6, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 6, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp5", "oif": "swp3", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 6, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
{"pathid": 7, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822008}, {"pathid": 7, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp6", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 7, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
{"pathid": 8, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
"", "timestamp": 1616644822167}, {"pathid": 8, "hopCount": 1, "namespace": "dual-evpn",
"hostname": "spine02", "iif": "swp5", "oif": "swp4", "vrf": "default", "isL2":
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 8, "hopCount": 2,
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp2", "oif": "bond02",
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983}]'
- command: path show --dest=10.0.0.11 --src=10.0.0.14 --namespace=ospf-single --format=json
data-directory: tests/data/parquet/
marks: path show cumulus
Expand Down Expand Up @@ -3405,3 +3507,9 @@ tests:
marks: path top cumulus
output: '[{"hostname": "leaf04"}, {"hostname": "leaf04"}, {"hostname": "spine01"},
{"hostname": "leaf01"}, {"hostname": "spine02"}]'
- command: path show --dest=172.16.2.104 --src=172.16.21.104 --namespace=dual-evpn
--format=json
data-directory: tests/data/parquet/
error:
error: '[{"error": "ERROR: Unable to find starting node for 172.16.21.104"}]'
marks: path show cumulus
Loading

0 comments on commit 59a2a32

Please sign in to comment.