From 6ade770bf1219436595c30496d67e30f7140f0e8 Mon Sep 17 00:00:00 2001 From: David Schott <36676176+daschott@users.noreply.github.com> Date: Thu, 18 May 2023 12:33:19 -0700 Subject: [PATCH 1/4] add monitorDNS.ps1 script Add script to monitor and track changes to DNS policylists and DNS rules in LB_DSR layer for all local endpoints --- Kubernetes/windows/debug/monitorDNS.ps1 | 187 ++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 Kubernetes/windows/debug/monitorDNS.ps1 diff --git a/Kubernetes/windows/debug/monitorDNS.ps1 b/Kubernetes/windows/debug/monitorDNS.ps1 new file mode 100644 index 00000000..6885a9a7 --- /dev/null +++ b/Kubernetes/windows/debug/monitorDNS.ps1 @@ -0,0 +1,187 @@ +Param( + [parameter(Mandatory = $false)] [string] $FileName = "dnsinfo.txt", + [parameter(Mandatory = $false)] [int] $WaitTime = 1200, # In seconds + [parameter(Mandatory = $false)] [int] $Interval = 3, # interval (in seconds) to query DNS rules + [parameter(Mandatory = $false)] [bool] $VerifyVfpRules = $true +) + +# Helper functions +function listDnsPolicies() { + $dnsPolicies = ((Get-HnsPolicyList | where { $_.Policies.InternalPort -EQ 53 } | where { $_.Policies.ExternalPort -EQ 53 }) | Select Policies, ID, References) + return $dnsPolicies +} + +function dnsPoliciesToString($dnsPolicies) { + $dnsPolicies | ForEach-Object { + $dnsPoliciesString = $dnsPoliciesString + ("`nID: $($_.ID) `nReferences: $($_.References)`nPolicies:$($_.Policies)") + } + return $dnsPoliciesString +} + +function getDnsRules($portGuid, $dnsServerIP) { + $lbRulesRaw = cmd /c "vfpctrl /port $portGuid /list-rule /layer LB_DSR" + + $lbRules = $lbRulesRaw | + Where-Object { $_ -match ' ' } | + ForEach-Object { + $_ -replace ' ', '' + } + # Filter out DNS rules + $dnsRules = $lbRules | + Where-Object { $_ -like "RULE:LB_DSR_*_*_$($dnsServerIP)_53_53_6" -or $_ -like "RULE:LB_DSR_*_*_$($dnsServerIP)_53_53_17" } + return $dnsRules, $lbRulesRaw +} + + +function consistentDnsRules($oldDnsRules, $currentDnsRules) { + if (($currentDnsRules.Count -ne 2) -or ($oldDnsRules.Count -ne 2)) { + return $false + } + for ($i = 0; $i -le (2); $i++) { + if ($oldDnsRules[$i] -ne $currentDnsRules[$i]) { + return $false + } + } + return $true +} + +function getPortGuidMap() { + $rawPortsOutput = cmd /c "vfpctrl /list-vmswitch-port" + # Some very case specific parsing + $rawPortsOutput = $rawPortsOutput | + Where-Object { $_ -match ' ' } | + ForEach-Object { + $_ -replace ' ', '' + } + + # Port names and MACs + $portMacs = $rawPortsOutput | + Where-Object { $_ -like 'Portname:*' -or $_ -like 'MACaddress:*' } + + $port = "" + $mac = "" + $macToPortGuids = @{} + $portMacs | + ForEach-Object { + $fields = $_ -split ":" + Switch ($fields[0]) { + "Portname" { $port = $fields[1] } + "MACaddress" { $mac = $fields[1] } + } + $macToPortGuids[$mac] = $port + } + return $macToPortGuids +} + +function getDnsRulesAll($endpoints, $dnsServerIP, $verbose = $false) { + $endpointDnsRules = @{} + $endpointLbRules = @{} + $macToPortGuids = getPortGuidMap + $expectedDnsRuleCount = ($endpoints).Count * 2 + $dnsRulesCount = 0 + $endpoints | + ForEach-Object { + $dnsRules, $lbRules = getDnsRules $macToPortGuids[$_.MACaddress] $dnsServerIP + $endpointDnsRules[$_.IPAddress] = $dnsRules + $endpointLbRules[$_.IPAddress] = $lbRules + $dnsRulesCount = $dnsRulesCount + $dnsRules.Count + if ($verbose) { + Write-Output "Found DNS rules for pod $($_.IPAddress):`n$($dnsRules)" >> $FileName + Write-Output "Found LB rules for pod $($_.IPAddress):`n$($lbRules)" >> $FileName + } + } + if ($dnsRulesCount -ne $expectedDnsRuleCount) { + Write-Output "Unexpected DNS rule count! Expected: $($expectedDnsRuleCount)`nActual: $($dnsRulesCount)." >> $FileName + } + else { + Write-Output "[OK] Found $($dnsRulesCount) DNS rules across all pods" >> $FileName + } + return $endpointDnsRules, $endpointLbRules +} + +# Main +Remove-Item $FileName -ErrorAction SilentlyContinue +while ((listDnsPolicies).Count -ne 2) { + Write-Output "Waiting for DNS policies..." + start-sleep 5 +} +Write-Output "Monitoring DNS rules in $FileName..." + +$oldDnsPolicies = listDnsPolicies +$dnsServerIP = $oldDnsPolicies[0].Policies.VIPs +Write-Output "[OK] Found DNS Server VIP $($dnsServerIP)" >> $FileName +Write-Output "[OK] Starting DNS policy: $(dnsPoliciesToString $oldDnsPolicies)" >> $FileName + +if ($VerifyVfpRules) { + # Get starting VFP DNS rules + $endpoints = (get-hnsendpoint | ? IsRemoteEndpoint -ne True) | Select-Object MACaddress, IPAddress + Write-Output "[OK] Querying starting DNS rules..." >> $FileName + $oldEndpointDnsRules, $oldEndpointLbRules = getDnsRulesAll $endpoints $dnsServerIP $true +} + +for ($i = 0; $i -le ($WaitTime / $Interval); $i++) { + $timeNow = Get-Date + Write-Output "#====== Iteration : $i . Time : $timeNow " >> $FileName + $iterationHealth = $true + # Verify HNS DNS policies are consistent + $currentDnsPolicies = listDnsPolicies + if (($currentDnsPolicies).Count -ne 2) { + Write-Output "DNS policies not found!`nOld: $(dnsPoliciesToString $oldDnsPolicies).`nNew: $(dnsPoliciesToString $currentDnsPolicies)" >> $FileName + # Skip analyzing the VFP rules until DNS policies are present again. + $iterationHealth = $false + Write-Output "#====== Iteration $i Completed. Health: $iterationHealth." >> $FileName + Start-Sleep -Seconds $Interval + continue + } + elseif ($currentDnsPolicies[0].ID -ne $oldDnsPolicies[0].ID) { + Write-Output "DNS policies have changed!`nOld: $(dnsPoliciesToString $oldDnsPolicies).`nNew: $(dnsPoliciesToString $currentDnsPolicies)" >> $FileName + Write-Output "Updating new DNS policy to $($currentDnsPolicies.ID)..." >> $FileName + $oldDnsPolicies = $currentDnsPolicies + $iterationHealth = $false + } + else { + Write-Output "[OK] DNS policies are consistent. Current: $($currentDnsPolicies.ID)" >> $FileName + } + + if ($VerifyVfpRules) { + # Verify DNS VFP rules are consistent across all endpoints + $endpoints = (get-hnsendpoint | ? IsRemoteEndpoint -ne True) | Select-Object MACaddress, IPAddress + $endpointDnsRules, $endpointLbRules = getDnsRulesAll $endpoints $dnsServerIP + # IP address in $endpoints currently exists. If it exists in old table, then pod was always here. + # If it does not exist in old table, then it is a new pod. Need to add it there. + $endpoints | + ForEach-Object { + if (-not ($endpointDnsRules.Keys.Contains($_.IPAddress))) { + Write-Output "DNS rules not found for pod $($_.IPAddress)!" >> $FileName + # Skip, DNS rules are not found... + $iterationHealth = $false + continue + } + elseif ( $endpointDnsRules[$_.IPAddress].Count -ne 2 ) { + Write-Output "DNS rules partially missing for pod $($_.IPAddress)!" >> $FileName + $iterationHealth = $false + } + elseif (-not ($oldEndpointDnsRules.Keys -contains $_.IPAddress)) { + # New pod + Write-Output "Found new pod with IP $($_.IPAddress)." >> $FileName + Write-Output "Found DNS rules for pod $($_.IPAddress):`n$($endpointDnsRules[$_.IPAddress])" >> $FileName + Write-Output "Found LB rules for pod $($_.IPAddress):`n$($endpointLbRules[$_.IPAddress])" >> $FileName + $oldEndpointDnsRules[$_.IPAddress] = $endpointDnsRules[$_.IPAddress] + $oldEndpointLbRules[$_.IPAddress] = $endpointLbRules[$_.IPAddress] + } + # If current != old, then something has changed. Print the rules. + if ((consistentDnsRules $oldEndpointDnsRules[$_.IPAddress] $endpointDnsRules[$_.IPAddress])) { + Write-Output "[OK] DNS rules are consistent for pod $($_.IPAddress)." >> $FileName + } + else { + Write-Output "DNS rules have changed for pod $($_.IPAddress)!.`nOld:`n$($oldEndpointLbRules[$_.IPAddress])`nNew:`n$($endpointLbRules[$_.IPAddress])" >> $FileName + Write-Output "Updating new DNS rules for pod $($_.IPAddress) to: $($endpointDnsRules[$_.IPAddress])..." >> $FileName + $iterationHealth = $false + $oldEndpointDnsRules[$_.IPAddress] = $endpointDnsRules[$_.IPAddress] + $oldEndpointLbRules[$_.IPAddress] = $endpointLbRules[$_.IPAddress] + } + } + } + Write-Output "Iteration $i Completed. Health: $iterationHealth." >> $FileName + Start-Sleep -Seconds $Interval +} \ No newline at end of file From 4f19744b1472cacfa5323ea658a5db3e3294d36f Mon Sep 17 00:00:00 2001 From: David Schott <36676176+daschott@users.noreply.github.com> Date: Thu, 18 May 2023 16:11:37 -0700 Subject: [PATCH 2/4] Add files via upload --- Kubernetes/windows/debug/monitorDNS.ps1 | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Kubernetes/windows/debug/monitorDNS.ps1 b/Kubernetes/windows/debug/monitorDNS.ps1 index 6885a9a7..93f29b3d 100644 --- a/Kubernetes/windows/debug/monitorDNS.ps1 +++ b/Kubernetes/windows/debug/monitorDNS.ps1 @@ -19,17 +19,24 @@ function dnsPoliciesToString($dnsPolicies) { } function getDnsRules($portGuid, $dnsServerIP) { - $lbRulesRaw = cmd /c "vfpctrl /port $portGuid /list-rule /layer LB_DSR" + $lbRulesRaw = cmd /c "vfpctrl /port $portGuid /list-rule /layer LB_DSR /group LB_DSR_IPv4_OUT" $lbRules = $lbRulesRaw | Where-Object { $_ -match ' ' } | ForEach-Object { $_ -replace ' ', '' } - # Filter out DNS rules - $dnsRules = $lbRules | + # Filter out DNS rule names + $dnsRuleNames = $lbRules | Where-Object { $_ -like "RULE:LB_DSR_*_*_$($dnsServerIP)_53_53_6" -or $_ -like "RULE:LB_DSR_*_*_$($dnsServerIP)_53_53_17" } - return $dnsRules, $lbRulesRaw + + $dnsRules = @() + $dnsRuleNames | + ForEach-Object { + $fields = $_ -split ":" + $dnsRules += cmd /c "vfpctrl /port $portGuid /list-rule /layer LB_DSR /group LB_DSR_IPv4_OUT /rule $($fields[1])" + } + return $dnsRuleNames, $dnsRules } From aa543fbfc0b2f787531020f2fe59e373c2400b08 Mon Sep 17 00:00:00 2001 From: David Schott <36676176+daschott@users.noreply.github.com> Date: Thu, 18 May 2023 16:20:40 -0700 Subject: [PATCH 3/4] Add DNS rule-level counters --- Kubernetes/windows/debug/monitorDNS.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Kubernetes/windows/debug/monitorDNS.ps1 b/Kubernetes/windows/debug/monitorDNS.ps1 index 93f29b3d..ae6b8fd2 100644 --- a/Kubernetes/windows/debug/monitorDNS.ps1 +++ b/Kubernetes/windows/debug/monitorDNS.ps1 @@ -34,7 +34,7 @@ function getDnsRules($portGuid, $dnsServerIP) { $dnsRuleNames | ForEach-Object { $fields = $_ -split ":" - $dnsRules += cmd /c "vfpctrl /port $portGuid /list-rule /layer LB_DSR /group LB_DSR_IPv4_OUT /rule $($fields[1])" + $dnsRules += cmd /c "vfpctrl /port $portGuid /get-rule-counter /layer LB_DSR /group LB_DSR_IPv4_OUT /rule $($fields[1])" } return $dnsRuleNames, $dnsRules } @@ -101,7 +101,7 @@ function getDnsRulesAll($endpoints, $dnsServerIP, $verbose = $false) { Write-Output "Unexpected DNS rule count! Expected: $($expectedDnsRuleCount)`nActual: $($dnsRulesCount)." >> $FileName } else { - Write-Output "[OK] Found $($dnsRulesCount) DNS rules across all pods" >> $FileName + Write-Output "[OK] Found $($dnsRulesCount) DNS rules across all pods." >> $FileName } return $endpointDnsRules, $endpointLbRules } From f1e0a2dd40588a44532a42f1b0de07978ae5cdaa Mon Sep 17 00:00:00 2001 From: David Schott <36676176+daschott@users.noreply.github.com> Date: Thu, 18 May 2023 20:53:04 -0700 Subject: [PATCH 4/4] Print DNS rule-level counters --- Kubernetes/windows/debug/monitorDNS.ps1 | 95 ++++++++++++++++++------- 1 file changed, 69 insertions(+), 26 deletions(-) diff --git a/Kubernetes/windows/debug/monitorDNS.ps1 b/Kubernetes/windows/debug/monitorDNS.ps1 index ae6b8fd2..71e77dca 100644 --- a/Kubernetes/windows/debug/monitorDNS.ps1 +++ b/Kubernetes/windows/debug/monitorDNS.ps1 @@ -18,6 +18,18 @@ function dnsPoliciesToString($dnsPolicies) { return $dnsPoliciesString } +function dnsCountersToString($dnsRuleNames, $dnsCounters){ + $dnsRuleNames | ForEach-Object { + if (-not ($dnsCounters.Keys -contains $_)){ + return "Counters not found for $($_)!" + } elseif ($dnsCounters[$_].Count -ne 4){ + return "Expected number of counters not found for $($_)!" + } else{ + return "Rule: $($_)`nMatched: $($dnsCounters[$_][0]), Dropped:$($dnsCounters[$_][1]), Pending:$($dnsCounters[$_][2]), Dropped unified flows:$($dnsCounters[$_][3])`n" + } + } +} + function getDnsRules($portGuid, $dnsServerIP) { $lbRulesRaw = cmd /c "vfpctrl /port $portGuid /list-rule /layer LB_DSR /group LB_DSR_IPv4_OUT" @@ -27,16 +39,42 @@ function getDnsRules($portGuid, $dnsServerIP) { $_ -replace ' ', '' } # Filter out DNS rule names - $dnsRuleNames = $lbRules | + $dnsRuleNamesRaw = $lbRules | Where-Object { $_ -like "RULE:LB_DSR_*_*_$($dnsServerIP)_53_53_6" -or $_ -like "RULE:LB_DSR_*_*_$($dnsServerIP)_53_53_17" } - $dnsRules = @() - $dnsRuleNames | - ForEach-Object { - $fields = $_ -split ":" - $dnsRules += cmd /c "vfpctrl /port $portGuid /get-rule-counter /layer LB_DSR /group LB_DSR_IPv4_OUT /rule $($fields[1])" - } - return $dnsRuleNames, $dnsRules + # Get DNS rules and counters + $dnsRulesRaw = @() + $dnsRuleNames = @() + $dnsRulesToCounters = @{} + $dnsRuleNamesRaw | + ForEach-Object { + # Get DNS rule name + $fields = $_ -split ":" + $dnsRuleName = $fields[1] + $dnsRuleNames += $dnsRuleName + # Query VFP for specific DNS rule and counters + $dnsRuleRaw = cmd /c "vfpctrl /port $portGuid /get-rule-counter /layer LB_DSR /group LB_DSR_IPv4_OUT /rule $($dnsRuleName)" + $dnsRulesRaw += $dnsRuleRaw + # Parse counters from raw DNS rule output + $dnsCounters = @() # Matched, Dropped, Pending, Packets, and Dropped unified flows + # Filter out whitespaces + $dnsRule = $dnsRuleRaw | + Where-Object { $_ -match ' ' } | + ForEach-Object { + $_ -replace ' ', '' + } + # Get Counters + $dnsRule = $dnsRule | + Where-Object { $_ -like "*packets:*" -or $_ -like "*flows:*" } + $dnsRule | + ForEach-Object { + $fields = $_ -split ":" + $dnsCounters += $fields[1] + } + $dnsRulesToCounters[$dnsRuleName] = $dnsCounters + } + + return $dnsRuleNames, $dnsRulesRaw, $dnsRulesToCounters } @@ -63,7 +101,7 @@ function getPortGuidMap() { # Port names and MACs $portMacs = $rawPortsOutput | - Where-Object { $_ -like 'Portname:*' -or $_ -like 'MACaddress:*' } + Where-Object { $_ -like 'Portname:*' -or $_ -like 'MACaddress:*' } $port = "" $mac = "" @@ -81,16 +119,18 @@ function getPortGuidMap() { } function getDnsRulesAll($endpoints, $dnsServerIP, $verbose = $false) { + $endpointDnsRuleNames = @{} $endpointDnsRules = @{} - $endpointLbRules = @{} + $endpointDnsCounters = @{} $macToPortGuids = getPortGuidMap $expectedDnsRuleCount = ($endpoints).Count * 2 $dnsRulesCount = 0 $endpoints | ForEach-Object { - $dnsRules, $lbRules = getDnsRules $macToPortGuids[$_.MACaddress] $dnsServerIP - $endpointDnsRules[$_.IPAddress] = $dnsRules - $endpointLbRules[$_.IPAddress] = $lbRules + $dnsRules, $lbRules, $dnsCounters = getDnsRules $macToPortGuids[$_.MACaddress] $dnsServerIP + $endpointDnsRuleNames[$_.IPAddress] = $dnsRules + $endpointDnsRules[$_.IPAddress] = $lbRules + $endpointDnsCounters[$_.IPAddress] = $dnsCounters $dnsRulesCount = $dnsRulesCount + $dnsRules.Count if ($verbose) { Write-Output "Found DNS rules for pod $($_.IPAddress):`n$($dnsRules)" >> $FileName @@ -103,7 +143,7 @@ function getDnsRulesAll($endpoints, $dnsServerIP, $verbose = $false) { else { Write-Output "[OK] Found $($dnsRulesCount) DNS rules across all pods." >> $FileName } - return $endpointDnsRules, $endpointLbRules + return $endpointDnsRuleNames, $endpointDnsRules, $endpointDnsCounters } # Main @@ -123,7 +163,7 @@ if ($VerifyVfpRules) { # Get starting VFP DNS rules $endpoints = (get-hnsendpoint | ? IsRemoteEndpoint -ne True) | Select-Object MACaddress, IPAddress Write-Output "[OK] Querying starting DNS rules..." >> $FileName - $oldEndpointDnsRules, $oldEndpointLbRules = getDnsRulesAll $endpoints $dnsServerIP $true + $oldEndpointDnsRuleNames, $oldEndpointDnsRules, $oldEndpointDnsCounters = getDnsRulesAll $endpoints $dnsServerIP $true } for ($i = 0; $i -le ($WaitTime / $Interval); $i++) { @@ -153,39 +193,42 @@ for ($i = 0; $i -le ($WaitTime / $Interval); $i++) { if ($VerifyVfpRules) { # Verify DNS VFP rules are consistent across all endpoints $endpoints = (get-hnsendpoint | ? IsRemoteEndpoint -ne True) | Select-Object MACaddress, IPAddress - $endpointDnsRules, $endpointLbRules = getDnsRulesAll $endpoints $dnsServerIP + $endpointDnsRuleNames, $endpointDnsRules, $endpointDnsCounters = getDnsRulesAll $endpoints $dnsServerIP # IP address in $endpoints currently exists. If it exists in old table, then pod was always here. # If it does not exist in old table, then it is a new pod. Need to add it there. $endpoints | ForEach-Object { - if (-not ($endpointDnsRules.Keys.Contains($_.IPAddress))) { + if (-not ($endpointDnsRuleNames.Keys -contains $_.IPAddress)) { Write-Output "DNS rules not found for pod $($_.IPAddress)!" >> $FileName # Skip, DNS rules are not found... $iterationHealth = $false continue } - elseif ( $endpointDnsRules[$_.IPAddress].Count -ne 2 ) { + elseif ( $endpointDnsRuleNames[$_.IPAddress].Count -ne 2 ) { Write-Output "DNS rules partially missing for pod $($_.IPAddress)!" >> $FileName $iterationHealth = $false } - elseif (-not ($oldEndpointDnsRules.Keys -contains $_.IPAddress)) { + elseif (-not ($oldEndpointDnsRuleNames.Keys -contains $_.IPAddress)) { # New pod Write-Output "Found new pod with IP $($_.IPAddress)." >> $FileName - Write-Output "Found DNS rules for pod $($_.IPAddress):`n$($endpointDnsRules[$_.IPAddress])" >> $FileName - Write-Output "Found LB rules for pod $($_.IPAddress):`n$($endpointLbRules[$_.IPAddress])" >> $FileName + Write-Output "Found DNS rules for pod $($_.IPAddress):`n$($endpointDnsRuleNames[$_.IPAddress])" >> $FileName + Write-Output "Found LB rules for pod $($_.IPAddress):`n$($endpointDnsRules[$_.IPAddress])" >> $FileName + $oldEndpointDnsRuleNames[$_.IPAddress] = $endpointDnsRuleNames[$_.IPAddress] $oldEndpointDnsRules[$_.IPAddress] = $endpointDnsRules[$_.IPAddress] - $oldEndpointLbRules[$_.IPAddress] = $endpointLbRules[$_.IPAddress] + $oldEndpointDnsCounters[$_.IPAddress] = $endpointDnsCounters[$_.IPAddress] } # If current != old, then something has changed. Print the rules. - if ((consistentDnsRules $oldEndpointDnsRules[$_.IPAddress] $endpointDnsRules[$_.IPAddress])) { + if ((consistentDnsRules $oldEndpointDnsRuleNames[$_.IPAddress] $endpointDnsRuleNames[$_.IPAddress])) { Write-Output "[OK] DNS rules are consistent for pod $($_.IPAddress)." >> $FileName + Write-Output "DNS rule counters for pod $($_.IPAddress):`n$(dnsCountersToString $endpointDnsRuleNames[$_.IPAddress] $endpointDnsCounters[$_.IPAddress])" >> $FileName } else { - Write-Output "DNS rules have changed for pod $($_.IPAddress)!.`nOld:`n$($oldEndpointLbRules[$_.IPAddress])`nNew:`n$($endpointLbRules[$_.IPAddress])" >> $FileName - Write-Output "Updating new DNS rules for pod $($_.IPAddress) to: $($endpointDnsRules[$_.IPAddress])..." >> $FileName + Write-Output "DNS rules have changed for pod $($_.IPAddress)!.`nOld:`n$($oldEndpointDnsRules[$_.IPAddress])`nNew:`n$($endpointDnsRules[$_.IPAddress])" >> $FileName + Write-Output "Updating DNS rules & counters for pod $($_.IPAddress) to: $($endpointDnsRuleNames[$_.IPAddress])..." >> $FileName $iterationHealth = $false + $oldEndpointDnsRuleNames[$_.IPAddress] = $endpointDnsRuleNames[$_.IPAddress] $oldEndpointDnsRules[$_.IPAddress] = $endpointDnsRules[$_.IPAddress] - $oldEndpointLbRules[$_.IPAddress] = $endpointLbRules[$_.IPAddress] + $oldEndpointDnsCounters[$_.IPAddress] = $endpointDnsCounters[$_.IPAddress] } } }