Network health script.

scripts/networkhealth/README.md

@@ -35,4 +35,30 @@ If the `-OutputMode` is set to `Event` or `All`, the script will register a new
     Example:
     kubectl logs -l name=networkhealth --all-containers=true >> networkhealth.txt
     Provide the generated networkhealth.txt
+```
+## Command to run startNetworkDiagnostics
+```
+Normal Execution: .\startNetworkDiagnostics.ps1 -TimeIntervalInSeconds 30 -PrintMatchedRules $true -PodNamePrefixes tcp-server,tcp-client
+
+Execution with DNS Packet Capture: .\startNetworkDiagnostics.ps1 -DnsPktCap $true
+
+Execution with DualStack Test: .\startNetworkDiagnostics.ps1 -DualStack $true
+
+Execution with Vfp Rule Counter Dump for Pods : .\startNetworkDiagnostics.ps1 -PodNamePrefixes tcp-client,tcp-server
+
+Execution with printing matched rule counter : .\startNetworkDiagnostics.ps1 -PodNamePrefixes tcp-client,tcp-server -PrintMatchedRules $true
+
+Execution with validate loadbalancer rules for Service IPS : .\startNetworkDiagnostics.ps1 -ServiceIPS "10.0.0.1,10.0.0.2"
+
+```
+
+## Command to run vfpDropCounterMetrics
+```
+.\vfpDropCounterMetrics.ps1 -TimeIntervalInSeconds 30 -PrintMatchedRules $true -PodNamePrefixes tcp-server,tcp-client
+```
+
+## DNS Health Check
+```
+Invoke-WebRequest https://raw.githubusercontent.com/microsoft/wcnscripts/2ea829ebaaf523cf58ef8e64120e54849eb4bd51/scripts/networkhealth/startNetworkDiagnostics.ps1 -OutFile startNetworkDiagnostics.ps1
+.\startNetworkDiagnostics.ps1
 ```

scripts/networkhealth/checkLbDsrMissing.ps1

@@ -0,0 +1,84 @@
+param (
+    [Parameter(Mandatory=$true)][string]$ServiceIPS = "",
+    [Parameter(Mandatory=$false)][string]$RunFromOutside = $false,
+    [Parameter(Mandatory=$false)][string]$CopyScriptToNode = $false,
+    [Parameter(Mandatory=$false)][string]$HpcPodPrefix = "hpc",
+    [Parameter(Mandatory=$false)][string]$HpcNamepsace = "demo",
+    [Parameter(Mandatory=$false)][string]$Layer = "LB_DSR",
+    [Parameter(Mandatory=$false)][string]$Group = "LB_DSR_IPv4_OUT"
+)
+
+# Eg: .\checkLbDsrMissing.ps1 -ServiceIPS "20.51.25.211,10.0.10.189,10.0.92.4"
+
+$Logfile = "health.log"
+
+function LogError {
+    param (
+        [parameter(Mandatory=$true)][string] $message
+    )
+    Write-Host $message -ForegroundColor Red
+    Add-content $Logfile -value "[FAILED] $message"
+}
+
+function LogSuccess {
+    param (
+        [parameter(Mandatory=$true)][string] $message
+    )
+    Write-Host $message -ForegroundColor Green
+    Add-content $Logfile -value "[SUCCESS] $message"
+}
+
+function CheckLbDsrRuleMissing {
+
+    Write-Host "Checking $Layer Rule missing"
+    $lbDsrRuleMissing = $false
+
+    $serviceIPList = $ServiceIPS.Split(",")
+
+    $portNameList = vfpctrl /list-vmswitch-port | sls "Port name"
+
+    foreach($portName in $portNameList) {
+	    $portId = $portName.ToString().Split(":")[1].Trim()
+	    $entries = vfpctrl /port $portId /layer $Layer /list-group
+	    if($entries.Count -le 8) {
+		    continue
+	    }
+	    foreach($serviceIP in $serviceIPList) {
+		    $entries = vfpctrl /port $portId /layer $Layer /group $Group /list-rule | sls $serviceIP
+		    if($entries.Count -le 1) {
+			    LogError "VFP $Layer Rule missing for Service IP :  $serviceIP in VFP Port : $portName"
+                $lbDsrRuleMissing = $true
+		    } else {
+			    LogSuccess "VFP $Layer Rule present for Service IP :  $serviceIP in VFP Port : $portName"
+		    }
+	    }
+    }
+
+    if($lbDsrRuleMissing){
+        LogError "Mitigation : Restart-Service -f kubeproxy "
+        return
+    }
+
+    LogSuccess "No issues identified with $Layer Rule Missing for Service IPS : $ServiceIPS."
+}
+
+function ExecFromOutside {
+    $hpcPods = kubectl get pods -n $HpcNamepsace | sls $HpcPodPrefix
+    if($CopyScriptToNode -eq $true) {
+        foreach($hpcPod in $hpcPods) {
+            $podId = $hpcPod.ToString().Split(" ")[0].Trim()
+            Write-Host "Copying script to $podId"
+            kubectl cp .\checkLbDsrMissing.ps1 "$podId`:`checkLbDsrMissing.ps1" -n $HpcNamepsace
+        }
+    }
+    foreach($hpcPod in $hpcPods) {
+	    $podId = $hpcPod.ToString().Split(" ")[0].Trim()
+	    kubectl exec -it $podId -n $HpcNamepsace -- powershell ".\checkLbDsrMissing.ps1 -ServiceIPS '$ServiceIPS' -Layer $Layer -Group $Group"
+    }
+}
+
+if($RunFromOutside -eq $true) {
+    ExecFromOutside
+}
+
+CheckLbDsrRuleMissing

scripts/networkhealth/checkPortExhaustion.ps1

@@ -0,0 +1,110 @@
+$Logfile = ".\health.log"
+$minThreshold = 50
+
+function LogMessage {
+    param (
+        [parameter(Mandatory=$true)][string] $message
+    )
+    Add-content $Logfile -value "$message"
+}
+
+function CountAvailableEphemeralPorts([string]$protocol = "TCP") {
+
+    [uint32]$portRangeSize = 64
+    # First, remove all the text bells and whistle (plain text, table headers, dashes, empty lines, ...) from netsh output 
+    $tcpRanges = (netsh int ipv4 sh excludedportrange $protocol) -replace "[^0-9,\ ]", '' | ? { $_.trim() -ne "" }
+
+    # Then, remove any extra space characters. Only capture the numbers representing the beginning and end of range
+    $tcpRangesArray = $tcpRanges -replace "\s+(\d+)\s+(\d+)\s+", '$1,$2' | ConvertFrom-String -Delimiter ","
+    #Convert from PSCustomObject to Object[] type
+    $tcpRangesArray = @($tcpRangesArray)
+
+    # Extract the ephemeral ports ranges
+    $EphemeralPortRange = (netsh int ipv4 sh dynamicportrange $protocol) -replace "[^0-9]", '' | ? { $_.trim() -ne "" }
+    $EphemeralPortStart = [Convert]::ToUInt32($EphemeralPortRange[0])
+    $EphemeralPortEnd = $EphemeralPortStart + [Convert]::ToUInt32($EphemeralPortRange[1]) - 1
+
+    # Find the external interface
+    $externalInterfaceIdx = (Get-NetRoute -DestinationPrefix "0.0.0.0/0")[0].InterfaceIndex
+    $hostIP = (Get-NetIPConfiguration -ifIndex $externalInterfaceIdx).IPv4Address.IPAddress
+
+    # Extract the used TCP ports from the external interface
+    $usedTcpPorts = (Get-NetTCPConnection -LocalAddress $hostIP -ErrorAction Ignore).LocalPort
+    $usedTcpPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
+
+    # Extract the used TCP ports from the 0.0.0.0 interface
+    $usedTcpGlobalPorts = (Get-NetTCPConnection -LocalAddress "0.0.0.0" -ErrorAction Ignore).LocalPort
+    $usedTcpGlobalPorts | % { $tcpRangesArray += [pscustomobject]@{P1 = $_; P2 = $_ } }
+    # Sort the list and remove duplicates
+    $tcpRangesArray = ($tcpRangesArray | Sort-Object { $_.P1 } -Unique)
+
+    $tcpRangesList = New-Object System.Collections.ArrayList($null)
+    $tcpRangesList.AddRange($tcpRangesArray)
+
+    # Remove overlapping ranges
+    for ($i = $tcpRangesList.P1.Length - 2; $i -gt 0 ; $i--) { 
+        if ($tcpRangesList[$i].P2 -gt $tcpRangesList[$i + 1].P1 ) { 
+            $tcpRangesList.Remove($tcpRangesList[$i + 1])
+            $i++
+        } 
+    }
+
+    # Remove the non-ephemeral port reservations from the list
+    $filteredTcpRangeArray = $tcpRangesList | ? { $_.P1 -ge $EphemeralPortStart }
+    $filteredTcpRangeArray = $filteredTcpRangeArray | ? { $_.P2 -le $EphemeralPortEnd }
+
+    if ($null -eq $filteredTcpRangeArray) {
+        $freeRanges = @($EphemeralPortRange[1])
+    }
+    else {
+        $freeRanges = @()
+        # The first free range goes from $EphemeralPortStart to the beginning of the first reserved range
+        $freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[0].P1) - $EphemeralPortStart)
+
+        for ($i = 1; $i -lt $filteredTcpRangeArray.length; $i++) {
+            # Subsequent free ranges go from the end of the previous reserved range to the beginning of the current reserved range
+            $freeRanges += ([Convert]::ToUInt32($filteredTcpRangeArray[$i].P1) - [Convert]::ToUInt32($filteredTcpRangeArray[$i - 1].P2) - 1)
+        }
+
+        # The last free range goes from the end of the last reserved range to $EphemeralPortEnd
+        $freeRanges += ($EphemeralPortEnd - [Convert]::ToUInt32($filteredTcpRangeArray[$filteredTcpRangeArray.length - 1].P2))
+    }
+
+    # Count the number of available free ranges
+    [uint32]$freeRangesCount = 0
+    ($freeRanges | % { $freeRangesCount += [Math]::Floor($_ / $portRangeSize) } )
+
+    return $freeRangesCount
+}
+
+function CheckPortExhaustion {
+    Write-Host "Checking Port Exhaustion"
+    $avTcpPorts = CountAvailableEphemeralPorts -protocol TCP
+    if($avTcpPorts -lt $minThreshold) {
+        $message = "Available TCP ports are $avTcpPorts. Port exhaustion suspected."
+        Write-Host "$message" -ForegroundColor Red
+        LogMessage -message $message
+        return $true
+    }
+    $avUdpPorts = CountAvailableEphemeralPorts -protocol UDP
+    if($avTcpPorts -lt $minThreshold) {
+        $message = "Available UDP ports are $avUdpPorts. Port exhaustion suspected."
+        Write-Host "$message" -ForegroundColor Red
+        LogMessage -message $message
+        return $true
+    }
+    Write-Host "Available TCP Ports :  $avTcpPorts , UDP Ports : $avUdpPorts . No port exhaustion suspected." -ForegroundColor Green
+    return $false
+}
+
+$i = 0
+Remove-Item $Logfile -ErrorAction Ignore
+
+While($true) {
+    $i++
+    Write-Host "#============== Iteration : $i"
+    if(CheckPortExhaustion) {
+        Write-Host "DNS Issue Found." -ForegroundColor Red
+    }
+    Start-Sleep -Seconds 10
+}