AzStackHciNetwork/AzStackHci.Network.Helpers.psm1
|
Import-LocalizedData -BindingVariable lnTxt -FileName AzStackHci.Network.Strings.psd1 ################################################################################################# # Network Validators # - Test-NwkValidator_StorageAdapterReadiness # - Test-NwkValidator_StorageConnection # - Test-NwkValidator_InfraIpPoolReadiness # - Test-NwkValidator_HostNetworkConfigurationReadiness # - Test-NwkValidator_AdapterDriverMgmtAdapterReadiness # - Test-NwkValidator_MgmtIPIPPoolRequirement # - Test-NwkValidator_MgmtIPForNewNode # - Test-NwkValidator_NetworkIntentStatus # # Note that the validator names are used in AzStackHci.Network.psm1 file to define which validators # to be run in different scenarios. Please make sure to keep the validator names consistent between # the files. # Check the comments there for more information. ################################################################################################# function Test-NwkValidator_StorageAdapterReadiness { [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSession, [PSObject[]] $AtcHostIntents ) try { [System.Management.Automation.Runspaces.PSSession[]] $allNodeSessions = EnsureTestSessionOpen -PSSessions $PSSession # For the AtcHostIntents object array, we only need to check the storage only intents [PSObject[]] $storageIntents = $AtcHostIntents | Where-Object { $_.TrafficType.Contains("Storage") -and (-not $_.TrafficType.contains("Management")) -and (-not $_.TrafficType.contains("Compute")) } [String[]] $allPhysicalAdaptersToCheck = @() if ($storageIntents.Count -eq 1) { $allPhysicalAdaptersToCheck = $storageIntents[0].Adapter Log-Info "Will try to check storage adatpers readiness for [ $($allPhysicalAdaptersToCheck -join ',') ]" } elseif ($storageIntents.Count -gt 1) { # Should not get into here as ATC does not support multiple storage intent, so the input $AtcHostIntents # object should not have multiple storage intent in it, but keep it here as a safe guard throw "More than one storage intent found in the AtcHostIntents object array. Fail the storage adapter configuration validation." } else { Log-Info "No storage only intent found in the AtcHostIntents object array. Skip the storage adapter configuration validation." return } $storageAdapterReadinessResults = @() #region Test storage adatper configuration # Now, check every session and make sure that storage pNIC don't have IP or VLANID foreach ($testSession in $allNodeSessions) { Log-Info "Checking storage adapter IP status on $($testSession.ComputerName)" $adapterReadinessResult = Invoke-Command -Session $testSession -ScriptBlock ${function:CheckStorageAdapterReadiness} -ArgumentList @(, $allPhysicalAdaptersToCheck) Log-Info "Got storage adapter readiness validation results from $($testSession.ComputerName)" $storageAdapterReadinessValidationStatus = if ($adapterReadinessResult.Pass) { 'SUCCESS' } else { 'FAILURE' } $storageAdapterReadinessValidationDetailMessage = $adapterReadinessResult.Message $storageAdapterReadinessRstObject = @{ Name = 'AzStackHci_Network_Test_StorageAdapterReadiness' Title = 'Test storage adapter readiness on node' DisplayName = 'Test storage adapter readiness on node' Severity = 'CRITICAL' Description = 'Test storage adapter readiness on node: no pre-defined IP on it, no VLANID configured on it' Tags = @{} Remediation = "Remove any pre-defined IP on the storage adapter, remove any VLANID configured on the storage adapter" TargetResourceID = "StorageAdapterReadiness" TargetResourceName = "StorageAdapterReadiness" TargetResourceType = "StorageAdapterReadiness" Timestamp = [datetime]::UtcNow Status = $storageAdapterReadinessValidationStatus AdditionalData = @{ Source = $testSession.ComputerName Resource = 'StorageAdapterReadiness' Detail = $storageAdapterReadinessValidationDetailMessage Status = $storageAdapterReadinessValidationStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $storageAdapterReadinessResults += New-AzStackHciResultObject @storageAdapterReadinessRstObject } #endregion return $storageAdapterReadinessResults } catch { throw $_ } } function Test-NwkValidator_StorageConnection { <# .SYNOPSIS Test stamp storage connection for deployment .DESCRIPTION This validator tests the storage connection for the deployment using Test-Cluster or ping MESH. It will only be run if the end user provides one storage intent: If the storage intent is not converged intent, we check pNIC connection only. If the storage intent is converged intent (a.k.a., with management or compute intent combined), we create VMSwitch and storage vNIC for connection testing. The validation will be done with one of below choices: Run “Test-Cluster” with network test With this test, there will be only 1 validation result generated Run ping mesh on each host for storage adapters using APIPA (Automatic Private IP Addressing) (address in subnet 169.254.0.0/16). We only run “ping” test and check if the results have “Reply from <TARGETIP>: bytes=” string in it. With this test, there will be 1 validation result generated for each of the server in the stamp. After the validation is finished, we clean the test artifacts created during the validation (like the VLANID configuration on pNIC, VMSwitch itself, etc.) .PARAMETER PSSession The remote session array to the nodes to be validated .PARAMETER AtcHostIntents The ATC host intent object array, which contains the intent information for the nodes Easiest way to get this is to convert from the "HostNetwork | Intents" object in the unattended JSON file. .PARAMETER StorageAdapterVLANIDInfo The hashtable that contains the physical storage adapter name and VLANID information .PARAMETER SessionCredential The credential to be used for the remote session .EXAMPLE Test-NwkValidator_StorageConnection -PSSession $PSSession -AtcHostIntents $AtcHostIntents -StorageAdapterVLANIDInfo $StorageAdapterVLANIDInfo #> [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSession, [PSObject[]] $AtcHostIntents, [System.Collections.Hashtable] $StorageAdapterVLANIDInfo ) try { Log-Info "Start to run Test-NwkValidator_StorageConnection function" [System.Management.Automation.Runspaces.PSSession[]] $newTestSessionsBeforeChecking = EnsureTestSessionOpen -PSSessions $PSSession #region Define variables used during the validation [PSObject[]] $allStorageAdaptersToCheck = @() [System.Collections.Hashtable] $storageAdapterVlanIdToConfig = @{} [System.Boolean] $needStorageVMSwitch = $false [System.String] $storageVMSwitchName= "" # If we need to create VMSwitch, we will use only 1 pNIC for the VMSwitch [System.String] $adapterForStorageVMSwitch = "" [System.Boolean] $needMgmtOnStorageVMSwitch = $false #endregion #region Prepare variables # Only need to worry about storage intent [PSObject[]] $tmpStorageIntents = $AtcHostIntents | Where-Object { $_.TrafficType.Contains("Storage") } [String[]] $tmpStorageIntentPhysicalAdapters = @() # Prepare $allStorageAdaptersToCheck, $needStorageVMSwitch, $storageVMSwitchName, $adapterForStorageVMSwitch, $needMgmtOnStorageVMSwitch, $storageAdapterVlanIdToConfig if ($tmpStorageIntents.Count -eq 1) { Log-Info "Found storage intent [ $($tmpStorageIntents[0].Name) ]. Need to run storage adapter connection validation." $tmpStorageIntentPhysicalAdapters = $tmpStorageIntents[0].Adapter if ($tmpStorageIntentPhysicalAdapters.Count -eq 0) { # Ideally should not come to here but keep it here as a safe guard Log-Info "No physical adapter found in the storage intent. Fail the storage adapter connection validation." throw "No physical adapter found in the storage intent. Fail the storage adapter connection validation." } Log-Info "Found physical adapter list [ $($tmpStorageIntentPhysicalAdapters -join ",") ] in the intent." # If it is a converged intent, we will need to create VMSwitch/vNIC for validation # if it is storage only intent, we just check the physical storage adapter if ($tmpStorageIntents[0].TrafficType.Contains("Management") -or $tmpStorageIntents[0].TrafficType.Contains("Compute")) { # Converged intent with storage, will need to create vNIC for storage Log-Info "Converged intent. Will need to create VMSwitch/vNIC for storage connection validation." foreach ($tempName in $tmpStorageIntentPhysicalAdapters) { # Pre-define the storage vNIC to be checked $allStorageAdaptersToCheck += "vStorage($($tempName))" $storageAdapterVlanIdToConfig.Add("vStorage($($tempName))", $StorageAdapterVLANIDInfo[$tempName]) } $needStorageVMSwitch = $true $tmpGuid = [System.Guid]::NewGuid() $storageVMSwitchName = "StorageTestVMSwitch$($tmpGuid)" Log-Info "Will need to create VMSwitch with name $($storageVMSwitchName) for storage connection validation." $adapterForStorageVMSwitch = $tmpStorageIntentPhysicalAdapters[-1] if ($tmpStorageIntentPhysicalAdapters.Count -eq 1) { $needMgmtOnStorageVMSwitch = $true Log-Info "Only one adapter in the converged intent. Will need make sure mgmt connection vNIC configured correctly on the VMSwitch." } } else { Log-Info "Storage only intent. Will need to run storage connection validation on pNIC only." $allStorageAdaptersToCheck = $tmpStorageIntentPhysicalAdapters $storageAdapterVlanIdToConfig = $StorageAdapterVLANIDInfo } Log-Info "Storage adapter NAME list used for the validation:" Log-Info "$($allStorageAdaptersToCheck -join ",")" Log-Info "Storage adapter VLANID list used for the validation:" Log-Info "$($storageAdapterVlanIdToConfig | Out-String)" } elseif ($tmpStorageIntents.Count -gt 1) { # Should not get into here as ATC does not support multiple storage intent, so the input $AtcHostIntents # object should not have multiple storage intent in it, but keep it here as a safe guard Log-Info "More than one storage intent found in the AtcHostIntents object array. Fail the storage adapter connection validation." throw "More than one storage intent found in the AtcHostIntents object array. Fail the storage adapter connection validation." } else { Log-Info "No storage intent found in the AtcHostIntents object array. Skip the storage adapter connection validation." return } #endregion # validation results to be returned $storageAdapterConnectionResults = @() [System.Boolean] $needCleanUpStorageVMSwitch = $false if ($needStorageVMSwitch) { # First make sure all the nodes have same configuration Log-Info "Need to create storage VMSwitch for the validation. Make sure same configuration on all host(s)." [System.Boolean] $allNodeSameConfig = $true [System.String] $allNodeVMSwitchStatus = $null [System.String] $existingVMSwitchOnMachine = $null [System.String] $nodesVMSwitchConfigStatusMessage = "Server VMSwitch Configuration Status:" foreach ($testSession in $newTestSessionsBeforeChecking) { Log-Info "Checking VMSwitch configuration on node $($testSession.ComputerName)" $currentNodeConfig = Invoke-Command -Session $testSession -ScriptBlock { param ($AdapterForStorageVMSwitch) $retVal = New-Object PSObject -Property @{ retValVMSwitchStatus = "" existingVMSwitchName = "" } [System.String] $retValVMSwitchStatus = "" [System.String] $existingVMSwitchName = "" $storageadapterGuidOnCurrentNode = (Get-NetAdapter -Physical -Name $AdapterForStorageVMSwitch -ErrorAction SilentlyContinue).InterfaceGuid.Trim('{').Trim('}') try { $vmSwitchTeam = Get-VMSwitchTeam -ErrorAction SilentlyContinue if ($vmSwitchTeam -and ($vmSwitchTeam.NetAdapterInterfaceGuid.Guid -contains $storageadapterGuidOnCurrentNode)) { # Already have a VMSwitch that contains the storage physical adapter, no need to create a new one $retValVMSwitchStatus = "EXIST_VMSWITCH" $existingVMSwitchName = $vmSwitchTeam.Name } else { # Need to create VMSwitch on node for storage connection validation $retValVMSwitchStatus = "NEED_CREATE_VMSWITCH" } } catch { # In case the Hyper-V PowerShell module is not ready, we cannot do anything here # This current validator will fail. Considering current it is only WARNING, so should be fine here. $retValVMSwitchStatus = "HYPER_V_POWERSHELL_NOT_READY" } $retVal.retValVMSwitchStatus = $retValVMSwitchStatus $retVal.existingVMSwitchName = $existingVMSwitchName return $retVal } -ArgumentList $adapterForStorageVMSwitch Log-Info "Current node $($testSession.ComputerName) with VMSwitch status $($currentNodeConfig.retValVMSwitchStatus)" if (-not $allNodeVMSwitchStatus) { $allNodeVMSwitchStatus = $currentNodeConfig.retValVMSwitchStatus $existingVMSwitchOnMachine = $currentNodeConfig.existingVMSwitchName } else { if ($allNodeVMSwitchStatus -ne $currentNodeConfig.retValVMSwitchStatus) { Log-Info "VMSwitch status is different with other nodes. Will fail the validation." $nodesVMSwitchConfigStatusMessage += "`n Server $($testSession.ComputerName): $($currentNodeConfig.retValVMSwitchStatus)" $allNodeSameConfig = $false } } } if ((-not $allNodeSameConfig) -or ($allNodeVMSwitchStatus -notin @("EXIST_VMSWITCH", "NEED_CREATE_VMSWITCH"))) { # In case the node VMSwitch configuration is different between each other, we cannot do anything here # Should not be here, but will fail the validation just in case end user did not configure the system correctly $nodesVMSwitchConfigStatusMessage += "`nCannot check VM switch configuration between servers for storage adapters. Skip the validation." $nodesVMSwitchConfigStatusMessage += "`nMake sure the storage adapter configuration are the same across all nodes." $nodesVMSwitchConfigStatusMessage += "`n If teamed within a VMSwitch, the teaming should be same on all nodes." $nodesVMSwitchConfigStatusMessage += "`n The VMSwitch might be a left over of previous environment validator run. You could clean it up manually." $nodesVMSwitchConfigStatusMessage += "`n For example: Remove-VMSwitch -Name `"StorageTestVMSwitch`" -Force" Log-Info $nodesVMSwitchConfigStatusMessage -Type 'WARNING' $storageAdapterConnectionRstObject = @{ Name = "AzStackHci_Network_Test_StorageConnections_VMSWITCH_CONFIG_DIFFERENT" Title = 'VMSwitch configuration different between nodes for storage adpaters.' DisplayName = 'VMSwitch configuration different between nodes for storage adpaters.' Severity = 'WARNING' Description = 'VMSwitch configuration different between nodes for storage adpaters' Tags = @{} Remediation = 'To validate storage connections, make sure the storage adapter configuration are the same across all nodes. If storage adapters are teamed within a VMSwitch, the teaming should be same on all nodes.' TargetResourceID = 'CannotFindSameStorageAdapterConfiguration' TargetResourceName = 'CannotFindSameStorageAdapterConfiguration' TargetResourceType = 'CannotFindSameStorageAdapterConfiguration' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = '' Resource = 'CannotFindSameStorageAdapterConfiguration' Detail = $nodesVMSwitchConfigStatusMessage Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $storageAdapterConnectionResults += New-AzStackHciResultObject @storageAdapterConnectionRstObject return $storageAdapterConnectionResults } if ($allNodeVMSwitchStatus -eq "EXIST_VMSWITCH") { $needCleanUpStorageVMSwitch = $false # Just need to create new storage vNIC to test the storage connection foreach ($testSession in $newTestSessionsBeforeChecking) { Invoke-Command -Session $testSession -ScriptBlock { New-Item -Path "Function:\CreateStorageVNIC" -Value ${using:function:CreateStorageVNIC} -Force | Out-Null CreateStorageVNIC -StorageVMSwitchName $using:existingVMSwitchOnMachine ` -StorageVNICNames $using:allStorageAdaptersToCheck ` -StorageAdapterVLANIDInfo $using:storageAdapterVlanIdToConfig } } } elseif ($allNodeVMSwitchStatus -eq "NEED_CREATE_VMSWITCH") { $needCleanUpStorageVMSwitch = $true # Need to create VMSwitch on each node for storage connection validation Log-Info "Prepare VMSwitch for storage connection validation on all nodes" Log-Info " VMSwitch name: $($storageVMSwitchName)" Log-Info " Physical adapter to be teamed: $($adapterForStorageVMSwitch)" Log-Info " Storage vNIC will be created: $($allStorageAdaptersToCheck)" Log-Info " Storage VLANID will be used: $($storageAdapterVlanIdToConfig | Out-String)" Log-Info " Need to create management VNIC: $($needMgmtOnStorageVMSwitch)" foreach ($testSession in $newTestSessionsBeforeChecking) { Log-Info "Trying to prepare storage connection validation VMSwitch/vNIC on $($testSession.ComputerName)" if ($needMgmtOnStorageVMSwitch) { Log-Info " Might lost computer connection momentarily..." } Log-Info " Make sure function ConfigureVMSwitchForTesting and CreateStorageVNIC exist in remote session on the machine..." Invoke-Command -Session $testSession -ScriptBlock { New-Item -Path "Function:\ConfigureVMSwitchForTesting" -Value ${using:function:ConfigureVMSwitchForTesting} -Force | Out-Null New-Item -Path "Function:\CreateStorageVNIC" -Value ${using:function:CreateStorageVNIC} -Force | Out-Null } $invokeCommandCreateStorageVMSwitchParam = @{ Session = $testSession ScriptBlock = ${function:ConfigureStorageVMSwitchVNICForTesting} ArgumentList = @($adapterForStorageVMSwitch, $storageVMSwitchName, $allStorageAdaptersToCheck, $storageAdapterVlanIdToConfig, $needMgmtOnStorageVMSwitch) } Log-Info " Calling ConfigureStorageVMSwitchVNICForTesting to create VMSwitch/storage vNIC on the machine..." $vmSwitchCreationInfo = Invoke-Command @invokeCommandCreateStorageVMSwitchParam Log-Info " Done with machine $($testSession.ComputerName)!" } } } else { Log-Info "No need to create VMSwitch for storage connection validation. Will need to set the VLANID on pNIC on all nodes for the validation." foreach ($testSession in $newTestSessionsBeforeChecking) { Log-Info "Trying to set storage adapter VLANID on $($testSession.ComputerName)" Invoke-Command -Session $testSession -ArgumentList $storageAdapterVlanIdToConfig -ScriptBlock { param ([System.Collections.Hashtable] $StorageAdapterVLANIDInfo) foreach ($adapterName in $StorageAdapterVLANIDInfo.Keys) { $vlanId = $StorageAdapterVLANIDInfo[$adapterName] if ($vlanId -ne 0) { $adapter = Get-NetAdapter -Name $adapterName -ErrorAction SilentlyContinue if ($adapter) { Set-NetAdapterAdvancedProperty -Name $adapterName -RegistryKeyword "VLANID" -RegistryValue $vlanId } } } } Log-Info "Done with $($testSession.ComputerName)!" } } # Need to make sure we have the pre-req that needed for the storage connection validation # Feature Failover-Clustering is installed, or storage adapter has IPv4 APIPA address # If converged intent with storage, then hyper-v feature should be there in the system. $usePingMeshOnIPv4 = $true $hostIPv4AddressTable = @{} $checkHostIPv4AddressScript = { [CmdletBinding()] param ([String[]] $StorageAdaptersToCheck) $retVal = New-Object PSObject -Property @{ Pass = $true AdapterIPv4Mapping = @{} } foreach ($storageAdapter in $StorageAdaptersToCheck) { #region Wait for storage adapter to get IPv4 APIPA address [System.Boolean] $currentAdapterIPReady = $false $ipStopWatch = [System.diagnostics.stopwatch]::StartNew() while (-not $currentAdapterIPReady -and ($ipStopWatch.Elapsed.TotalSeconds -lt 60)) { # Using Get-NetIPConfiguration here as Get-NetIPAddress seems has issue while executed for local machine in Invoke-Command [PSObject[]] $adapterIpv4Addresses = (Get-NetIPConfiguration -InterfaceAlias $storageAdapter).IPV4Address ` | Where-Object { $_.AddressState -eq "Preferred" -and $_.PrefixOrigin -eq "WellKnown" -and $_.IPAddress -like "169.254.*" } if ($adapterIpv4Addresses.Count -gt 0) { $currentAdapterIPReady = $true break } else { Start-Sleep -Seconds 3 } } #endregion if ($adapterIpv4Addresses.Count -gt 0) { $retVal.AdapterIPv4Mapping.Add($storageAdapter, $adapterIpv4Addresses[0].IPAddress) } else { $retVal.Pass = $false break } } return $retVal } # In case of VMSwitch creation scenario, the old PSSession might be in state of "Broken", so trying to re-open the session00 [System.Management.Automation.Runspaces.PSSession[]] $newTestSessions = EnsureTestSessionOpen -PSSessions $newTestSessionsBeforeChecking foreach ($testSession in $newTestSessions) { Log-Info "Check if we could use ping mesh by using IPv4 APIPA address on stroage adapter to validate storage connection on $($testSession.ComputerName)" $storageAdapterIPv4ForCurrentNode = Invoke-Command -Session $testSession -ScriptBlock $checkHostIPv4AddressScript -ArgumentList @(, $allStorageAdaptersToCheck) if (-not $storageAdapterIPv4ForCurrentNode.Pass) { Log-Info "Cannot find IPv4 APIPA address on storage adapter on $($testSession.ComputerName). Will not run the validation." $usePingMeshOnIPv4 = $false } else { Log-Info "IPv4 APIPA address found on storage adapter on $($testSession.ComputerName)." $hostIPv4AddressTable.Add($testSession.ComputerName, $storageAdapterIPv4ForCurrentNode.AdapterIPv4Mapping) } } if ($usePingMeshOnIPv4) { $validationFunctionParams = @{ allNodeSessions = $newTestSessions HostIPv4Table = $hostIPv4AddressTable } $storageAdapterConnectionResults = ValidateStorageConnections @validationFunctionParams } else { $tmpMessage = "No valid method available to test storage connection. Skip the validation." Log-Info $tmpMessage -Type 'WARNING' $tmpRemediationMsg = "To validate storage connections, make sure IPv4 APIPA address is available on all storage adapters on all servers." $tmpRemediationMsg += "`nIf you have DHCP enabled on storage adapter, please disable it." $tmpRemediationMsg += "`n Set-NetIPInterface -InterfaceAlias <StorageAdapterAlias> -Dhcp Disabled" $tmpRemediationMsg += "`nIf you configured static IP on storage adapter, please remove it." $tmpRemediationMsg += "`n Remove-NetIPAddress -InterfaceAlias <StorageAdapterAlias>" $storageAdapterConnectionRstObject = @{ Name = "AzStackHci_Network_Test_StorageConnections_NO_VALID_METHOD" Title = 'Cannot find valid validation method to test storage adapter connections' DisplayName = 'Cannot find valid validation method to test storage adapter connections' Severity = 'WARNING' Description = 'Cannot find valid validation method to test storage adapter connections' Tags = @{} Remediation = $tmpRemediationMsg TargetResourceID = 'CannotValidateStorageConnections' TargetResourceName = 'CannotValidateStorageConnections' TargetResourceType = 'CannotValidateStorageConnections' Timestamp = [datetime]::UtcNow Status = 'FAILURE' AdditionalData = @{ Source = '' Resource = 'CannotValidateStorageConnections' Detail = $tmpMessage Status = 'FAILURE' TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $storageAdapterConnectionResults += New-AzStackHciResultObject @storageAdapterConnectionRstObject } return $storageAdapterConnectionResults } catch { throw $_ } finally { # Need to clean the storage vNIC Log-Info "Done with validation run. Now clean up test artifacts created during validation." Log-Info "Try to clean up storage vNIC and/or storage pNIC VLANID" foreach ($testSession in $newTestSessions) { Log-Info "Working to remove storage vNIC and or storage pNIC VLANID on $($testSession.ComputerName)" Invoke-Command -Session $testSession -ScriptBlock { foreach ($adapterName in $using:allStorageAdaptersToCheck) { Remove-VMNetworkAdapter -ManagementOS -Name $adapterName -ErrorAction SilentlyContinue } if (-not $using:needStorageVMSwitch) { foreach ($adapterName in $using:allStorageAdaptersToCheck) { Set-NetAdapterAdvancedProperty -Name $adapterName -RegistryKeyword "VLANID" -RegistryValue 0 -ErrorAction SilentlyContinue } } # Device Management Service might need to be restarted to refresh the nic details # It also might not be there if (Get-Service -Name DeviceManagementService -ErrorAction SilentlyContinue) { Restart-Service -Name DeviceManagementService -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 20 } } Log-Info " Done with $($testSession.ComputerName)!" } if ($needCleanUpStorageVMSwitch) { Log-Info "Need to clean up VMSwitch created during the validation" $mgmtVlanIdToRestore = $vmSwitchCreationInfo.MgmtVlanId foreach ($testSession in $newTestSessions) { Log-Info "Working to remove VMSwitch on $($testSession.ComputerName)..." if ($needMgmtOnStorageVMSwitch) { Log-Info " Might lost computer connection momentarily..." if ( $mgmtVlanIdToRestore -ne 0) { Log-Info " Need to restore management VLANID to: $mgmtVlanIdToRestore" } } Invoke-Command -Session $testSession -ScriptBlock { Remove-VMSwitch -Name $using:storageVMSwitchName -Force if ($using:needMgmtOnStorageVMSwitch) { # Need to restore the mgmt VLANID on the mgmt adapter if ($using:mgmtVlanIdToRestore -ne 0) { Set-NetAdapterAdvancedProperty -Name $using:adapterForStorageVMSwitch -RegistryKeyword "VlanID" -RegistryValue $using:mgmtVlanIdToRestore } #region Wait for the NIC to get the IP address back # In case of DHCP scenario, after VMSwitch removed, the pNIC might not get the IP address immediately # Wait for some time (60 seconds) to make sure the new IP is settled correctly. [System.Boolean] $currentIPReady = $false $ipStopWatch = [System.diagnostics.stopwatch]::StartNew() while (-not $currentIPReady -and ($ipStopWatch.Elapsed.TotalSeconds -lt 60)) { # If the pNIC has Manual or Dhcp IPv4 address with "Preferred" state, we consider it as "ready" $ipConfig = Get-NetIPAddress -InterfaceAlias $using:adapterForStorageVMSwitch -ErrorAction SilentlyContinue | Where-Object { ($_.PrefixOrigin -eq "Manual" -or $_.PrefixOrigin -eq "Dhcp") -and $_.AddressFamily -eq "IPv4" -and $_.AddressState -eq "Preferred" } if ($ipConfig) { $currentIPReady = $true break } else { Start-Sleep -Seconds 3 } } #endregion if (-not $currentIPReady) { # should not get into here, but keep it here for safety $ipInfoAll = Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | Format-Table IPAddress, InterfaceAlias, PrefixLength, PrefixOrigin -AutoSize Write-Host "$($ipInfoAll | Out-String)" Write-Host "Cannot get the IP address back to the pNIC after VMSwitch removed. Please check the system manually." throw "Cannot get the IP address back to the pNIC after VMSwitch removed. Please check the system manually." } else { Write-Host "IP address back to the pNIC after VMSwitch removed. System is ready for connection." } } # Same need to restart Device Management Service to refresh the nic details here if (Get-Service -Name DeviceManagementService -ErrorAction SilentlyContinue) { Restart-Service -Name DeviceManagementService -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 20 } } Log-Info " Done with machine $($testSession.ComputerName)!" } } else { Log-Info "No VMSwitch created during the storage connection validation. Skip clean up." } Log-Info "End of Test-NwkValidator_StorageConnection function" } } function Test-NwkValidator_InfraIpPoolReadiness { [CmdletBinding()] param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools, [Parameter(Mandatory = $false, HelpMessage = "Specify Management Subnet")] [string] $ManagementSubnetValue, [Parameter(Mandatory = $false, HelpMessage = "Specify the region name to target for connectivity validation.")] [string] $RegionName, [int[]] $port = @(5986, 5985, 22), [int] $Timeout = 1000, [int] $Minimum = 6, [int] $Maximum = 255, [PSObject[]] $AtcHostIntents, [System.Boolean] $ProxyEnabled = $false ) try { $instanceResults = @() # Check no repeating ips in pool and all in management subnet Log-Info "Test no repeating ips in all IP pools" $TestMgmtIpPools = TestMgmtIpPools -IpPools $IpPools -ManagementSubnetValue $ManagementSubnetValue $Status = if ($TestMgmtIpPools) { 'SUCCESS' } else { 'FAILURE' } $params = @{ Name = "AzStackHci_Network_Test_IP_Pools_Subnet_No_Duplicates" Title = 'Test IP Pools in Management Subnet and No duplicate IPs in IpPools' DisplayName = "Test IP Pools $ManagementSubnetValue" Severity = 'CRITICAL' Description = 'Checking start and end address are on the same subnet' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = "IpPool-$ManagementSubnetValue" TargetResourceName = "ManagementIPRange" TargetResourceType = 'Network Range' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = 'CustomerNetwork' Resource = 'CustomerSubnet' Detail = if ($TestMgmtIpPools) { $lnTxt.TestIpPoolPass -f $ManagementSubnetValue } else { $lnTxt.TestIpPoolFail -f $ManagementSubnetValue } Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params Log-Info "Test all IP in IP pools are in management subnet, and no IP in IP pools are in use" foreach ($ipPool in $IpPools) { $StartingAddress = $ipPool.StartingAddress $EndingAddress = $ipPool.EndingAddress # Check ip in management subnet $TestMgmtSubnet = (TestIPInSubnet -IpToCheck $StartingAddress -ExpectedSubnetCIDR $ManagementSubnetValue) -and (TestIPInSubnet -IpToCheck $EndingAddress -ExpectedSubnetCIDR $ManagementSubnetValue) $Status = if ($TestMgmtSubnet) { 'SUCCESS' } else { 'FAILURE' } $params = @{ Name = 'AzStackHci_Network_Test_Management_IP_Range_Subnet' Title = 'Test Management IP Subnet' DisplayName = "Test Management IP Subnet $StartingAddress - $EndingAddress" Severity = 'CRITICAL' Description = 'Checking start and end address are on the same subnet' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = "$StartingAddress-$EndingAddress" TargetResourceName = "ManagementIPRange" TargetResourceType = 'Network Range' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = 'CustomerNetwork' Resource = 'CustomerSubnet' Detail = if ($TestMgmtSubnet) { $lnTxt.TestMgmtSubnetPass -f $StartingAddress, $EndingAddress } else { $lnTxt.TestMgmtSubnetFail -f $StartingAddress, $EndingAddress } Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params # Test IP in Range are not in use $MgmtIpRange = GetMgmtIpRange -StartingAddress $StartingAddress -EndingAddress $EndingAddress foreach ($Ip in $MgmtIpRange) { $result = @{} $result += @{ 'Ping' = Test-NetConnection -ComputerName $Ip -InformationLevel Quiet -WarningAction SilentlyContinue } foreach ($p in $port) { $result += @{ $p = IsTcpPortInUse -Ip $ip -Port $p -Timeout $Timeout } } $Status = if ($true -notin $result.Values) { 'SUCCESS' } else { 'FAILURE' } $msg = $lnTxt.ActiveHostCheck -f $ip, (($result.Keys | ForEach-Object { "{0}:{1}" -f $psitem,$result[$psitem] }) -join ', ') $Type = if ($result.Values -contains $true) { 'WARNING' } else { 'INFORMATIONAL' } Log-Info $msg -Type $Type $params = @{ Name = 'AzStackHci_Network_Test_Management_IP_No_Active_Hosts' Title = 'Test Management IP Range for Active Hosts' DisplayName = "Test Management IP Range $Ip for Active Hosts" Severity = 'CRITICAL' Description = 'Checking no hosts respond on Management IP range' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = $Ip TargetResourceName = "ManagementIPRange" TargetResourceType = 'Network Range' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = $Ip Resource = 'ICMP/SSH/WINRM' Detail = ($result.Keys | ForEach-Object { "{0}:{1}" -f $psitem,$result[$psitem] }) -join ', ' Status = $Status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } } # Check range size Log-Info "Test infra pool range size" $TestMgmtRangeSize = TestMgmtRangeSize -IpPools $IpPools -Minimum $Minimum -Maximum $Maximum $status = if ($TestMgmtRangeSize) { 'SUCCESS' } else { 'FAILURE' } $allIps = GetMgmtIpRangeFromPools -IpPools $IpPools $ipCount = $allIps.Count $params = @{ Name = 'AzStackHci_Network_Test_Management_IP_Range_Size' Title = 'Test Management IP Range Size' DisplayName = "Test Management IP Range Size of all the pools. $ipCount ips found." Severity = 'CRITICAL' Description = "Checking management IP range size is between $minimum-$maximum" Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = "Size:$ipCount " TargetResourceName = "ManagementIPRange" TargetResourceType = 'Network Range' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = 'CustomerNetwork' Resource = 'CustomerRange' Detail = if ($TestMgmtRangeSize) { $lnTxt.TestMgmtRangeSizePass -f $Minimum, $Maximum } else { $lnTxt.TestMgmtRangeSizeFail -f $Minimum, $Maximum } Status = if ($TestMgmtRangeSize) { 'SUCCESS' } else { 'FAILURE' } TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params # Check pool sizes Log-Info "Test infra pool size" $TestMgmtRangePoolCount = TestMgmtRangePoolCount -IpPools $IpPools -Minimum $Minimum $poolCount = $IpPools.Count $status = if ($TestMgmtRangePoolCount) { 'SUCCESS' } else { 'FAILURE' } $params = @{ Name = 'AzStackHci_Network_Test_Management_IP_Range_Pool_Count' Title = 'Test Management IP Pool Count' DisplayName = "Test Management IP Range Number of IP Pools." Severity = 'CRITICAL' Description = "Checking management IP pools has one or two pools. First pool must only have 1 ip if 2 pools" Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = "Count:$poolCount " TargetResourceName = "ManagementIPRange" TargetResourceType = 'Network Range' Timestamp = [datetime]::UtcNow Status = $Status AdditionalData = @{ Source = 'CustomerNetwork' Resource = 'CustomerRange' Detail = if ($TestMgmtRangePoolCount) { $lnTxt.TestMgmtRangePoolCountPass -f $poolCount} else { $lnTxt.TestMgmtRangePoolCountFail -f $poolCount, ($Minimum - 1) } Status = if ($TestMgmtRangePoolCount) { 'SUCCESS' } else { 'FAILURE' } TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params #region check infra IP conneciton # A new vSwitch and management vNIC is created using Network ATC naming standards. The vSwitch is created using the intents configuration provided by the customer in the deployment json. # We will rotate the vNIC IP with the infra IPs to be tested. # Whatever is the Mgmt intent, we will create the vSwitch using the selected pNICs from the customer. # curl tool seems to provide the solution to test from specific source IP and allows to check TCP ports and also URLs # Only the first 9 IPs from the infra range will be tested. # DNS registration must be avoided on the IPs tested from the vNIC Log-Info "Test infra IP connections" $infraIPRangeToValidate = GetMgmtIpRangeFromPools -IpPools $IpPools if ($ProxyEnabled) { Log-Info "Proxy is enabled on the host. Will check public endpoint connection via proxy." } else { Log-Info "Proxy is not enabled on the host. Will check public endpoint connetion directly." } if ((Get-Command Get-VMSwitch -ErrorAction SilentlyContinue) -and (Get-WindowsFeature -Name Hyper-V -ErrorAction SilentlyContinue).Installed) { [PSObject[]] $mgmtIntent = $AtcHostIntents | Where-Object { $_.TrafficType.Contains("Management") } [System.String] $mgmtIntentName = $mgmtIntent[0].Name [System.String[]] $mgmtAdapters = GetSortedMgmtIntentAdapter -MgmtAdapterNames $mgmtIntent[0].Adapter [System.Guid[]] $intentAdapterGuids = (Get-NetAdapter -Name $mgmtAdapters -ErrorAction SilentlyContinue).InterfaceGuid try { $needCleanUpVMSwitch = $false $mgmtVlanIdToRestore = 0 #region prepare VMSwitch for testing infra IP connection [PSObject[]] $allExistingVMSwitches = Get-VMSwitch -SwitchType External $externalVMSwitchsCount = $allExistingVMSwitches.Count [PSObject] $foundVMSwitchToUse = $null if ($externalVMSwitchsCount -eq 0) { # if we found 0 VMSwitch, we will need to create one for this testing # Note that this operation will have the host disconnected from network for a moment (due to VMSwitch/vNIC creation) # Since the code is executed locally, this disconnection should not affect the execution Log-Info "No VMSwitch exists in system. Will create VMSwitch for testing infra IP connection." $tmpVMSwitchConfigInfo = ConfigureVMSwitchForTesting -SwitchAdapterNames $mgmtAdapters -MgmtIntentName $mgmtIntentName $foundVMSwitchToUse = $tmpVMSwitchConfigInfo.VMSwitchInfo $needCleanUpVMSwitch = $tmpVMSwitchConfigInfo.NeedCleanUp $mgmtVlanIdToRestore = $tmpVMSwitchConfigInfo.MgmtVlanId if (-not $tmpVMSwitchConfigInfo.IPReady) { Log-Info "Cannot get a VMSwitch ready on $($env:COMPUTERNAME) with valid IP on the vNIC created. Fail the validation" throw "Cannot get a VMSwitch ready on $($env:COMPUTERNAME) with valid IP on the vNIC created. Fail the validation" } } else { # if we found at least 1 VMSwitch in the system, we then need to check # If there is one VMSwitch that has the same mgmt intent adapters Log-info "Found $($externalVMSwitchsCount) VMSwitch in the system. Need to check if a valid one could be used for validation." foreach ($externalVMSwitch in $allExistingVMSwitches) { # Need to check the switch is good for deployment: using same adapter as the intent [System.Guid[]] $switchAdapterGuids = $externalVMSwitch.NetAdapterInterfaceGuid if (Compare-Object -ReferenceObject $switchAdapterGuids -DifferenceObject $intentAdapterGuids) { # Adapters used in pre-defined VMSwitch and the intent are different. Ignore that VMSwitch Log-Info "Found $($externalVMSwitch.Name) with different adapters than the mgmt intent. Skip it." } else { # if the system already have a VMSwitch with the same mgmt adpaters in its teaming, we will just use that adpater $foundVMSwitchToUse = $externalVMSwitch break } } if (-not $foundVMSwitchToUse) { Log-info "No valid VMSwitch found! Check if we could create a new VMSwitch for validaton." # At this moment, we need further checking: # If all adapters in the mgmt intent is not used by any adapter, we will need to create a new VMSwitch # If any of the adapter in the mgmt intent is used by any adapter, we will need to error out as this is not a supported scenario [System.Guid[]] $allSwitchAdapterGuids = $allExistingVMSwitches.NetAdapterInterfaceGuid [System.Boolean] $intentAdapterAlreadyUsed = $false foreach ($tmpAdapterGuid in $intentAdapterGuids) { if ($allSwitchAdapterGuids.Contains($tmpAdapterGuid)) { $intentAdapterAlreadyUsed = $true break } } if (-not $intentAdapterAlreadyUsed) { # if none of the adapter in the mgmt intent is used by any adapter, we will create a new VMSwitch Log-Info "VMSwitch found, but no VMSwitch mgmt adapters in the system. Will create VMSwitch for testing infra IP connection." $tmpVMSwitchConfigInfo = ConfigureVMSwitchForTesting -SwitchAdapterNames $mgmtAdapters -MgmtIntentName $mgmtIntentName $foundVMSwitchToUse = $tmpVMSwitchConfigInfo.VMSwitchInfo $needCleanUpVMSwitch = $tmpVMSwitchConfigInfo.NeedCleanUp $mgmtVlanIdToRestore = $tmpVMSwitchConfigInfo.MgmtVlanId if (-not $tmpVMSwitchConfigInfo.IPReady) { Log-Info "Cannot get a VMSwitch ready on $($env:COMPUTERNAME) with valid IP on the vNIC created. Fail the validation" throw "Cannot get a VMSwitch ready on $($env:COMPUTERNAME) with valid IP on the vNIC created. Fail the validation" } } else { # This is an error situation: some of the mgmt intent adapters is already used by an existing VMSwitch, some other mgmt # intent adapter is still "free" in the system. We don't know what to do with this, so need to error out Log-Info "VMSwitch found, mgmt adapter list is not matching to any VMSwitch adapter list. Wrong configuration. Will to fail the validation" } } } #endregion if ($RegionName -eq "AzureLocal") { $allPublicEndpointServicesToCheck = Get-AzstackHciConnectivityTarget -LocalOnly:$true -RegionName $RegionName | Where-Object { $_.Name -Like "Azure_Kubernetes_Service_*" -or $_.Name -Like "AzStackHci_MOCStack_*" -or $_.Name -Like "Vm_Management_HCI_*" } } else { $allPublicEndpointServicesToCheck = Get-AzstackHciConnectivityTarget | Where-Object { $_.Name -Like "Azure_Kubernetes_Service_*" -or $_.Name -Like "AzStackHci_MOCStack_*" -or $_.Name -Like "Vm_Management_HCI_*" } } if ($foundVMSwitchToUse) { Log-Info "Got VMSwitch $($foundVMSwitchToUse.Name) to use for validating infra IP connection. Start the validation..." $mgmtAlias = "vManagement($($MgmtIntentName))" [PSOBject[]] $mgmtAdapterIP = Get-NetIPAddress -InterfaceAlias $mgmtAlias -ErrorAction SilentlyContinue | Where-Object { ($_.PrefixOrigin -eq "Manual" -or $_.PrefixOrigin -eq "Dhcp") -and $_.AddressFamily -eq "IPv4" -and $_.AddressState -eq "Preferred" } if ($mgmtAdapterIP -and $mgmtAdapterIP.Count -gt 0) { $prefixLength = $mgmtAdapterIP[0].PrefixLength[0] $mgmtIPConfig = Get-NetIPConfiguration -InterfaceAlias $mgmtAlias $defaultGateway = $mgmtIPconfig.IPv4DefaultGateway[0].NextHop # Try to get DNS server IP from running system [PSObject[]] $getDNSServers = Get-DnsClientServerAddress -InterfaceAlias $mgmtAlias -AddressFamily IPv4 -ErrorAction SilentlyContinue if ($getDNSServers -and ($getDNSServers.Count -gt 0) -and ($getDNSServers[0].ServerAddresses.Count -gt 0)) { # Get up to 3 DNS servers to check [System.String[]] $dnsServersIPToCheck = $getDNSServers[0].ServerAddresses[0..2] try { #region configure vNIC for testing infra IP connection $tmpGuid = [System.Guid]::NewGuid() $newVNICName = "TestvNIC$($tmpGuid)" Log-Info "Prepare vNIC $($newVNICName) on the VMSwitch to use for infra IP connection validation" if (Get-VMNetworkAdapter -All | where-Object { $_.Name -eq $newVNICName }) { Remove-VMNetworkAdapter -ManagementOS -SwitchName $foundVMSwitchToUse.Name -Name $newVNICName -Confirm:$false } Add-VMNetworkAdapter -ManagementOS -SwitchName $foundVMSwitchToUse.Name -Name $newVNICName Get-NetAdapter -name "vEthernet ($($newVNICName))" -ErrorAction SilentlyContinue | Rename-NetAdapter -NewName $newVNICName Set-DnsClient -InterfaceAlias $newVNICName -RegisterThisConnectionsAddress $false # There is a possibility that the mgmt VMNetworkAdapter is not there # (NOTE: we have the validation in Test-NwkValidator_AdapterDriverMgmtAdapterReadiness, however if the result failed there, the current test will still run. # So there is a chance that below call will throw exception out if we do not specify -ErrorAction SilentlyContinue) $tempVMNetworkAdapterIsolation = Get-VMNetworkAdapterIsolation -ManagementOS -VMNetworkAdapterName $mgmtAlias -ErrorAction SilentlyContinue if ($tempVMNetworkAdapterIsolation -and $tempVMNetworkAdapterIsolation.DefaultIsolationID -ne 0) { Set-VMNetworkAdapterIsolation -ManagementOS ` -VMNetworkAdapterName $newVNICName ` -IsolationMode Vlan ` -AllowUntaggedTraffic $true ` -DefaultIsolationID $tempVMNetworkAdapterIsolation.DefaultIsolationID } Set-NetIPInterface -InterfaceAlias $newVNICName -Dhcp Disabled # Need to wait until the DHCP is disabled on the adapter. Otherwise, following call might fail [System.Boolean] $vNicReady = $false $stopWatch = [System.diagnostics.stopwatch]::StartNew() while (-not $vNicReady -and ($stopWatch.Elapsed.TotalSeconds -lt 30)) { if ((Get-VMNetworkAdapter -ManagementOS -Name $newVNICName -ErrorAction SilentlyContinue) -and ((Get-NetIPInterface -InterfaceAlias $newVNICName -AddressFamily IPv4).Dhcp -eq "Disabled")) { $vNicReady = $true break } else { Start-Sleep -Seconds 3 } } #endregion if ($vNICReady) { Log-Info "VMNetworkAdapter [ $($newVNICName) ] ready and DHCP is disabled on the adapter." $retryTimes = 10 Log-Info "Will check connection from Infra IP to DNS server(s) port 53 and public endpoints for max of $($retryTimes) times" Log-Info "DNS Server to check: $($dnsServersIPToCheck | Out-String)" ################################### # Start testing infra IP connection ################################### # Magic number: we will test only first 9 IPs from the infra range as: # 6 are the one we requested right now for services running in HCI cluster # 3 are the additional that might be used in the future (for example, SLB VM, etc.) # We don't want to test all the infra IP as it will requires a lot of time to finish the validation $ipNumberToCheck = 9 if ($infraIPRangeToValidate.Count -lt $ipNumberToCheck) { $ipNumberToCheck = $infraIPRangeToValidate.Count } for ($i=0; $i -lt $ipNumberToCheck; $i++) { $endpointIndex = 1 $ipToCheck = $infraIPRangeToValidate[$i] Log-Info "`nCheck IP $($i+1) / $($ipNumberToCheck): [ $($ipToCheck) ]" #region Set new IP on the adapter # Make sure no IP on the adapter $oldIpAddresses = Get-NetIPAddress -InterfaceAlias $newVNICName -ErrorAction SilentlyContinue foreach ($ip in $oldIpAddresses) { Remove-NetIPAddress -InterfaceAlias $newVNICName -IPAddress $ip.IPAddress -Confirm:$false -ErrorAction SilentlyContinue } if (Get-NetRoute -InterfaceAlias $newVNICName -DestinationPrefix 0.0.0.0/0 -ErrorAction SilentlyContinue) { New-NetIPAddress -InterfaceAlias $newVNICName -IPAddress $ipToCheck -PrefixLength $prefixLength -SkipAsSource $true | Out-Null } else { New-NetIPAddress -InterfaceAlias $newVNICName -IPAddress $ipToCheck -PrefixLength $prefixLength -DefaultGateway $defaultGateway -SkipAsSource $true | Out-Null } #region Wait for the new IP to be ready [System.Boolean] $currentIPReady = $false $ipStopWatch = [System.diagnostics.stopwatch]::StartNew() while (-not $currentIPReady -and ($ipStopWatch.Elapsed.TotalSeconds -lt 60)) { $ipConfig = Get-NetIPAddress -InterfaceAlias $newVNICName -ErrorAction SilentlyContinue | Where-Object { $_.IPAddress -eq $ipToCheck -and $_.PrefixOrigin -eq "Manual" -and $_.AddressFamily -eq "IPv4" -and $_.AddressState -eq "Preferred" } if ($ipConfig) { $currentIPReady = $true break } else { Start-Sleep -Seconds 3 } } #endregion #endregion #region Check connection from infra IP to DNS server # Note that we cannot use Resolve-DnsName or nslookup here directly, as those call cannot specify the source IP foreach ($currentDNSServerToCheck in $dnsServersIPToCheck) { Log-Info " >> Trying DNS connection to $($currentDNSServerToCheck) port 53." [System.Boolean] $isDnsConnected = $false $retry = 1 while ((-not $isDnsConnected) -and ($retry -le $retryTimes)) { try { $src = [System.Net.IPEndPoint]::new([ipaddress]::Parse($ipToCheck),0) $tc = [System.Net.Sockets.TcpClient]::new($src) $tc.Connect($currentDNSServerToCheck, 53) if ($tc.Connected) { Log-Info " == DNS connection ESTABLISHED to $($currentDNSServerToCheck) on attempt $($retry)" $isDnsConnected = $true break } else { Log-Info " ?? FAILED DNS connection to $($currentDNSServerToCheck) on attempt $($retry)" } } catch { Log-Info " ?? FAILED! Got exception while checking DNS connection on attempt ($($retry))!" } finally { if ($tc) { $tc.Dispose() } } Start-Sleep -Seconds 3 $retry++ } if ($isDnsConnected) { Log-Info " == Found valid DNS connection" break } } #endregion $dnsConnectionRstParams = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_DNS_Server_Port_53' Title = 'Test DNS server port connection for all IP in infra IP pool' DisplayName = "Test DNS server port connection for all IP in infra IP pool" Severity = 'CRITICAL' Description = 'Test DNS server port connection for all IP in infra IP pool' Tags = @{} Remediation = "Make sure infra IP $ipToCheck could connect to your DNS server correctly." TargetResourceID = "Infra_IP_Connection_$($ipToCheck)" TargetResourceName = "Infra_IP_Connection_$($ipToCheck)" TargetResourceType = "Infra_IP_Connection_$($ipToCheck)" Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = $($ipToCheck) Detail = "[FAILED] Connection from $ipToCheck to DNS server port 53 failed after 3 attempts. DNS server used: $($dnsServersIPToCheck | Out-String)" Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } if ($ProxyEnabled) { # In case proxy is enabled, we will downgrade the severity to WARNING as the DNS resolution might happen on proxy server $dnsConnectionRstParams.Severity = 'WARNING' } if ($isDnsConnected) { $dnsConnectionRstParams.Status = "SUCCESS" $dnsConnectionRstParams.AdditionalData.Detail = "[PASSED] Connection from $ipToCheck to DNS server port 53 passed. DNS server used: $($dnsServersIPToCheck | Out-String)" $dnsConnectionRstParams.AdditionalData.Status = "SUCCESS" #region Check connection from infra IP to well known endpoints # Since we rely on DNS naming resolution, we put the checking here in this if statement foreach ($service in $allPublicEndpointServicesToCheck) { foreach ($endpointInService in $service.EndPoint) { if (($endpointInService -match "(:[\d]+)$") -and $ProxyEnabled) { Log-Info " >> Need to skip checking connection to $($endpointInService) in a proxy enabled environment as proxy might not allow HTTP/HTTPS query to non-standard port via curl.exe." continue } # Will try $retryTimes curl connection to the remote endpoint # Note that curl.exe honor the system HTTP_PROXY/HTTPS_PROXY settings, so we don't need to specify "--proxy" parameter here if ($RegionName -eq "AzureLocal") { $endpointToCheck = "$($service.Protocol[0])://$($endpointInService)" $curlGetExpression = "curl.exe -sS --connect-timeout 5 -m 6 --ssl-revoke-best-effort `"$($endpointToCheck)`" --interface $($ipToCheck)" $curlHeaderExpression = "curl.exe -i -sS --connect-timeout 5 -m 6 --ssl-revoke-best-effort `"$($endpointToCheck)`" --interface $($ipToCheck)" } else { $endpointToCheck = "$($service.Protocol[0])://$($endpointInService)" $curlGetExpression = "curl.exe -sS --connect-timeout 5 -m 6 `"$($endpointToCheck)`" --interface $($ipToCheck)" $curlHeaderExpression = "curl.exe -i -sS --connect-timeout 5 -m 6 `"$($endpointToCheck)`" --interface $($ipToCheck)" } # If Arcgateway is enabled, then we need to ensure the endpoint is reachable via the Arc proxy. # Detailed Explanation: The curl command uses the proxy settings from the HTTP_PROXY/HTTPS_PROXY environment variables, # which is pointing to Arc proxy on Arc gateway enabled system. Thus, Without the --interface parameter it will just make sure # the endpoint is reachable via the Arc proxy. $azcmagent = "$env:ProgramW6432\AzureConnectedMachineAgent\azcmagent.exe" if (Test-Path $azcmagent) { $arcConnectionType = & $azcmagent config get connection.type if ( $arcConnectionType -eq "gateway") { # Just check if Arc proxy is able to route this traffic $curlGetExpression = "curl.exe -sS --connect-timeout 5 -m 6 `"$($endpointToCheck)`" " $curlHeaderExpression = "curl.exe -i -sS --connect-timeout 5 -m 6 `"$($endpointToCheck)`" " } } Log-Info " >> Trying public Endpoint connection $($endpointIndex) to $($endpointToCheck)." [System.Boolean] $isPublicEndpointConnected = $false $retry = 1 while ((-not $isPublicEndpointConnected) -and ($retry -le $retryTimes)) { try { Log-Info " == Run $($curlGetExpression)" $curlGetContent = Invoke-Expression $curlGetExpression if (-not [System.String]::IsNullOrWhiteSpace($curlGetContent)) { Log-Info " == Connection ESTABLISHED with GET on attempt $($retry)" $isPublicEndpointConnected = $true break } else { Log-Info " == Run $($curlHeaderExpression)" $curlHeaderContent = Invoke-Expression $curlHeaderExpression # Need to analyze the output of $curlHeaderContent to see if the connection is established # If proxy enabled, the response will need to contain something in addition to the "HTTP/1.1 200 Connection established" if ($ProxyEnabled) { $curlHeaderContent = $curlHeaderContent -replace "^HTTP\/\d\.\d 200 Connection established", "" } if (-not [System.String]::IsNullOrWhiteSpace($curlHeaderContent)) { Log-Info " == Connection ESTABLISHED with HEADER only on attempt $($retry)" $isPublicEndpointConnected = $true break } else { Log-Info " ?? FAILED connection on attempt $($retry)" } } } catch { Log-Info " ?? FAILED! Got exception while checking public endpoint connection on attempt ($($retry))!" Log-Info " ?? FAILED! $($_)!" } Start-Sleep -Seconds 3 $retry++ } if ([System.String]::IsNullOrEmpty($service.Severity) -or [System.String]::IsNullOrWhiteSpace($service.Severity)) { $currentSeverity = "CRITICAL" } else { $currentSeverity = $service.Severity } $publicEndpointRstParams = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_' + $service.Name Title = 'Test outbound connection for IP in infra IP pool' DisplayName = "Test outbound connection for IP in infra IP pool" + $service.Title Severity = $currentSeverity Description = 'Test connection for IP in infra IP pool ' + $service.Description Tags = @{} Remediation = "Make sure infra IP $ipToCheck could connect to public endpoint $endpointToCheck correctly. `nhttps://learn.microsoft.com/en-us/azure/azure-arc/servers/network-requirements?tabs=azure-cloud#urls" TargetResourceID = $service.TargetResourceID TargetResourceName = $service.TargetResourceName TargetResourceType = $service.TargetResourceType Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = $($ipToCheck) Detail = "[FAILED] Connection from $ipToCheck to $endpointToCheck failed after $retryTimes attempts" Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } if ($isPublicEndpointConnected) { $publicEndpointRstParams.Status = "SUCCESS" $publicEndpointRstParams.AdditionalData.Detail = "[PASSED] Connection from $ipToCheck to $endpointToCheck passed." $publicEndpointRstParams.AdditionalData.Status = "SUCCESS" } else { Log-info "Public Endpoint connection failed for infra IP $ipToCheck." } $instanceResults += New-AzStackHciResultObject @publicEndpointRstParams $endpointIndex++ } } #endregion } else { Log-info "DNS connection failed for infra IP $ipToCheck." } $instanceResults += New-AzStackHciResultObject @dnsConnectionRstParams } } else { # vNIC creation failure. Normally won't hit this path, but keep it here for safety Log-Info "Cannot get a vNIC ready on VMSwitch $($foundVMSwitchToUse.Name) in $($env:COMPUTERNAME) for validating infra IP connection. Fail the validation" $params = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_vNIC_Readiness' Title = 'Test virtual adapter readiness for all IP in infra IP pool' DisplayName = "Test virtual adapter readiness for all IP in infra IP pool" Severity = 'CRITICAL' Description = 'Test virtual adapter readiness for all IP in infra IP pool' Tags = @{} Remediation = "Make sure Add/Get-VMNetworkAdapter on $($env:COMPUTERNAME) can run correctly." TargetResourceID = "Infra_IP_Connection_VNICReadiness" TargetResourceName = "Infra_IP_Connection_VNICReadiness" TargetResourceType = "Infra_IP_Connection_VNICReadiness" Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = 'VNICReadiness' Detail = "[FAILED] Cannot test connection for infra IP. VM network adapter is not configured correctly on host $($env:COMPUTERNAME)." Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } } finally { # Best effort to clean the IP used, as the last IP checked might not be cleaned in the previous checking for ($i=0; $i -lt $ipNumberToCheck; $i++) { Remove-NetIPAddress -IPAddress $infraIPRangeToValidate[$i] -ErrorAction SilentlyContinue -Confirm:$false } # Clean up the vNIC if (Get-VMNetworkAdapter -ManagementOS -Name $newVNICName -ErrorAction SilentlyContinue) { Remove-VMNetworkAdapter -ManagementOS -SwitchName $foundVMSwitchToUse.Name -Name $newVNICName -Confirm:$false } } } else { # No DNS client server address found on the adapter Log-Info "Cannot get DNS client server address correctly on $($env:COMPUTERNAME) for validating infra IP connection. Fail the validation" $params = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_DNSClientServerAddress_Readiness' Title = 'Test DNS client server addresses readiness for all IP in infra IP pool' DisplayName = "Test DNS client server addresses readiness for all IP in infra IP pool" Severity = 'CRITICAL' Description = 'Test DNS client server addresses readiness for all IP in infra IP pool' Tags = @{} Remediation = "Set DNS client server address correctly on management adapter [ $($mgmtAlias) ] on $($env:COMPUTERNAME). Check it using Get-DnsClientServerAddress" TargetResourceID = "Infra_IP_Connection_DNSClientReadiness" TargetResourceName = "Infra_IP_Connection_DNSClientReadiness" TargetResourceType = "Infra_IP_Connection_DNSClientReadiness" Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = 'DNSClientReadiness' Detail = "[FAILED] Cannot find correctly DNS client server address on host $($env:COMPUTERNAME)." Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } } else { Log-Info "Got VMSwitch, but cannot get a valid vNIC to use on $($env:COMPUTERNAME) for validating infra IP connection. Fail the validation" $params = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_MANAGEMENT_VNIC_Readiness' Title = 'Test VMSwitch/Management VMNetworkAdapter readiness for all IP in infra IP pool' DisplayName = 'Test VMSwitch/Management VMNetworkAdapter readiness for all IP in infra IP pool' Severity = 'CRITICAL' Description = 'Test VMSwitch/Management VMNetworkAdapter readiness for all IP in infra IP pool' Tags = @{} Remediation = "Make sure at least one management VMNetworkAdapter with name $($mgmtAlias) configured correctly on the host $($env:COMPUTERNAME)." TargetResourceID = "Infra_IP_Connection_ManagementVMNetworkAdapterReadiness" TargetResourceName = "Infra_IP_Connection_ManagementVMNetworkAdapterReadiness" TargetResourceType = 'Infra_IP_Connection_ManagementVMNetworkAdapterReadiness' Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = 'ManagementVMNetworkAdapterReadiness' Detail = "[FAILED] Cannot test connection for infra IP with wrong management VMNetworkAdapter configured on host $($env:COMPUTERNAME)." Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } } else { Log-Info "Cannot get a VMSwitch to use on $($env:COMPUTERNAME) for validating infra IP connection. Fail the validation" $params = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_VMSwitch_Readiness' Title = 'Test VMSwitch readiness for all IP in infra IP pool' DisplayName = "Test VMSwitch readiness for all IP in infra IP pool" Severity = 'CRITICAL' Description = 'Test VMSwitch readiness for all IP in infra IP pool' Tags = @{} Remediation = "Make sure at least one VMSwitch preconfigured on the host $($env:COMPUTERNAME) has the same set of adapters defined in management intent." TargetResourceID = "Infra_IP_Connection_VMSwitchReadiness" TargetResourceName = "Infra_IP_Connection_VMSwitchReadiness" TargetResourceType = 'Infra_IP_Connection_VMSwitchReadiness' Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = 'VMSwitchReadiness' Detail = "[FAILED] Cannot test connection for infra IP with wrong VMSwitch configured on host $($env:COMPUTERNAME)." Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } } finally { if ($needCleanUpVMSwitch) { # Clean up the VMSwitch created for testing Log-Info "Clean up VMSwitch $($foundVMSwitchToUse.Name) created during the testing" Remove-VMSwitch -Name $foundVMSwitchToUse.Name -Force -ErrorAction SilentlyContinue if ($mgmtVlanIdToRestore -ne 0) { foreach ($tmpAdapter in $mgmtAdapters) { Log-Info "Restore VlanId for adapter $tmpAdapter to $mgmtVlanIdToRestore" Set-NetAdapterAdvancedProperty -Name $tmpAdapter -RegistryKeyword "VlanID" -RegistryValue $mgmtVlanIdToRestore } } #region Wait for the IP address back to the pNIC # In case of DHCP scenario, after VMSwitch removed, the pNIC might not get the IP address immediately # Wait for some time (60 seconds) to make sure the new IP is settled correctly. [System.Boolean] $currentIPReady = $false $ipStopWatch = [System.diagnostics.stopwatch]::StartNew() while (-not $currentIPReady -and ($ipStopWatch.Elapsed.TotalSeconds -lt 60)) { # If the pNIC has Manual or Dhcp IPv4 address with "Preferred" state, we consider it as "ready" $ipConfig = Get-NetIPAddress -InterfaceAlias $mgmtAdapters[0] -ErrorAction SilentlyContinue | Where-Object { ($_.PrefixOrigin -eq "Manual" -or $_.PrefixOrigin -eq "Dhcp") -and $_.AddressFamily -eq "IPv4" -and $_.AddressState -eq "Preferred" } if ($ipConfig) { $currentIPReady = $true break } else { Start-Sleep -Seconds 3 } } #endregion if (-not $currentIPReady) { # should not get into here, but keep it here for safety $ipInfoAll = Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | Format-Table IPAddress, InterfaceAlias, PrefixLength, PrefixOrigin -AutoSize Log-Info "$($ipInfoAll | Out-String)" Log-Info "Cannot get the IP address back to the pNIC after VMSwitch removed. Please check the system manually." throw "Cannot get the IP address back to the pNIC after VMSwitch removed. Please check the system manually." } else { Log-Info "IP address back to the pNIC after VMSwitch removed. System is ready for next validation." } } else { Log-Info "VMSwitch $($foundVMSwitchToUse.Name) pre-exist in the system. No need to clean up." } } } else { Log-Info "Hyper-V is not working correctly on $($env:COMPUTERNAME). Fail testing infra IP connection." $params = @{ Name = 'AzStackHci_Network_Test_Infra_IP_Connection_Hyper_V_Readiness' Title = 'Test Hyper-V readiness for all IP in infra IP pool' DisplayName = "Test Hyper-V readiness for all IP in infra IP pool" Severity = 'CRITICAL' Description = 'Test Hyper-V readiness for all IP in infra IP pool' Tags = @{} Remediation = "Make sure that Hyper-V is installed on host $($env:COMPUTERNAME) and rerun the validation." TargetResourceID = "Infra_IP_Connection_HYPERVReadiness" TargetResourceName = "Infra_IP_Connection_HYPERVReadiness" TargetResourceType = 'Infra_IP_Connection_HYPERVReadiness' Timestamp = [datetime]::UtcNow Status = "FAILURE" AdditionalData = @{ Source = $env:COMPUTERNAME Resource = 'HYPERVReadiness' Detail = "[FAILED] Cannot test connection for infra IP without Hyper-V on host $($env:COMPUTERNAME)." Status = "FAILURE" TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } #endregion return $instanceResults } catch { throw $_ } finally { # Device Management Service might need to be restarted to refresh the nic details # It also might not be there if (Get-Service -Name DeviceManagementService -ErrorAction SilentlyContinue) { Restart-Service -Name DeviceManagementService -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 20 Log-Info "Restarted the Device Management Service successfully and waited 20 seconds which should refresh the nic details" } } } function Test-NwkValidator_HostNetworkConfigurationReadiness { [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSession, [PSObject[]] $AtcHostIntents, [ValidateSet('Small','Medium','Large')] [System.String] $HardwareClass = "Medium" ) try { if (($PSSession.Count -eq 0) -or ($AtcHostIntents.Count -eq 0)) { Log-Info "No PSSession or AtcHostIntents provided. Skip run of Test-NwkValidator_HostNetworkConfigurationReadiness" return } else { Log-Info "Will check host network adapter RDMA status, adapter symmetry and bandwidth, and other host network" Log-Info "configuration (include DNS client configuraion, Hyper-V is running correctly, VMSwitch (if exists)" Log-Info "has mgmt intent adapters, VlanId for adapters, physical adapter used in JSON." } [System.Management.Automation.Runspaces.PSSession[]] $allPSSessions = EnsureTestSessionOpen -PSSessions $PSSession switch ($HardwareClass) { "Small" { $expectedAdapterMinBandWidth = 1000000000 } # 1Gbps "Medium" { $expectedAdapterMinBandWidth = 10000000000 } # 10Gbps "Large" { $expectedAdapterMinBandWidth = 10000000000 } # 10Gbps Default { $expectedAdapterMinBandWidth = 10000000000 } # 10Gbps } # Check host network readiness status $hostNetworkReadinessTestResults = @() foreach ($session in $allPSSessions) { #region Check RDMA status Log-Info "Checking NetAdapter RDMA status on $($session.ComputerName)" $rdmaResult = Invoke-Command -Session $session -ScriptBlock ${function:CheckNetAdapterRDMAStatus} -ArgumentList @(, $AtcHostIntents) if ($null -ne $rdmaResult) { Log-Info "Got RDMA validation results from $($session.ComputerName)" $currentMachineRdmaStatus = if ($rdmaResult.Pass) { 'SUCCESS' } else { 'FAILURE' } $currentMachineRdmaTestDetailMessage = $rdmaResult.Message Log-Info " Result: $($currentMachineRdmaTestDetailMessage)" } else { Log-Info "NO RDMA validation results found from $($session.ComputerName)" $currentMachineRdmaStatus = 'FAILURE' $currentMachineRdmaTestDetailMessage = "NO RDMA validation results returned by function CheckNetAdapterRDMAStatus from server $($session.ComputerName)" } $rdmaRstObject = @{ Name = 'AzStackHci_Network_Test_NetAdapter_RDMA_Operational' Title = 'Test NetAdapter RDMA requirement' DisplayName = "Test if RDMA requirement meets for the deployment on all servers" Severity = 'CRITICAL' Description = 'Checking RDMA Operational Status on {0}' -f $session.ComputerName Tags = @{} Remediation = 'Make sure adapter RDMA is operational. Use Get-NetAdapterRdma cmdlet to check the status of RDMA for the network adapter in the system.' TargetResourceID = $session.ComputerName TargetResourceName = "NetAdapter" TargetResourceType = 'Network Adapter RDMA' Timestamp = [datetime]::UtcNow Status = $currentMachineRdmaStatus AdditionalData = @{ Source = $session.ComputerName Resource = 'Network Adapter RDMA Operational Status' Detail = $currentMachineRdmaTestDetailMessage Status = $currentMachineRdmaStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $hostNetworkReadinessTestResults += New-AzStackHciResultObject @rdmaRstObject #endregion #region Check adapter symmetry and bandwidth requirement Log-Info "Checking NetAdapter symmetry bandwidth requirement on $($session.ComputerName)" $adapterSymmetryAndBandwidthResult = Invoke-Command -Session $session -ScriptBlock ${function:CheckAdapterSymmetryAndBandwidth} -ArgumentList $AtcHostIntents, $expectedAdapterMinBandWidth if ($null -ne $adapterSymmetryAndBandwidthResult) { Log-Info "Got adapter symmetry and bandwidth validation results from $($session.ComputerName)" $currentMachineAdapterSymmetryBandwidthStatus = if ($adapterSymmetryAndBandwidthResult.Pass) { 'SUCCESS' } else { 'FAILURE' } $currentMachineAdapterSymmetryBandwidthTestDetailMessage = $adapterSymmetryAndBandwidthResult.Message } else { Log-Info "NO adapter symmetry and bandwidth validation results found from $($session.ComputerName)" $currentMachineAdapterSymmetryBandwidthStatus = 'FAILURE' $currentMachineAdapterSymmetryBandwidthTestDetailMessage = "NO adapter symmetry and bandwidth validation results returned by function CheckAdapterSymmetryAndBandwidth from server $($session.ComputerName)" } $adapterSymmetryRstObject = @{ Name = 'AzStackHci_Network_Test_NetAdapter_Symmetry_Bandwidth' Title = 'Test NetAdapter symmetry and bandwidth requirement' DisplayName = "Test if network adapters used in one intent is symmetry and if bandwidth meets minimum requirement" Severity = 'CRITICAL' Description = 'Checking network adapters and bandwidth Status on {0}' -f $session.ComputerName Tags = @{} Remediation = 'Make sure adapters used in intent are symmetry and mininum bandwidth to use for RDMA is 10G. Use Get-NetAdapter cmdlet on the system to check the adapter information.' TargetResourceID = $session.ComputerName TargetResourceName = "NetAdapter" TargetResourceType = 'Network Adapter Symmetry and Bandwidth' Timestamp = [datetime]::UtcNow Status = $currentMachineAdapterSymmetryBandwidthStatus AdditionalData = @{ Source = $session.ComputerName Resource = 'Network Adapter Symmetry and Bandwidth' Detail = $currentMachineAdapterSymmetryBandwidthTestDetailMessage Status = $currentMachineAdapterSymmetryBandwidthStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $hostNetworkReadinessTestResults += New-AzStackHciResultObject @adapterSymmetryRstObject #endregion #region Host network configuration readiness Log-Info "Checking host network readiness configuration on $($session.ComputerName)" $networkReadinessResult = Invoke-Command -Session $session -ScriptBlock ${function:CheckHostNetworkConfigurationReadiness} -ArgumentList @(, $AtcHostIntents) if ($null -ne $networkReadinessResult) { Log-Info "Network readiness check results from $($session.ComputerName)" $currentMachineNetworkReadinessStatus = if ($networkReadinessResult.Pass) { 'SUCCESS' } else { 'FAILURE' } $currentMachineNetworkReadinessTestDetailMessage = $networkReadinessResult.Message } else { Log-Info "NO host network configuration readiness validation results found from $($session.ComputerName)" $currentMachineNetworkReadinessStatus = 'FAILURE' $currentMachineNetworkReadinessTestDetailMessage = "NO host network configuration readiness validation results returned by function CheckHostNetworkConfigurationReadiness from $($session.ComputerName)" } $networkReadinessRstObject = @{ Name = 'AzStackHci_Network_Test_HostNetworkConfigurationReadiness' Title = 'Test host network configuration readiness' DisplayName = "Test if host network requirement meets for the deployment on all servers" Severity = 'CRITICAL' Description = 'Checking host network configuration readiness status on {0}' -f $session.ComputerName Tags = @{} Remediation = 'Make sure host network configuration readiness is correct. Review detail message to find out the issue.' TargetResourceID = $session.ComputerName TargetResourceName = "HostNetworkReadiness" TargetResourceType = 'HostNetworkReadiness' Timestamp = [datetime]::UtcNow Status = $currentMachineNetworkReadinessStatus AdditionalData = @{ Source = $session.ComputerName Resource = 'HostNetworkReadiness configuration status' Detail = $currentMachineNetworkReadinessTestDetailMessage Status = $currentMachineNetworkReadinessStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $hostNetworkReadinessTestResults += New-AzStackHciResultObject @networkReadinessRstObject #endregion } return $hostNetworkReadinessTestResults } catch { throw $_ } } function Test-NwkValidator_AdapterDriverMgmtAdapterReadiness { [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSession, [PSObject[]] $AtcHostIntents, [System.Boolean] $DhcpEnabled ) try { [System.Management.Automation.Runspaces.PSSession[]] $allPSSessions = EnsureTestSessionOpen -PSSessions $PSSession $inboxDriverMgmtIPTestResults = @() $checkInboxDriverScript = { [CmdletBinding()] param ( [String[]] $AdapterNames ) $retVal = New-Object PSObject -Property @{ Pass = $true Message = "Adapter inbox driver check on $($ENV:COMPUTERNAME)" } [PSObject[]] $allAdaptersInfo = Get-NetAdapter -Name $AdapterNames -ErrorAction SilentlyContinue if (($allAdaptersInfo.Count -eq 0) -or ($allAdaptersInfo.Count -ne $AdapterNames.Count)) { $retVal.Pass = $false $retVal.Message += "`nFailed: Adapter(s) not found on the system" $retVal.Message += "`nExpected adapter(s): $($AdapterNames | Out-String)" $retVal.Message += "`nFound adapter(s): $($allAdaptersInfo | Out-String)" } else { $adaptersUsingInboxDriver = $allAdaptersInfo | Where-Object { $_.DriverProvider -match "Microsoft" -or $_.DriverProvider -match "Windows" } $hardwareType = (Get-WmiObject -Class Win32_ComputerSystem).Model # Check adatper is not inbox driver for physical environment only if (-not $adaptersUsingInboxDriver -or ($hardwareType -eq "Virtual Machine")) { $retVal.Message += "`nPassed: No adapter using inbox driver found, or current system is a virtual environment." $retVal.Message += "`nAdapter(s) checked: $($AdapterNames)" } else { $retVal.Pass = $false $retVal.Message += "`nFailed: Adapter(s) using inbox driver found on non-virtual environment`n" $retVal.Message += ($adaptersUsingInboxDriver.Name | Out-String) } } return $retVal } $checkMgmtAdapterScript = { [CmdletBinding()] param ( [String[]] $MgmtAdapterNames, [String] $MgmtIntentName, [System.Boolean] $DhcpEnabled ) $retVal = New-Object PSObject -Property @{ Pass = $true Message = "Management adapter IP and DNS check on $($ENV:COMPUTERNAME)" } $mgmtVNicName = "vManagement($($MgmtIntentName))" [PSObject[]] $allExistingVMSwitches = @() try { $allExistingVMSwitches = Get-VMSwitch -SwitchType External -ErrorAction SilentlyContinue } catch { } [System.Boolean] $expectedVMSwitchReadyInSystem = $false if ($allExistingVMSwitches.Count -gt 0) { # VMSwitch should contains 0 or all of the mgmt adapters # if VMSwitch contains all mgmt adapters, then there should be 1 vNIC named as "vManagement(mgmtintentname)"" foreach ($externalVMSwitch in $allExistingVMSwitches) { # Need to check the switch is good for deployment: using same adapter as the intent [System.Guid[]] $switchAdapterGuids = $externalVMSwitch.NetAdapterInterfaceGuid [System.Guid[]] $intentAdapterGuids = (Get-NetAdapter -Name $MgmtAdapterNames -Physical -ErrorAction SilentlyContinue).InterfaceGuid if (Compare-Object -ReferenceObject $switchAdapterGuids -DifferenceObject $intentAdapterGuids) { # Adapters used in pre-defined VMSwitch and the intent are different. Need to make sure 0 mgmt adapter used by VMSwitch foreach ($mgmtAdapter in $intentAdapterGuids) { if ($switchAdapterGuids -contains $mgmtAdapter) { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Adapter with GUID $($mgmtAdapter) defined in management intent is used by VMSwitch $($externalVMSwitch.Name). Please make sure VMSwitch use ALL adapters defined in the management intent." } } } else { $retVal.Message += "`n Passed: Found one VMSwitch [ $($externalVMSwitch.Name) ] in the system that uses all physical adapters defined in the management intent." $expectedVMSwitchReadyInSystem = $true # VMSwitch uses same set of adapters defined in mgmt intent, will need to check there is a vNIC named as "vManagement(mgmtintentname)" [PSObject[]] $expectedVMNetworkAdapterMgmtNIC = Get-VMNetworkAdapter -ManagementOS -Name $mgmtVNicName -ErrorAction SilentlyContinue if ($expectedVMNetworkAdapterMgmtNIC.Count -ne 1) { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Expected 1 VMNetworkAdapter with name [ $($mgmtVNicName) ] in the system. But found $($expectedVMNetworkAdapterMgmtNIC.Count). Please check by running Get-VMNetWorkAdapter." } else { $retVal.Message += "`n Passed: Found one VMNetworkAdapter named as $mgmtVNicName in the system" } [PSObject[]] $expectedNetAdapterMgmtNIC = Get-NetAdapter -Name $mgmtVNicName -ErrorAction SilentlyContinue if ($expectedNetAdapterMgmtNIC.Count -ne 1) { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Expected 1 NetAdapter with name [ $($mgmtVNicName) ] in the system. But found $($expectedNetAdapterMgmtNIC.Count). Please check by running Get-NetAdapter." } else { $retVal.Message += "`n Passed: Found one NetAdapter named as $mgmtVNicName in the system" } } } } [String[]] $adaptersToCheck = @() if ($expectedVMSwitchReadyInSystem) { $adaptersToCheck = @($mgmtVNicName) } else { $adaptersToCheck = $MgmtAdapterNames } # Following checks only be performed on the 1st adapter in the list as other adapters might not have valid IP address configured on it during the test $firstMgmtAdapter = $adaptersToCheck[0] # Adapter IP checking if ($DhcpEnabled) { $prefixOriginExpected = "Dhcp" $prefixOriginNotExpected = "Manual" $wrongIpTypeMsg = "Expecting DHCP IP on mgmt adapters, but found static IP." } else { $prefixOriginExpected = "Manual" $prefixOriginNotExpected = "Dhcp" $wrongIpTypeMsg = "Expecting static IP on mgmt adapters, but found DHCP IP." } [PSObject[]] $currentAdapterAddresses = Get-NetIPAddress -InterfaceAlias $firstMgmtAdapter -AddressFamily IPv4 -PolicyStore ActiveStore -ErrorAction SilentlyContinue | Where-Object { ($_.PrefixOrigin -eq $prefixOriginExpected) -and ($_.AddressState -eq "Preferred") } if ($currentAdapterAddresses.Count -ne 1) { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Expected one and only one valid IP address on management adapter $($firstMgmtAdapter). Found [ $($currentAdapterAddresses.Count) ]." $retVal.Message += "`n IP addresses found: $($currentAdapterAddresses.IPAddress | Out-String)" $retVal.Message += "`n Run `"Get-NetIPAddress -InterfaceAlias $($firstMgmtAdapter)`" and check the IP address." } else { $retVal.Message += "`n Passed: Got one and only one valid IP address on management adapter $($firstMgmtAdapter)" $retVal.Message += "`n IP address found: $($currentAdapterAddresses.IPAddress | Out-String)" } # For all mgmt adapter, if DHCP enabled, we should not have static IP address configured on it, otherwise, cluster creation will have problem foreach ($mgmtAdatper in $adaptersToCheck) { [PSObject[]] $currentAdapterAddresses = Get-NetIPAddress -InterfaceAlias $mgmtAdatper -AddressFamily IPv4 -PolicyStore ActiveStore -ErrorAction SilentlyContinue | Where-Object { ($_.PrefixOrigin -eq $prefixOriginNotExpected) -and ($_.AddressState -eq "Preferred") } if ($currentAdapterAddresses.Count -gt 0) { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Found invalid IP address type on adapter $($mgmtAdatper). $($wrongIpTypeMsg)" $retVal.Message += "`n IP addresses found: $($currentAdapterAddresses.IPAddress | Out-String)" $retVal.Message += "`n Run `"Get-NetIPAddress -InterfaceAlias $($mgmtAdatper)`" and check the IP address configuration." } else { $retVal.Message += "`n Passed: No invalid IP address type found on adapter $($mgmtAdatper)" } } # Adapter default gateway checking [PSObject[]] $currentAdapterNetIPConfiguration = Get-NetIPConfiguration -InterfaceAlias $firstMgmtAdapter -ErrorAction SilentlyContinue if ($currentAdapterNetIPConfiguration) { [PSObject[]] $allDefaultGateway = $currentAdapterNetIPConfiguration[0].IPv4DefaultGateway if ($allDefaultGateway.Count -ne 1) { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Cannot find correctly number of default gateway for management adapter $($FirstMgmtPNICName) or $($firstMgmtAdapter)" $retVal.Message += "`n Expect [ 1 ], got [ $($allDefaultGateway.Count) ] default gateway(s)." $retVal.Message += "`n Run `"Get-NetIPConfiguration -InterfaceAlias $($firstMgmtAdapter)`" and check property `"IPv4DefaultGateway`"." } else { $retVal.Message += "`n Found correct default gateway information for management adapter $($firstMgmtAdapter)" } } else { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Cannot find valid IP configuration for management adapter $($firstMgmtAdapter)" $retVal.Message += "`n Run `"Get-NetIPConfiguration -InterfaceAlias $($firstMgmtAdapter)`" to check the IP configuration." } # Adapter DNS server checking. [PSObject[]] $mgmtAdapterDNSClientServerAddresses = Get-DnsClientServerAddress -InterfaceAlias $firstMgmtAdapter -AddressFamily IPv4 -ErrorAction SilentlyContinue if ($mgmtAdapterDNSClientServerAddresses -and ($mgmtAdapterDNSClientServerAddresses.Count -gt 0) -and ($mgmtAdapterDNSClientServerAddresses[0].ServerAddresses.Count -gt 0)) { $retVal.Message += "`n Passed: DNS client server addresses found on management adapter $($firstMgmtAdapter)" $retVal.Message += "`n IP address found: $($mgmtAdapterDNSClientServerAddresses.ServerAddresses | Out-String)" } else { $retVal.Pass = $false $retVal.Message += "`n !!! FAILED: Cannot find any DNS client server addresses on management adapter $($firstMgmtAdapter)" $retVal.Message += "`n Run `"Get-DnsClientServerAddress -InterfaceAlias $firstMgmtAdapter -AddressFamily IPv4`" to check DNS client server address configuration." } return $retVal } Log-Info "DHCP enabled environment? [ $($DhcpEnabled) ]" foreach ($sessionToCheck in $allPSSessions) { #region TEST1: Check inbox driver for all storage adapters in the host Log-Info "Check storage intent adapter inbox driver on $($sessionToCheck.ComputerName)" [System.String[]] $storageAdapters = $AtcHostIntents | Where-Object { $_.TrafficType.Contains("Storage") } | Select-Object -ExpandProperty Adapter if ($storageAdapters.Count -gt 0) { $tmpInboxDriverCheckRst = Invoke-Command -Session $sessionToCheck -ScriptBlock $checkInboxDriverScript -ArgumentList @(, $storageAdapters) if ($null -ne $tmpInboxDriverCheckRst) { Log-Info "Got inbox driver validation results from $($sessionToCheck.ComputerName)" $currentMachineInboxDriverTestStatus = if ($tmpInboxDriverCheckRst.Pass) { 'SUCCESS' } else { 'FAILURE' } $currentMachineInboxDriverTestDetailMessage = $tmpInboxDriverCheckRst.Message } else { # Should not come here, just a safe guard Log-Info "NO inbox driver validation results found from $($sessionToCheck.ComputerName)" $currentMachineInboxDriverTestStatus = 'FAILURE' $currentMachineInboxDriverTestDetailMessage = "NO inbox driver validation results returned from server $($session.ComputerName)" } $inboxDriverCheckRstObject = @{ Name = 'AzStackHci_Network_Test_AdapterDriver' Title = 'Test system adapter driver provider' DisplayName = 'Test system adapter driver provider' Severity = 'CRITICAL' Description = 'Checking adapter driver on {0}' -f $sessionToCheck.ComputerName Tags = @{} Remediation = 'https://learn.microsoft.com/en-us/azure-stack/hci/concepts/host-network-requirements#driver-requirements' TargetResourceID = $sessionToCheck.ComputerName TargetResourceName = "AdapterDriver" TargetResourceType = 'AdapterDriver' Timestamp = [datetime]::UtcNow Status = $currentMachineInboxDriverTestStatus AdditionalData = @{ Source = $sessionToCheck.ComputerName Resource = 'AdapterDriver' Detail = $currentMachineInboxDriverTestDetailMessage Status = $currentMachineInboxDriverTestStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $inboxDriverMgmtIPTestResults += New-AzStackHciResultObject @inboxDriverCheckRstObject } else { Log-Info "No storage adapter found in intent definition for host $($sessionToCheck.ComputerName). Skip inbox driver check for storage intent adapter." } #endregion #region TEST2: Check no more than 1 IP address and DNS client server address on mgmt adapter, IP type is correct Log-Info "Check on mgmt adapter readiness on $($sessionToCheck.ComputerName)" [PSObject[]] $mgmtIntent = $AtcHostIntents | Where-Object { $_.TrafficType.Contains("Management") } [System.String] $mgmtIntentName = $mgmtIntent[0].Name [System.String[]] $mgmtAdapters = GetSortedMgmtIntentAdapter -MgmtAdapterNames $mgmtIntent[0].Adapter $tmpMgmtAdapterIPCheckRst = Invoke-Command -Session $sessionToCheck -ScriptBlock $checkMgmtAdapterScript -ArgumentList @($mgmtAdapters, $mgmtIntentName, $DhcpEnabled) if ($null -ne $tmpMgmtAdapterIPCheckRst) { Log-Info "Got mgmt adapter IP validation results from $($sessionToCheck.ComputerName)" $currentMachineMgmtIPTestStatus = if ($tmpMgmtAdapterIPCheckRst.Pass) { 'SUCCESS' } else { 'FAILURE' } $currentMachineMgmtIPTestDetailMessage = $tmpMgmtAdapterIPCheckRst.Message } else { # Should not come here, just a safe guard Log-Info "NO mgmt adapter IP validation results found from $($sessionToCheck.ComputerName)" $currentMachineMgmtIPTestStatus = 'FAILURE' $currentMachineMgmtIPTestDetailMessage = "NO mgmt adapter IP validation results returned from server $($session.ComputerName)" } $mgmtAdapterCheckRstObject = @{ Name = 'AzStackHci_Network_Test_MgmtAdapterReadiness' Title = 'Test system management adapter readiness' DisplayName = 'Test system management adapter readiness' Severity = 'CRITICAL' Description = 'Checking there is one valid management adapter on {0}: make sure it has only one valid IP assigned on it on and DNS server/Gateway are set correctly on it' -f $sessionToCheck.ComputerName Tags = @{} Remediation = 'Make sure there is one valid outbound management adapter on host and have only one valid IP with correct type (DHCP or static) assigned to it and DNS server/Gateway are set correctly on it as well.' TargetResourceID = $sessionToCheck.ComputerName TargetResourceName = "MgmtAdapterIP" TargetResourceType = 'MgmtAdapterIP' Timestamp = [datetime]::UtcNow Status = $currentMachineMgmtIPTestStatus AdditionalData = @{ Source = $sessionToCheck.ComputerName Resource = 'MgmtAdapterIP' Detail = $currentMachineMgmtIPTestDetailMessage Status = $currentMachineMgmtIPTestStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $inboxDriverMgmtIPTestResults += New-AzStackHciResultObject @mgmtAdapterCheckRstObject #endregion } return $inboxDriverMgmtIPTestResults } catch { throw $_ } } function Test-NwkValidator_MgmtIPIPPoolRequirement { <# .SYNOPSIS Run during both Deployment and AddNode 1. Mgmt NIC IP should not be overlapping with IP Pool 2. Ensure Mgmt NIC IPs and IP Pool are in the same subnet #> [CmdletBinding()] param ( [Parameter(Mandatory = $true, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools, [Parameter(Mandatory = $false, HelpMessage = "Specify Management Subnet")] [string] $ManagementSubnetValue, [System.Management.Automation.Runspaces.PSSession[]] $PSSession ) try { [System.Management.Automation.Runspaces.PSSession[]] $allPSSessions = EnsureTestSessionOpen -PSSessions $PSSession $instanceResults = @() foreach ($ipPool in $IpPools) { $StartingAddress = $ipPool.StartingAddress $EndingAddress = $ipPool.EndingAddress foreach ($currentSession in $allPSSessions) { $sb = { $env:COMPUTERNAME ( Get-NetIPConfiguration | Where-Object { $null -ne $_.IPv4DefaultGateway -and $_.NetAdapter.Status -eq "Up" } ).IPv4Address.IPAddress } $NewNodeData = Invoke-Command $currentSession -ScriptBlock $sb $NodeName = $NewNodeData[0] # Check for all of the IPs found on the Host, note that array $NewNodeData is having IPAddress info from item with index 1 for ($i = 1; $i -lt $NewNodeData.count; $i++) { $NodeManagementIPAddress = $NewNodeData[$i] Log-Info "Node Name retrieved from session: $NodeName" Log-Info "Node Management IP Address retrieved from session: $NodeManagementIPAddress" #region Check node management IP is not in infra pool range Log-Info "Start testing Mgmt IP on $($currentSession.ComputerName) is not in Infra IP Pool..." $ip = [system.net.ipaddress]::Parse($NodeManagementIPAddress).GetAddressBytes() [array]::Reverse($ip) $ip = [system.BitConverter]::ToUInt32($ip, 0) $from = [system.net.ipaddress]::Parse($StartingAddress).GetAddressBytes() [array]::Reverse($from) $from = [system.BitConverter]::ToUInt32($from, 0) $to = [system.net.ipaddress]::Parse($EndingAddress).GetAddressBytes() [array]::Reverse($to) $to = [system.BitConverter]::ToUInt32($to, 0) $mgmtIPOutsideRange = ($ip -le $from) -or ($ip -ge $to) if ($mgmtIPOutsideRange) { $TestMgmtIPInfraRangeDetail = $lnTxt.TestMgmtIPInfraRangePass -f $NodeManagementIPAddress, $StartingAddress, $EndingAddress } else { $TestMgmtIPInfraRangeDetail = $lnTxt.TestMgmtIPInfraRangeFail -f $NodeManagementIPAddress, $StartingAddress, $EndingAddress Log-Info $TestMgmtIPInfraRangeDetail -Type Warning } $status = if ($mgmtIPOutsideRange) { 'SUCCESS' } else { 'FAILURE' } $params = @{ Name = 'AzStackHci_Network_Test_Validity_MgmtIp_NotIn_Infra_Pool' Title = 'Test Validity Management IP not in Infra Pool' DisplayName = "Test Validity Management IP not in Infra Pool" Severity = 'CRITICAL' Description = 'Checking management IPs are not in infra IP pool' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = "$StartingAddress-$EndingAddress" TargetResourceName = "ManagementIPIPPoolConfiguration" TargetResourceType = 'ManagementIPIPPoolConfiguration' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = $currentSession.ComputerName Resource = 'NodeManagementIP' Detail = $TestMgmtIPInfraRangeDetail Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params #endregion #region Check node management IP is within the same subnet as the IP pools Log-Info "Start testing Mgmt IP on $($currentSession.ComputerName) is within the same subnet as that of Infra IP Pool..." $TestMgmtSubnet = TestIPInSubnet -IpToCheck $NodeManagementIPAddress -ExpectedSubnetCIDR $ManagementSubnetValue $status = if ($TestMgmtSubnet) { 'SUCCESS' } else { 'FAILURE' } $params = @{ Name = 'AzStackHci_Network_Test_Validity_MgmtIp_In_Infra_Subnet' Title = 'Test Validity Management IP in same infra subnet as IP pools' DisplayName = "Test Validity Management IP in same infra subnet as IP pools" Severity = 'CRITICAL' Description = 'Checking management IPs are in same subnet as infra IP pool' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = "$StartingAddress-$EndingAddress" TargetResourceName = "ManagementIPIPPoolConfiguration" TargetResourceType = 'ManagementIPIPPoolConfiguration' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = "$($currentSession.ComputerName)AndCustomerNetwork" Resource = 'NodeManagementIPAndCustomerSubnet' Detail = if ($TestMgmtSubnet) { $lnTxt.TestMgmtSubnetPass -f $NodeManagementIPAddress, $EndingAddress } else { $lnTxt.TestMgmtSubnetFail -f $NodeManagementIPAddress, $EndingAddress } Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params #endregion } } } return $instanceResults } catch { throw $_ } } # Initial tests to determine if Mgmt IP of new Node is OK # Below Tests are for Static IP Allocation (Non-DHCP) function Test-NwkValidator_MgmtIPForNewNode { [CmdletBinding()] param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools, [System.Management.Automation.Runspaces.PSSession[]] $PSSession, [Hashtable] $NodeToManagementIPMap, [PSObject[]] $AtcHostIntents ) try { [System.Management.Automation.Runspaces.PSSession[]] $allPSSessions = EnsureTestSessionOpen -PSSessions $PSSession $instanceResults = @() $newNodeSession = $allPSSessions[0] [PSObject[]] $mgmtIntent = $AtcHostIntents | Where-Object { $_.TrafficType.Contains("Management") } $intentName = $mgmtIntent[0].Name $firstAdapterName = $mgmtIntent[0].Adapter[0] $sb = { $env:COMPUTERNAME ( Get-NetIPConfiguration | Where-Object { $null -ne $_.IPv4DefaultGateway -and $_.NetAdapter.Status -eq "Up" } ).IPv4Address.IPAddress } $NewNodeData = Invoke-Command $newNodeSession -ScriptBlock $sb $NodeName = $NewNodeData[0] $NodeManagementIPAddress = $NewNodeData[1] Log-Info "Node Name retrieved from PSSession: $NodeName" Log-Info "Node Management IP Address retrieved from PSSession: $NodeManagementIPAddress" foreach ($ipPool in $IpPools) { $StartingAddress = $ipPool.StartingAddress $EndingAddress = $ipPool.EndingAddress # Check node management IP is not in infra pool range Log-Info "Starting Test Mgmt IP is not in Infra IP Pool for $($newNodeSession.ComputerName)" $ip = [system.net.ipaddress]::Parse($NodeManagementIPAddress).GetAddressBytes() [array]::Reverse($ip) $ip = [system.BitConverter]::ToUInt32($ip, 0) $from = [system.net.ipaddress]::Parse($StartingAddress).GetAddressBytes() [array]::Reverse($from) $from = [system.BitConverter]::ToUInt32($from, 0) $to = [system.net.ipaddress]::Parse($EndingAddress).GetAddressBytes() [array]::Reverse($to) $to = [system.BitConverter]::ToUInt32($to, 0) $mgmtIPOutsideRange = ($ip -le $from) -or ($ip -ge $to) if ($mgmtIPOutsideRange) { $TestMgmtIPInfraRangeDetail = $lnTxt.TestMgmtIPInfraRangePass -f $NodeManagementIPAddress, $StartingAddress, $EndingAddress $status = 'SUCCESS' } else { $TestMgmtIPInfraRangeDetail = $lnTxt.TestMgmtIPInfraRangeFail -f $NodeManagementIPAddress, $StartingAddress, $EndingAddress Log-Info $TestMgmtIPInfraRangeDetail -Type Warning $status = 'FAILURE' } $params = @{ Name = 'AzStackHci_Network_Test_New_Node_Validity_Outside_Mgmt_Range' Title = 'Test New Node Configuration Outside Management Range' DisplayName = "Test New Node Configuration Outside Management Range" Severity = 'CRITICAL' Description = 'Checking New Node IP' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = $NodeManagementIPAddress TargetResourceName = "IPAddress" TargetResourceType = 'IPAddress' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = $NodeName Resource = 'NewNodeManagementIP' Detail = $TestMgmtIPInfraRangeDetail Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } # Check that no management IPs are the same (Mgmt IP shouldn't conflict with existing node) Log-Info "Starting Test for No Mgmt IPs are the same for any Nodes" $duplicateIPs = $false $numDuplicates = $NodeToManagementIPMap.GetEnumerator() | Group-Object Value | Where-Object { $_.Count -gt 1 } if ($null -ne $numDuplicates) { $duplicateIPs = $true Log-Info 'Duplicate IPs found for Node Management IPs' -Type Warning } if ($duplicateIPs) { $dtl = 'Duplicate IPs found for Node Management IPs' $status = 'FAILURE' } else { $dtl = 'No Duplicate IPs found for Node Management IPs' $status = 'SUCCESS' } $params = @{ Name = 'AzStackHci_Network_Test_New_Node_Validity_Duplicate_IP' Title = 'Test New Node Configuration Duplicate IP' DisplayName = "Test New Node Configuration Duplicate IP" Severity = 'CRITICAL' Description = 'Checking New Node IP is not a duplicate' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = $NodeManagementIPAddress TargetResourceName = "IPAddress" TargetResourceType = 'IPAddress' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = 'NodeAndManagementIPMapping' Resource = 'NodeManagementIPs' Detail = $dtl Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params # Check that host name exists, and the name and mgmt IP both match current node Log-Info "Starting Test to check if Mgmt IP is on a different node as $NodeName" Log-Info "Starting simultaneous Test to check if HostName and Mgmt IP Match for $NodeName" $ipOnAnotherNode = $false $NodeNameAndIPMatches = $false $nodeNameForIP = $null foreach ($NodeIP in $NodeToManagementIPMap.GetEnumerator()) { Write-Host "$($NodeIP.Name): $($NodeIP.Value)" if ($NodeIP.Name -eq $NodeName) { if ($NodeIP.Value -eq $NodeManagementIPAddress) { $NodeNameAndIPMatches = $true $nodeNameForIP = $NodeIP.Name } } else { if ($NodeIP.Value -eq $NodeManagementIPAddress) { $ipOnAnotherNode = $true $nodeNameForIP = $NodeIP.Name } } } if ($ipOnAnotherNode) { $CheckMgmtIPNotOnOtherNodeDetail = $lnTxt.CheckMgmtIPNotOnOtherNodeFail -f $NodeManagementIPAddress, $nodeNameForIP Log-Info $CheckMgmtIPNotOnOtherNodeDetail -Type Warning } else { $CheckMgmtIPNotOnOtherNodeDetail = $lnTxt.CheckMgmtIPNotOnOtherNodePass -f $NodeManagementIPAddress, $nodeNameForIP } $status = if ($ipOnAnotherNode) { 'FAILURE' } else { 'SUCCESS' } $params = @{ Name = 'AzStackHci_Network_Test_New_Node_Validity_IP_Conflict' Title = 'Test New Node Configuration Conflicting IP' DisplayName = "Test New Node Configuration Conflicting IP" Severity = 'CRITICAL' Description = 'Checking New Node IP is not on another node' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = $NodeManagementIPAddress TargetResourceName = "IPAddress" TargetResourceType = 'IPAddress' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = 'NodeAndManagementIPMapping' Resource = 'NodeNameAndManagementIP' Detail = $CheckMgmtIPNotOnOtherNodeDetail Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params if ($NodeNameAndIPMatches) { $CheckMgmtIPOnNewNodeDetail = $lnTxt.CheckMgmtIPOnNewNodePass -f $NodeManagementIPAddress, $nodeNameForIP $status = 'SUCCESS' } else { $CheckMgmtIPOnNewNodeDetail = $lnTxt.CheckMgmtIPOnNewNodeFail -f $NodeManagementIPAddress, $nodeNameForIP Log-Info $CheckMgmtIPOnNewNodeDetail -Type Warning $status = 'FAILURE' } $params = @{ Name = 'AzStackHci_Network_Test_New_Node_And_IP_Match' Title = 'Test New Node Configuration Name and IP Match' DisplayName = "Test New Node Configuration Name and IP Match" Severity = 'CRITICAL' Description = 'Checking New Node Name and IP match' Tags = @{} Remediation = 'https://aka.ms/hci-envch' TargetResourceID = $NodeManagementIPAddress TargetResourceName = "IPAddress" TargetResourceType = 'IPAddress' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = 'NodeAndManagementIPMapping' Resource = 'NewNodeNameAndManagementIP' Detail = $CheckMgmtIPOnNewNodeDetail Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params # Check that New Node has the first physical adapter and the physical adapter has the mgmt IP Log-Info "Starting Test to see if $firstAdapterName on $NodeName has the correct Mgmt IP" $adapterSB = { param($adapterName) $returnDict = @{} $returnDict["GetNetIPAddressOutput"] = Get-NetIPAddress -ErrorAction SilentlyContinue $returnDict["GetNetAdapterOutput"] = Get-NetAdapter $AdapterIPObject = Get-NetIPAddress -InterfaceAlias $adapterName -AddressFamily IPv4 -ErrorAction SilentlyContinue if ($null -eq $AdapterIPObject) { $returnDict["Result"] = $false $returnDict["AdapterName"] = $adapterName return $returnDict } $returnDict["Result"] = $true $returnDict["AdapterName"] = $adapterName $returnDict["AdapterIP"] = $AdapterIPObject.IPAddress return $returnDict } $AdapterContainsMgmtIP = $false $physicalAdapterExists = $false $VirtualNICName = "vManagement($intentName)" try { $NewNodeAdapterData = Invoke-Command $newNodeSession -ScriptBlock $adapterSB -ArgumentList $firstAdapterName Log-Info "Data found for New Node Adapter ($firstAdapterName): $($NewNodeAdapterData | Out-String)" if ($NewNodeAdapterData['Result'] -eq $false) { Log-Info "Physical Adapter Not Found" Log-Info "Get-NetIPAddress output: $($NewNodeAdapterData['GetNetIPAddressOutput'] | Out-String)" Log-Info "Get-NetAdapter output: $($NewNodeAdapterData['GetNetAdapterOutput'] | Out-String)" } elseif ($NewNodeAdapterData['Result'] -eq $true -and $NewNodeAdapterData['AdapterIP'] -eq $NodeManagementIPAddress) { Log-Info "Physical Adapter found with Correct IP: $($NewNodeAdapterData['AdapterIP'] | Out-String)" $physicalAdapterExists = $true $AdapterContainsMgmtIP = $true $CheckAdapterContainsIPDetail = $lnTxt.CheckAdapterContainsIPPass -f $firstAdapterName, $NodeManagementIPAddress } else { Log-Info "Physical Adapter found but with incorrect IP" Log-Info "Get-NetIPAddress output: $($NewNodeAdapterData['GetNetIPAddressOutput'] | Out-String)" Log-Info "Get-NetAdapter output: $($NewNodeAdapterData['GetNetAdapterOutput'] | Out-String)" } # In certain cases, new node will be set up with the vNIC instead and need to check that for mgmt IP if (!$physicalAdapterExists) { Log-Info "Physical Adapter does not exist or mgmt IP is wrong. Checking Virtual Adapter" -Type Warning $NewNodeVirtualAdapterData = Invoke-Command $newNodeSession -ScriptBlock $adapterSB -ArgumentList $VirtualNICName Log-Info "Data found for New Node Virtual Adapter ($VirtualNICName): $($NewNodeVirtualAdapterData | Out-String)" if ($NewNodeVirtualAdapterData['Result'] -eq $false) { Log-Info "Virtual Adapter Not Found" Log-Info "Get-NetIPAddress output: $($NewNodeVirtualAdapterData['GetNetIPAddressOutput'] | Out-String)" Log-Info "Get-NetAdapter output: $($NewNodeVirtualAdapterData['GetNetAdapterOutput'] | Out-String)" } elseif ($NewNodeVirtualAdapterData['Result'] -eq $true -and $NewNodeVirtualAdapterData['AdapterIP'] -eq $NodeManagementIPAddress) { Log-Info "Virtual Adapter found with Correct IP: $($NewNodeVirtualAdapterData['AdapterIP'] | Out-String)" $AdapterContainsMgmtIP = $true $CheckAdapterContainsIPDetail = $lnTxt.CheckAdapterContainsIPPass -f $VirtualNICName, $NodeManagementIPAddress } else { Log-Info "Virtual Adapter found but with incorrect IP" Log-Info "Get-NetIPAddress output: $($NewNodeVirtualAdapterData['GetNetIPAddressOutput'] | Out-String)" Log-Info "Get-NetAdapter output: $($NewNodeVirtualAdapterData['GetNetAdapterOutput'] | Out-String)" } } } catch { Log-Info "Exception thrown when checking New Node Adapter: $_" -Type Warning } if (!$AdapterContainsMgmtIP) { $CheckAdapterContainsIPDetail = $lnTxt.CheckAdapterContainsIPFail -f $firstAdapterName, $VirtualNICName, $NodeManagementIPAddress Log-Info $CheckAdapterContainsIPDetail -Type Warning $status = 'FAILURE' } else { $status = 'SUCCESS' } $params = @{ Name = 'AzStackHci_Network_Test_New_Node_First_Adapter_Validity' Title = 'Test New Node Configuration First Network Adapter has Management IP' DisplayName = "Test New Node Configuration First Network Adapter has Management IP" Severity = 'CRITICAL' Description = 'Checking New Node first adapter has management IP' Tags = @{} Remediation = 'https://learn.microsoft.com/en-us/azure-stack/hci/deploy/deployment-tool-checklist' TargetResourceID = $NodeManagementIPAddress TargetResourceName = $firstAdapterName TargetResourceType = 'Network Adapter' Timestamp = [datetime]::UtcNow Status = $status AdditionalData = @{ Source = 'NewNodeAdapter' Resource = 'NewNodeAdapterIP' Detail = $CheckAdapterContainsIPDetail Status = $status TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params return $instanceResults } catch { throw $_ } } function Test-NwkValidator_NetworkIntentStatus { <# .SYNOPSIS This test is run in the AddNode context only. This test validates if the intents configured on the existing cluster and the new node to be added are not in errored state. .DESCRIPTION This test performs the following Validations: 1) Check the ATC Intent status on existing nodes are successfully allocated 2) Check if NetworkATC service is running on the new node 3) Check if the existing nodes have storage intent configured in them. .PARAMETERS [System.Management.Automation.Runspaces.PSSession[]] $PSSession #> [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSession ) try { [System.Management.Automation.Runspaces.PSSession[]] $allPSSessions = EnsureTestSessionOpen -PSSessions $PSSession $sessionToCheck = $allPSSessions[0] Log-Info "Checking ATC Intent status on existing nodes and if NetworkATC service is running on the new node." $instanceResults = @() # Get the names of all nodes with an Up Status $activeNodes = (Get-ClusterNode | Where-Object {$_.State -eq "Up"}).Name Log-Info "Active nodes: $($activeNodes | Out-String)" # Get all intents on the active nodes $intents = Get-NetIntentStatus | Where-Object {$activeNodes -contains $_.Host} # Checks the intent status on the existing nodes. foreach ($intent in $intents) { $intentHealthy = $true if ($intent.ConfigurationStatus -ne "Success" -or $intent.ProvisioningStatus -ne "Completed") { $intentHealthy = $false $TestNetworkIntentStatusDetail = $lnTxt.TestNetworkIntentStatusFail -f $intent.Host, $intent.ConfigurationStatus, $intent.ProvisioningStatus Log-Info $TestNetworkIntentStatusDetail -Type Warning } else { $intentHealthy = $true $TestNetworkIntentStatusDetail = $lnTxt.TestNetworkIntentStatusPass -f $intent.Host, $intent.ConfigurationStatus, $intent.ProvisioningStatus } $params = @{ Name = 'AzStackHci_Network_Test_Network_AddNode_Intent_Status' Title = 'Test Network intent on existing nodes' DisplayName = 'Test Network intent on existing nodes' Severity = 'CRITICAL' Description = 'Checking if Network intent is unhealthy on existing nodes' Tags = @{} Remediation = 'https://learn.microsoft.com/en-us/azure-stack/hci/deploy/deployment-tool-checklist' TargetResourceID = 'NetworkIntent' TargetResourceName = 'NetworkIntent' TargetResourceType = 'NetworkIntent' Timestamp = [datetime]::UtcNow Status = if ($intentHealthy) { 'SUCCESS' } else { 'FAILURE' } AdditionalData = @{ Source = $intent.Host Resource = 'AddNodeIntentStatusCheck' Detail = $TestNetworkIntentStatusDetail Status = if ($intentHealthy) { 'SUCCESS' } else { 'FAILURE' } TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params } Log-Info "Checking if the storage intent is configured on the existing cluster before add node." $storageIntent = $intents | Where-Object {$_.IsStorageIntentSet -eq $true} try { $source = Get-Cluster } catch { $source = $Env:COMPUTERNAME Log-Info "Error getting the cluster, we could be running this test in standalone mode on $($source)" } if ($null -eq $storageIntent) { $TestNetworkIntentStatusDetail = $lnTxt.TestStorageIntentNotConfigured -f $source Log-Info $TestNetworkIntentStatusDetail -Type Warning } else { $TestNetworkIntentStatusDetail = $lnTxt.TestStorageIntentConfigured -f $source Log-Info $TestNetworkIntentStatusDetail -Type Success } $params = @{ Name = 'AzStackHci_Network_Test_Network_AddNode_Storage_Intent' Title = 'Test Storage intent on existing nodes' DisplayName = 'Test Storage intent on existing nodes' Severity = 'CRITICAL' Description = 'Check if the storage intent is configured on the existing cluster before add node' Tags = @{} Remediation = 'https://learn.microsoft.com/en-us/azure-stack/hci/deploy/deployment-tool-checklist' TargetResourceID = 'StorageIntent' TargetResourceName = 'StorageIntent' TargetResourceType = 'StorageIntent' Timestamp = [datetime]::UtcNow Status = if ($null -eq $storageIntent) { 'FAILURE' } else { 'SUCCESS' } AdditionalData = @{ Source = $source Resource = 'AddNodeStorageIntentCheck' Detail = $TestNetworkIntentStatusDetail Status = if ($null -eq $storageIntent) { 'FAILURE' } else { 'SUCCESS' } TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params # Check if NetworkATC service is running on the new node $sb = { $retVal = New-Object psobject -Property @{ Pass = $true Status = [string]::Empty } $atcFeature = Get-WindowsFeature -Name NetworkATC if ($atcFeature.Installstate -eq "Installed") { $atcService = Get-Service NetworkATC -ErrorAction SilentlyContinue $retVal.Status = "Feature Installed Service $($atcService.Status)" } elseif ($atcFeature.Installstate -eq "Available") { $retVal.Status = "Feature Available" } else { $retVal.Pass = $false } return $retVal } $NetworkATCStatus = Invoke-Command $sessionToCheck -ScriptBlock $sb $ATCStatusHealthy = $true if (!$NetworkATCStatus.Pass) { # NetworkATC feature not Installed, not Available on the system $ATCStatusHealthy = $false $TestNetworkATCServiceDetail = $lnTxt.TestNetworkATCFeatureNotInSystem -f $sessionToCheck.ComputerName Log-Info $TestNetworkATCServiceDetail -Type Warning } elseif (-not (($NetworkATCStatus.Status -eq 'Feature Installed Service Running') -or ($NetworkATCStatus.Status -eq 'Feature Available'))) { # NetworkATC feature installed but service not 'Running', or feature not available $ATCStatusHealthy = $false $TestNetworkATCServiceDetail = $lnTxt.TestNetworkATCFeatureServiceStatus -f $NetworkATCStatus.Status, $sessionToCheck.ComputerName Log-Info $TestNetworkATCServiceDetail -Type Warning } else { $ATCStatusHealthy = $true $TestNetworkATCServiceDetail = $lnTxt.TestNetworkATCFeatureServiceStatus -f $NetworkATCStatus.Status, $sessionToCheck.ComputerName Log-Info $TestNetworkATCServiceDetail -Type Success } $params = @{ Name = 'AzStackHci_Network_Test_Network_AddNode_NetworkATC_Service' Title = 'Test NetworkATC service is running on new node' DisplayName = 'Test NetworkATC service is running on new node' Severity = 'CRITICAL' Description = 'Check NetworkATC service is running on new node' Tags = @{} Remediation = 'https://learn.microsoft.com/en-us/azure-stack/hci/deploy/deployment-tool-checklist' TargetResourceID = 'NetworkATCService' TargetResourceName = 'NetworkATCService' TargetResourceType = 'NetworkATCService' Timestamp = [datetime]::UtcNow Status = if ($ATCStatusHealthy) { 'SUCCESS' } else { 'FAILURE' } AdditionalData = @{ Source = $sessionToCheck.ComputerName Resource = 'AddNodeNewNodeNetworkATCServiceCheck' Detail = $TestNetworkATCServiceDetail Status = if ($ATCStatusHealthy) { 'SUCCESS' } else { 'FAILURE' } TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $instanceResults += New-AzStackHciResultObject @params return $instanceResults } catch { throw $_ } } ################################################################################################# # Helper Functions ################################################################################################# function TestIPInSubnet { param ( [System.Net.IPAddress]$IpToCheck, [System.String]$ExpectedSubnetCIDR ) # $ExpectedSubnetCIDR should be normalized CIDR format # Get prefix length from expected CIDR $expectedCIDRParts = $ExpectedSubnetCIDR -split '/' $prefixLength = [int]$expectedCIDRParts[1] $networkMask = [Convert]::ToUInt32(("1" * $prefixLength).PadRight(32, "0"), 2) # Trying to get the normalized CIDR for the $IpToCheck and $prefexLength # Calculate the network address $inputIpBytes = $IpToCheck.GetAddressBytes() [Array]::Reverse($inputIpBytes) $inputIpValue = [BitConverter]::ToUInt32($inputIpBytes, 0) $transformedValue = $inputIpValue -band $networkMask $networkIpBytes = [BitConverter]::GetBytes($transformedValue) [Array]::Reverse($networkIpBytes) $networkAddress = New-Object System.Net.IPAddress -ArgumentList (, $networkIpBytes) # Construct the normalized CIDR string $ipSubnetCIDR = $networkAddress.IPAddressToString + '/' + $prefixLength return ($ipSubnetCIDR -eq $ExpectedSubnetCIDR) } function TestMgmtIpPools { <# .SYNOPSIS Ensure all ip are in management subnet. #> param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools, [Parameter(Mandatory = $false, HelpMessage = "Specify Management Subnet")] [string] $ManagementSubnetValue ) try { $allIps = GetMgmtIpRangeFromPools -IpPools $IpPools $uniqueIPs = @{} $firstIp = $IpPools[0].StartingAddress $match = $firstIp -replace "\.[0-9]{1,3}$", "" foreach ($ip in $allIps) { $ipString = $ip.ToString() if ($uniqueIPs.ContainsKey($ipString)) { return $false } else { $uniqueIPs[$ipString] = $true } # Test to make sure all ips in the management subnet in the DHCP scenario $toMatch = $ip -replace "\.[0-9]{1,3}$", "" if ($toMatch -ne $match) { return $false } } # More reliable test to make sure all ips in the management pool in non-DHCP scenarios if (-not ([string]::IsNullOrEmpty($ManagementSubnetValue))) { foreach ($ipPool in $IpPools) { $StartingAddress = $ipPool.StartingAddress $EndingAddress = $ipPool.EndingAddress if (!(CheckIPInRange -IPAddress $StartingAddress -Range $ManagementSubnetValue)) { return $false } if (!(CheckIPInRange -IPAddress $EndingAddress -Range $ManagementSubnetValue)) { return $false } } } return $true } catch { throw "Failed to check ip pools. Error: $_" } } function CheckIPInRange { param( [Parameter(Mandatory=$true)] [string] $IPAddress, # Range in which to search using CIDR notation. (ippaddr/bits) [Parameter(Mandatory=$true)] [string] $Range ) # Split range into the address and the CIDR notation [String]$CIDRAddress = $Range.Split('/')[0] [int]$CIDRBits = $Range.Split('/')[1] # Address from range and the search address are converted to Int32 and the full mask is calculated from the CIDR notation. [int]$BaseAddress = [System.BitConverter]::ToInt32((([System.Net.IPAddress]::Parse($CIDRAddress)).GetAddressBytes()), 0) [int]$Address = [System.BitConverter]::ToInt32(([System.Net.IPAddress]::Parse($IPAddress).GetAddressBytes()), 0) [int]$Mask = [System.Net.IPAddress]::HostToNetworkOrder(-1 -shl ( 32 - $CIDRBits)) return (($BaseAddress -band $Mask) -eq ($Address -band $Mask)) } function GetMgmtIpRangeFromPools { param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools ) $result = @() foreach ($ipPool in $IpPools) { $result += GetMgmtIpRange -StartingAddress $ipPool.StartingAddress -EndingAddress $ipPool.EndingAddress } return $result } function GetMgmtIpRange { param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Net.IPAddress] $StartingAddress, [Parameter(Mandatory = $false, HelpMessage = "Specify end Management IP Range")] [System.Net.IPAddress] $EndingAddress ) try { $first3 = $StartingAddress -replace "\.[0-9]{1,3}$", "" $start = $StartingAddress -split "\." | Select-Object -Last 1 $end = $EndingAddress -split "\." | Select-Object -Last 1 $range = $start..$end | ForEach-Object { ([System.Net.IPAddress]("{0}.{1}" -f $first3, $PSITEM)).IPAddressToString } Log-info "Start: $start and end: $end gives range: $($range -join ',')" return $range } catch { throw "Failed to get Mgmt range. Error: $_" } } function TestMgmtRangeSize { <# .SYNOPSIS Ensure IP range is within boundaries. #> param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools, [int] $Minimum = 6, [int] $Maximum = 16 ) try { $totalCount = 0 foreach ($ipPool in $IpPools) { $StartingAddress = $ipPool.StartingAddress $EndingAddress = $ipPool.EndingAddress $start = $StartingAddress -split "\." | Select-Object -Last 1 $end = $EndingAddress -split "\." | Select-Object -Last 1 $hostCount = ($start..$end).count Log-info "Start: $start and end: $end gives host count: $hostcount" $totalCount += $hostCount } if ($totalCount -gt $Maximum -or $totalCount -lt $Minimum) { return $false } else { return $true } } catch { throw "Failed to check range size. Error: $_" } } function TestMgmtRangePoolCount { <# .SYNOPSIS #1, either one single pool that is big enought (>= Minimum IPs) (for both DHCP and static scenario) <== for general customers , or #2, 2 pools with 1 pool having 1 IP and 2 pool have at least (Minimum - 1 IPs) (for non-DHCP scenario only) <== for specific customers? #> param ( [Parameter(Mandatory = $false, HelpMessage = "Specify starting Management IP Range")] [System.Collections.ArrayList] $IpPools, [int] $Minimum = 5 ) try { $poolCount = $IpPools.Count if ($poolCount -gt 2) { Log-info "Found more than 2 IP pools. Test Failed" return $false } elseif ($poolCount -eq 1) { Log-info "Found only 1 IP Pool. Test Passed" return $true } else # 2 pools { $StartingAddress = $IpPools[0].StartingAddress $EndingAddress = $IpPools[0].EndingAddress $start = $StartingAddress -split "\." | Select-Object -Last 1 $end = $EndingAddress -split "\." | Select-Object -Last 1 $ipCount = ($start..$end).count Log-info "Start: $start and end: $end" if ($ipCount -ne 1) { Log-info "Found more than 1 ip in first IP pool. Test Failed" return $false } $StartingAddress = $IpPools[1].StartingAddress $EndingAddress = $IpPools[1].EndingAddress $start = $StartingAddress -split "\." | Select-Object -Last 1 $end = $EndingAddress -split "\." | Select-Object -Last 1 $ipCount = ($start..$end).count if ($ipCount -lt ($Minimum - 1)) { Log-info "Found less then enough IPs in second IP pool. Test Failed" return $false } Log-info "Test Passed" return $true } } catch { throw "Failed to check IP Pool Count. Error: $_" } } function IsTcpPortInUse { param( [System.Net.IPAddress] $Ip, [int] $Port = 5986, [int] $Timeout = 500 ) try { $tcpClient = New-Object System.Net.Sockets.TcpClient $portOpened = $tcpClient.ConnectAsync($ip, $Port).Wait($timeout) $tcpClient.Dispose() return ($portOpened -contains $true) } catch { throw "Failed to check TCP ports. Error: $_" } } function CheckNetAdapterRDMAStatus { param ( [PSObject[]] $IntentsInfoFromJson ) $retVal = New-Object psobject -Property @{ Pass = $true Message = "on $($ENV:COMPUTERNAME)" } enum NetworkDirectEnabledState { Disabled = 0; Enabled = 1 } # Read RDMA state info for all adapters [PSObject[]] $allAdapterRdmaInfo = Get-NetAdapterRdma [System.Boolean] $validSystemRdmaConfig = $true # need to check each adapter for all intents foreach ($currentIntent in $IntentsInfoFromJson) { [System.String[]] $adaptersToCheck = $currentIntent.Adapter [PSObject[]] $rdmaInfoForAdaptersToCheck = $allAdapterRdmaInfo | Where-Object { $_.Name -in $adaptersToCheck } [Boolean] $currentIntentAdapterOverride = $currentIntent.OverrideAdapterProperty [System.Int32] $currentIntentNetworkDirectOverride = 0 if (-Not [System.String]::IsNullOrEmpty($currentIntent.AdapterPropertyOverrides.NetworkDirect)) { $currentIntentNetworkDirectOverride = [System.Int32] [NetworkDirectEnabledState] $currentIntent.AdapterPropertyOverrides.NetworkDirect } $retVal.Message += "`n Intent $($currentIntent.Name) Adapter Override - [ $currentIntentAdapterOverride ]; NetworkDirect - [ $currentIntentNetworkDirectOverride ]" if ($rdmaInfoForAdaptersToCheck.Count -ne $adaptersToCheck.Count) { # End user provided adapter(s) that don't have RDMA support (a.k.a, Get-NetAdapterRdma returns nothing for the adapter) # So the intent adapter override should have NetworkDirect Disabled. Otherwise the configuration will fail the ATC configuration if (($currentIntentAdapterOverride -and $currentIntentNetworkDirectOverride -eq 0) -or ($rdmaInfoForAdaptersToCheck.Count -eq 0)) { $retVal.Message += "`n Correct configuration for adapters $($adaptersToCheck | Out-String): RDMA not supported, and configured with intent adapter override to disable NetworkDirect" $retVal.Message += "`n Or Get-NetAdapterRdma call returned nothing for the adapter(s) $($adaptersToCheck | Out-String)" } else { $retVal.Message += "`n Wrong configuration for adapters $($adaptersToCheck | Out-String): RDMA not supported, but not configured with intent adapter override to disable NetworkDirect" $validSystemRdmaConfig = $false } } else { foreach ($currentRdmaInfo in $rdmaInfoForAdaptersToCheck) { # The following conditions are valid for RDMA configuration: # RDMA Enabled | RDMA OperationalStatus | Override | OverrideValue # True | True | - | - # - | False | True | 0 $rdmaEnabled = $currentRdmaInfo.Enabled $rdmaOperationalState = $currentRdmaInfo.OperationalState $validRdmaForCurrentAdapter = ($rdmaEnabled -and $rdmaOperationalState) -or ((-not $rdmaOperationalState) -and $currentIntentAdapterOverride -and $currentIntentNetworkDirectOverride -eq 0) if (-not $validRdmaForCurrentAdapter) { $retVal.Message += "`n Wrong configuration for adapter $($currentRdmaInfo.Name): RDMA Enabled - [ $rdmaEnabled ]; RDMA OperationalState - [ $rdmaOperationalState ]" } else { $retVal.Message += "`n Correct configuration for adapter $($currentRdmaInfo.Name): RDMA Enabled - [ $rdmaEnabled ], RDMA OperationalState - [ $rdmaOperationalState ]" } $validSystemRdmaConfig = $validSystemRdmaConfig -and $validRdmaForCurrentAdapter } } } if (-not $validSystemRdmaConfig) { $retVal.Pass = $false $retVal.Message = "`nERROR: RDMA setting on adapters are invalid " + $retVal.Message } else { $retVal.Pass = $true $retVal.Message = "`nPASS: RDMA setting on adapters are valid " + $retVal.Message } return $retVal } function CheckAdapterSymmetryAndBandwidth { param ( [PSObject[]] $IntentsInfoFromJson, [System.Int64] $ExpectedBandWidth = 10000000000 ) enum NetworkDirectEnabledState { Disabled = 0; Enabled = 1 } $nodeName = $env:COMPUTERNAME $retVal = New-Object psobject -Property @{ Pass = $true Message = "on $($nodeName)`n" } [PSObject[]] $allAdapterInfo = Get-NetAdapter foreach ($currentIntent in $IntentsInfoFromJson) { [System.String[]] $adaptersToCheck = $currentIntent.Adapter $intentAdapterInfoToCheck = $allAdapterInfo | Where-Object { $_.Name -in $adaptersToCheck } # Check adapter symmetry $retVal.Message += "`n--- Adapter Symmetry Check: Link speed and Component ID should be same for all adapters in the intent" $compIDFail = $false $linkSpeedFail = $false $expectedSpeed = $null $expectedComponentID = $null foreach ($nicInfo in $intentAdapterInfoToCheck) { if ($null -eq $expectedSpeed) { $expectedSpeed = $nicInfo.Speed } if ($null -eq $expectedComponentID) { $expectedComponentID = $nicInfo.ComponentID } if ($expectedSpeed -ne $nicInfo.Speed) { $linkSpeedFail = $true } if ($expectedComponentID -ne $nicInfo.ComponentID) { $compIDFail = $true } if ($linkSpeedFail -Or $compIDFail) { $retVal.Pass = $false } $retVal.Message += "`n -- $nodeName ($($nicInfo.Name),`t$($nicInfo.LinkSpeed),`t$($nicInfo.ComponentID))" } # Check adapter bandwidth # This is needed if current intent is for storage traffic and adapter property is not overridden with NetworkDirect Disabled [Boolean] $currentIntentAdapterOverride = $currentIntent.OverrideAdapterProperty [System.Int32] $currentIntentNetworkDirectOverride = 0 if (-Not [System.String]::IsNullOrEmpty($currentIntent.AdapterPropertyOverrides.NetworkDirect)) { $currentIntentNetworkDirectOverride = [System.Int32] [NetworkDirectEnabledState] $currentIntent.AdapterPropertyOverrides.NetworkDirect } $needCheckBandwidth = $currentIntent.TrafficType.Contains("Storage") -and (-not $currentIntentAdapterOverride -or $currentIntentNetworkDirectOverride -ne 0) if ($needCheckBandwidth) { $retVal.Message += "`n--- Adapter Bandwidth Check for storage adapters when RDMA enabled: Need to be 10Gbps or higher" foreach ($nicInfo in $intentAdapterInfoToCheck) { if ($nicInfo.Speed) { if ([System.Int64] $nicInfo.Speed -lt $ExpectedBandWidth) { $retVal.Pass = $false } $retVal.Message += "`n -- $nodeName ($($nicInfo.Name),`t$($nicInfo.LinkSpeed))" } else { $retVal.Pass = $false $retVal.Message += "`n -- $nodeName ($($nicInfo.Name), Speed not available)" } } } } if ($retVal.Pass) { $retVal.Message = "`nPASS: Network adapter(s) are symmetric and meet bandwidth requirement " + $retVal.Message } else { $retVal.Message = "`nERROR: Network adapter(s) are not symmetric or do not meet bandwidth requirement " + $retVal.Message } return $retVal } function CheckHostNetworkConfigurationReadiness { param ( [PSObject[]] $IntentsInfoFromJson ) $retVal = New-Object psobject -Property @{ Pass = $true Message = "On $($ENV:COMPUTERNAME):" } [System.String[]] $intentAdapters = $IntentsInfoFromJson | ForEach-Object { $_.Adapter } | Select-Object -Unique [PSObject[]] $extSwitchInfo = @() if ((Get-Command Get-VMSwitch -ErrorAction SilentlyContinue) -and (Get-WindowsFeature -Name Hyper-V -ErrorAction SilentlyContinue).Installed) { $extSwitchInfo = Get-VMSwitch -SwitchType External } [System.String] $interimPassMessage = "" #region Check DNS client configuration [PSObject[]] $adapterDnsClientInfo = Get-DNSClient [System.String[]] $adpaterWithDNSClientInfo = $adapterDnsClientInfo.InterfaceAlias | Select-Object -Unique [System.String[]] $adaptersToCheck = @() if ($extSwitchInfo.Count -eq 0) { # In case there is no VMSwitch in the system, we will need to make sure all adapters used in intents are in the result of Get-DNSClient $adaptersToCheck = $intentAdapters } else { # if there is a VMSwitch, we will need to make sure that those adapters not in VMSwitch but in intent are in the result of Get-DNSClient [System.Guid[]] $switchAdapterGuids = $extSwitchInfo | ForEach-Object { $_.NetAdapterInterfaceGuid } [System.String[]] $adaptersNotInVMSwitchNames = Get-NetAdapter -Physical | Where-Object { $_.InterfaceGuid -notin $switchAdapterGuids } | ForEach-Object { $_.Name } $adaptersToCheck = $intentAdapters | Where-Object { $_ -in $adaptersNotInVMSwitchNames } } if ($adaptersToCheck.Count -eq 0) { #This means all the adapters defined in intent are used in VMSwitch $intentAdapterMissingDnsClient = $null } else { $intentAdapterMissingDnsClient = Compare-Object $adaptersToCheck $adpaterWithDNSClientInfo | Where-Object { $_.SideIndicator -eq "<=" } | ForEach-Object { $_.InputObject } } if ($intentAdapterMissingDnsClient.Count -gt 0) { $retVal.Pass = $false $retVal.Message += "`nERROR: DNS Client configuration is missing for the following adapter(s): $($intentAdapterMissingDnsClient -join ', ')" } else { $interimPassMessage += "`nPASS: DNS Client configuration has valid data for all adapters defined in intent" } #endregion #region Check Hyper-V running status by calling Get-VMHost if ((Get-Command Get-VMHost -ErrorAction SilentlyContinue) -and (Get-WindowsFeature -Name Hyper-V -ErrorAction SilentlyContinue).Installed) { [PSObject[]] $vmHostInfo = Get-VMHost -ErrorAction SilentlyContinue if ($vmHostInfo.Count -eq 0) { $retVal.Pass = $false $retVal.Message += "`nERROR: Hyper-V is not running correctly on the system" } else { $interimPassMessage += "`nPASS: Hyper-V is running correctly on the system" } } else { $interimPassMessage += "`nWARNING: Hyper-V-PowerShell might not installed correctly on the system. Will skip VM host check." } #endregion #region Check VMSwitch readiness # At leas 1 VMSwitch is having the network adapter defined in the management intent # Or management intent adapters are not included in any VMSwitch if ($extSwitchInfo.Count -ge 1) { [System.String[]] $mgmtIntentAdapterNames = $IntentsInfoFromJson | Where-Object { $_.TrafficType.Contains("Management") } | ForEach-Object { $_.Adapter } | Select-Object -Unique [System.Boolean] $foundMgmtVMSwitch = $false foreach ($currentSwitchInfo in $extSwitchInfo) { [System.Guid[]] $currentSwitchAdapterGuids = $currentSwitchInfo | ForEach-Object { $_.NetAdapterInterfaceGuid } [System.String[]] $currentSwitchAdapterNames = Get-NetAdapter -Physical | Where-Object { $_.InterfaceGuid -in $currentSwitchAdapterGuids } | ForEach-Object { $_.Name } $tempRst = Compare-Object $mgmtIntentAdapterNames $currentSwitchAdapterNames | Where-Object { $_.SideIndicator -eq "<=" } | ForEach-Object { $_.InputObject } if ($tempRst.Count -eq 0) { $foundMgmtVMSwitch = $true break } } if ($foundMgmtVMSwitch) { $interimPassMessage += "`nPASS: At least 1 VMSwitch is having the network adapter defined in the management intent" } else { $retVal.Pass = $false $retVal.Message += "`nERROR: No VMSwitch is having the network adapter defined in the management intent" } } #endregion #Region Check advanced property VlanId on adapters foreach ($pNIC in $intentAdapters) { $currentAdapterAdvancedPropertyVlanId = Get-NetAdapterAdvancedProperty -Name $pNIC -RegistryKeyword VlanId -ErrorAction SilentlyContinue if (($null -eq $currentAdapterAdvancedPropertyVlanId) -or ($null -eq $currentAdapterAdvancedPropertyVlanId.RegistryValue)) { $retVal.Pass = $false $retVal.Message += "`nERROR: Cannot find valid advanced property VlanId for adapter $pNIC. Use Get-NetAdapterAdvancedProperty/Set-NetAdapterAdvancedProperty with parameter RegistryKeyword set to VlanId to verify and configure it." } } if ($retVal.Pass) { $interimPassMessage += "`nPASS: Advanced property VlanId exist for all adapters defined in intent" } #endregion #region Check RSS property on adapters foreach ($pNIC in $intentAdapters) { [PSOBject[]] $currentAdapterRSSSetting = Get-NetAdapterRss -Name $pNIC -ErrorAction SilentlyContinue if (($null -eq $currentAdapterRSSSetting) -or ($currentAdapterRSSSetting.Count -eq 0)) { $retVal.Pass = $false $retVal.Message += "`nERROR: Cannot find valid RSS property for adapter $pNIC. Adapter need support RSS. Use Get-NetAdapterRss to verify it." } } if ($retVal.Pass) { $interimPassMessage += "`nPASS: RSS property exists for all adapters defined in intent" } #endregion #region Check pNIC are in the intent adapters [System.String[]] $allpNicUpInSystem = Get-NetAdapter -Physical -Name $intentAdapters -ErrorAction SilentlyContinue | Where-Object { $_.Status -eq "Up" } | ForEach-Object { $_.Name } $adapterCompareResult = Compare-Object $intentAdapters $allpNicUpInSystem | Where-Object { $_.SideIndicator -eq "<=" } | ForEach-Object { $_.InputObject } if ($adapterCompareResult.Count -gt 0) { $retVal.Pass = $false $retVal.Message += "`nERROR: The following adapter(s) are not physical adapter or not Up in the system: $($adapterCompareResult -join ', '). Intent adapters should be physical adapters and Up in the system." } else { $interimPassMessage += "`nPASS: All adapters defined in intent are physical NICs and Up in the system" } #endregion $retVal.Message += $interimPassMessage return $retVal } function ConfigureVMSwitchForTesting { [CmdletBinding()] param ( [System.String[]] $SwitchAdapterNames, [System.String] $MgmtIntentName = "", [System.Boolean] $UpdateMgmtAdapter = $true, [System.String] $ExpectedVMSwitchName = "ConvergedSwitch($($MgmtIntentName))", [System.String] $ExpectedMgmtVNicName = "vManagement($($MgmtIntentName))" ) [PSObject] $retVal = New-Object PSObject -Property @{ VMSwitchInfo = $null MgmtVlanId = 0 NeedCleanUp = $false IPReady = $false } $tmpVMSwitch = New-VMSwitch -Name $ExpectedVMSwitchName -NetAdapterName $SwitchAdapterNames -EnableEmbeddedTeaming $true -AllowManagementOS $UpdateMgmtAdapter if ($tmpVMSwitch) { $retVal.VMSwitchInfo = $tmpVMSwitch $retVal.MgmtVlanId = 0 $retVal.NeedCleanUp = $true if ($UpdateMgmtAdapter) { $mgmtVlanId = 0 $existingPNICVlanId = Get-NetAdapterAdvancedProperty -RegistryKeyword VlanID -Name $SwitchAdapterNames[0] -ErrorAction SilentlyContinue if ($existingPNICVlanId -and $existingPNICVlanId.RegistryValue) { $mgmtVlanId = $existingPNICVlanId.RegistryValue[0] } Rename-VMNetworkAdapter -ManagementOS -Name $ExpectedVMSwitchName -NewName $ExpectedMgmtVNicName Get-NetAdapter -name "vEthernet ($($ExpectedMgmtVNicName))" -ErrorAction SilentlyContinue | Rename-NetAdapter -NewName $ExpectedMgmtVNicName if ($mgmtVlanId -ne 0) { $retVal.MgmtVlanId = $mgmtVlanId Set-VMNetworkAdapterIsolation -ManagementOS ` -VMNetworkAdapterName $ExpectedMgmtVNicName ` -IsolationMode Vlan ` -AllowUntaggedTraffic $true ` -DefaultIsolationID $mgmtVlanId } # In case of DHCP scenario, the new adapter might not get the IP address immediately # Wait for some time (60 seconds) to make sure the new IP is settled correctly. [System.Boolean] $currentIPReady = $false $ipStopWatch = [System.diagnostics.stopwatch]::StartNew() while (-not $currentIPReady -and ($ipStopWatch.Elapsed.TotalSeconds -lt 60)) { # If the vNIC has Manual or Dhcp IPv4 address with "Preferred" state, we consider it as "ready" $ipConfig = Get-NetIPAddress -InterfaceAlias $ExpectedMgmtVNicName -ErrorAction SilentlyContinue | Where-Object { ($_.PrefixOrigin -eq "Manual" -or $_.PrefixOrigin -eq "Dhcp") -and $_.AddressFamily -eq "IPv4" -and $_.AddressState -eq "Preferred" } if ($ipConfig) { $currentIPReady = $true $retVal.IPReady = $true break } else { Start-Sleep -Seconds 3 } } if (-not $currentIPReady) { # should not get into here, but keep it here for safety Write-Host "Cannot get the IP address bind to the vNIC after VMSwitch created. Please check the system manually." } else { Write-Host "VMSwitch created successfully. VMSwitch: $($ExpectedVMSwitchName), MgmtVNic: $($ExpectedMgmtVNicName)" } } } return $retVal } function New-PsSessionWithRetriesInternal { param ( [System.String] $Node, [PSCredential] $Credential, [System.Int16] $Retries = 60, [System.Int16] $WaitSeconds = 10 ) for ($i=1; $i -le $Retries; $i++) { try { Trace-Execution "Creating PsSession ($i/$Retries) to $Node as $($Credential.UserName)..." $psSessionCreated = Microsoft.PowerShell.Core\New-PSSession -ComputerName $Node -Credential $Credential -ErrorAction Stop $computerNameFromSession = Microsoft.PowerShell.Core\Invoke-Command -Session $psSessionCreated -ScriptBlock { $ENV:COMPUTERNAME } -ErrorAction Stop $isAdminSession = Microsoft.PowerShell.Core\Invoke-Command -Session $psSessionCreated -ScriptBlock { ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] 'Administrator') } -ErrorAction Stop if (-not $isAdminSession) { throw ("PsSession was successful but user: {0} is not an administrator on computer {1} " -f $psSessionCreated.Runspace.ConnectionInfo.Credential.Username, $computerName) } break } catch { Trace-Execution "Creating PsSession ($i/$Retries) to $Node failed: $($_.exception.message)" $errMsg = $_.tostring() Start-Sleep -Seconds $WaitSeconds } } if ($psSessionCreated -and $computerNameFromSession -and $isAdminSession) { Trace-Execution ("PsSession to {0} created after {1} retries. (Remote machine name: {2})" -f $Node, ("$i/$retries"), $computerNameFromSession) return $psSessionCreated } else { throw "Unable to create a valid session to $Node`: $errMsg" } } function GetSortedMgmtIntentAdapter { param ( [System.String[]] $MgmtAdapterNames ) Log-Info "Make sure 1st mgmt intent adapter is the one with valid IP address in it" Log-Info "$($MgmtAdapterNames -join ",")" # Re-arrange the order in $MgmtAdapterNames to make sure the nic having a valid IPv4 address appears before the other NIC in the array $mgmtNicNamesTemp = [System.Collections.ArrayList] $MgmtAdapterNames foreach($name in $MgmtAdapterNames) { $a = Get-NetIPAddress -InterfaceAlias $name -AddressFamily ipv4 -Type Unicast -AddressState Preferred -PrefixOrigin Dhcp -ErrorAction SilentlyContinue $b = Get-NetIPAddress -InterfaceAlias $name -AddressFamily ipv4 -Type Unicast -AddressState Preferred -PrefixOrigin Manual -ErrorAction SilentlyContinue if (($null -ne $a) -or ($null -ne $b)) { # move the NIC name to the top $mgmtNicNamesTemp.Remove($name) $mgmtNicNamesTemp.Insert(0, $name) break } } [System.String[]] $retVal = [System.String[]] $mgmtNicNamesTemp Log-Info "Got sorted adapters list:" Log-Info "$($retVal -join ",")" return $retVal } function CheckStorageAdapterReadiness { param ([String[]] $AdaptersToCheck) $retVal = New-Object PSObject -Property @{ Pass = $true Message = "Storage adapter IP and VLAN check on $($ENV:COMPUTERNAME)" } foreach ($expectedAdapter in $AdaptersToCheck) { [PSObject[]] $currentAdapterInfo = Get-NetAdapter -Physical -Name $expectedAdapter -ErrorAction SilentlyContinue if ($currentAdapterInfo.Count -ne 1) { $retVal.Pass = $false $retVal.Message += "`n !! Expect [ 1 ] adapter with name [ $expectedAdapter ]. But found [ $($currentAdapterInfo.Count) ]." $retVal.Message += "`n !! Get-NetAdapter -Physical -Name $($expectedAdapter)" } else { [PSObject[]] $currentAdapterIPAddressInfo = Get-NetIPAddress -AddressFamily IPv4 -InterfaceIndex $currentAdapterInfo.InterfaceIndex -PrefixOrigin @("Dhcp", "Manual") -ErrorAction SilentlyContinue if ($currentAdapterIPAddressInfo.Count -ne 0) { $retVal.Pass = $false $retVal.Message += "`n !! Found valid Dhcp/Manual IP address(es) [ $($currentAdapterIPAddressInfo.IPAddress) ] on storage adapter [ $expectedAdapter ]. Please remove the IP from the adapter and disable DHCP on it." $retVal.Message += "`n !! Set-NetIPInterface -InterfaceIndex $($currentAdapterInfo.InterfaceIndex) -Dhcp Disabled" $retVal.Message += "`n !! Remove-NetIPAddress -InterfaceIndex $($currentAdapterInfo.InterfaceIndex)" } else { $retVal.Message += "`n Passed: Storage adapter [ $expectedAdapter ] don't have any IP address on it." } [String[]] $currentAdapterVlanInfo = (Get-NetAdapterAdvancedProperty -Name $expectedAdapter -RegistryKeyword VLANID -ErrorAction SilentlyContinue).RegistryValue if (-not $currentAdapterVlanInfo) { $retVal.Pass = $false $retVal.Message += "`n !! Cannot get info of advanced property `"VLANID`" from adapter [ $expectedAdapter ]. Please make sure the adapter support VLANID." $retVal.Message += "`n !! Get-NetAdapterAdvancedProperty -Name $($expectedAdapter)" } else { if (($currentAdapterVlanInfo.Count -gt 1) -or ($currentAdapterVlanInfo[0] -ne "0")) { $retVal.Pass = $false $retVal.Message += "`n !! Found VLANID [ $($currentAdapterVlanInfo) ] on [ $expectedAdapter ]. Please remove the VLANID from the adapter" $retVal.Message += "`n !! Set-NetAdapterAdvancedProperty -Name $($expectedAdapter) -RegistryKeyword VLANID -RegistryValue 0" } else { $retVal.Message += "`n Passed: Storage adapter [ $expectedAdapter ] don't have any VLANID configured on it." } } } } return $retVal } function ConfigureStorageVMSwitchVNICForTesting { [CmdletBinding()] param ( [System.String] $StorageVMSwitchAdapter, [System.String] $StorageVMSwitchName, [System.String[]] $StorageVNICNames, [System.Collections.Hashtable] $StorageAdapterVLANIDInfo, [System.Boolean] $IncludeMgmt = $false ) $tmpRst = ConfigureVMSwitchForTesting -SwitchAdapterNames @($StorageVMSwitchAdapter) -UpdateMgmtAdapter $IncludeMgmt -ExpectedVMSwitchName $StorageVMSwitchName CreateStorageVNIC -StorageVMSwitchName $tmpRst.VMSwitchInfo.Name ` -StorageVNICNames $StorageVNICNames ` -StorageAdapterVLANIDInfo $StorageAdapterVLANIDInfo return $tmpRst } function ValidateStorageConnections { [CmdletBinding()] param ( [Parameter(Mandatory = $true, HelpMessage = "Specify PSSession array of all validation session needed for ping mesh testing.")] [System.Management.Automation.Runspaces.PSSession[]] $AllNodeSessions, [Parameter(Mandatory = $true, HelpMessage = "Specify host IPV4 table of all nodes.")] [System.Collections.Hashtable] $HostIPv4Table ) $retVal = @() Log-Info "Use ping mesh to check storage adapter connections." foreach ($sourceMachineSession in $AllNodeSessions) { $validationRst = New-Object PSObject -Property @{ Pass = $true Message = "Storage adapter connection validation using ping test on $($sourceMachineSession.ComputerName)." } Log-Info "Run ping mesh on $($sourceMachineSession.ComputerName)." $nodePingRst = Invoke-Command -Session $sourceMachineSession -ScriptBlock { param ($HostAdapterIPv4Table, $SourceNodeKey) $pingRst = New-Object PSObject -Property @{ Pass = $true Message = "Ping result from $($ENV:COMPUTERNAME) [ $($SourceNodeKey) ] to other host(s):" } $sourceAdapterIPv4 = $HostAdapterIPv4Table[$SourceNodeKey] foreach ($destHost in $HostAdapterIPv4Table.Keys) { if ($destHost -ne $SourceNodeKey) { $pingRst.Message += "`n === Destination host [ $($destHost) ]" $destAdapterIPv4 = $HostAdapterIPv4Table[$destHost] foreach ($adapter in $sourceAdapterIPv4.Keys) { $pingRst.Message += "`n adapter [ $($adapter) ]" $sourceIp = $sourceAdapterIPv4[$adapter] $destIp = $destAdapterIPv4[$adapter] # Check ping result: if contains string like "Reploy from 169.254.123.12: bytes=", then it's successful $output = ping $destIp -S $sourceIp -n 2 $success = $output | Select-String "Reply from $($destIp): bytes=" -Quiet if (-not $success) { $pingRst.Pass = $false $pingRst.Message += "`n !!! FAILED: Ping from [ $sourceIp ] to [ $destIp ] !!!" } else { $pingRst.Message += "`n PASSED: Ping from [ $sourceIp ] to [ $destIp ]" } } } } return $pingRst } -ArgumentList $HostIPv4Table, $sourceMachineSession.ComputerName if (-not $nodePingRst.Pass) { Log-Info " Ping test failed on $($sourceMachineSession.ComputerName)." $validationRst.Pass = $false } else { Log-Info " Ping test passed on $($sourceMachineSession.ComputerName)." } $validationRst.Message += "`n$($nodePingRst.Message)" $validationRstStatus = if ($validationRst.Pass) { 'SUCCESS' } else { 'FAILURE' } $storageAdapterConnectionRstObject = @{ Name = "AzStackHci_Network_Test_StorageConnections_PING_MESH" Title = "Test storage adapter connections with ping mesh on $($sourceMachineSession.ComputerName)" DisplayName = "Test storage adapter connections with ping mesh on $($sourceMachineSession.ComputerName)" Severity = 'WARNING' Description = "Run ping mesh to validate storage adapter connections from $($sourceMachineSession.ComputerName) to other hosts" Tags = @{} Remediation = "Make sure network used by storage adapter is configured correctly so the adapters could be pinged each other" TargetResourceID = 'ValidateStorageConnectionsPingMesh' TargetResourceName = 'ValidateStorageConnectionsPingMesh' TargetResourceType = 'ValidateStorageConnectionsPingMesh' Timestamp = [datetime]::UtcNow Status = $validationRstStatus AdditionalData = @{ Source = $sourceMachineSession.ComputerName Resource = 'ValidateStorageConnectionsPingMesh' Detail = $validationRst.Message Status = $validationRstStatus TimeStamp = [datetime]::UtcNow } HealthCheckSource = $ENV:EnvChkrId } $retVal += New-AzStackHciResultObject @storageAdapterConnectionRstObject Log-Info " Finsihed ping test on $($sourceMachineSession.ComputerName)." } return $retVal } function CreateStorageVNIC { [CmdletBinding()] param ( [System.String] $StorageVMSwitchName, [System.String[]] $StorageVNICNames, [System.Collections.Hashtable] $StorageAdapterVLANIDInfo ) foreach ($tmpStorageVNIC in $StorageVNICNames) { Add-VMNetworkAdapter -ManagementOS -SwitchName $StorageVMSwitchName -Name $tmpStorageVNIC Get-NetAdapter -name "vEthernet ($($tmpStorageVNIC))" -ErrorAction SilentlyContinue | Rename-NetAdapter -NewName $tmpStorageVNIC Set-NetIPInterface -InterfaceAlias $tmpStorageVNIC -Dhcp Disabled Set-DnsClient -InterfaceAlias $tmpStorageVNIC -RegisterThisConnectionsAddress $false Set-VMNetworkAdapterIsolation -ManagementOS ` -VMNetworkAdapterName $tmpStorageVNIC ` -IsolationMode Vlan ` -AllowUntaggedTraffic $true ` -DefaultIsolationID $StorageAdapterVLANIDInfo[$tmpStorageVNIC] } } function EnsureTestSessionOpen { <# .SYNOPSIS Make sure the test session is opened for the given PSSessions .DESCRIPTION Make sure the test session is opened for the given PSSessions. If the session is not opened, open a new session for it. .PARAMETER PSSessions The PSSessions to be checked .EXAMPLE EnsureTestSessionOpen -PSSessions $PSSessions #> [CmdletBinding()] param ( [System.Management.Automation.Runspaces.PSSession[]] $PSSessions ) [System.Management.Automation.Runspaces.PSSession[]] $newTestSessionsAfterChecking = @() foreach ($testSession in $PSSessions) { [System.Management.Automation.Runspaces.PSSession] $sessionToReturn = $null Log-Info "[EnsureTestSessionOpen] Clean up PSSession on $($testSession.ComputerName) and create a new session" Remove-PSSession -Session $testSession -ErrorAction SilentlyContinue $sessionCredential = $testSession.Runspace.ConnectionInfo.Credential $sessionToReturn = New-PsSessionWithRetriesInternal -Node $testSession.ComputerName -Credential $sessionCredential $newTestSessionsAfterChecking += $sessionToReturn } return $newTestSessionsAfterChecking } # SIG # Begin signature block # MIIoQwYJKoZIhvcNAQcCoIIoNDCCKDACAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCA/mn3VKyOfW9UU # LlmP6q7NOE2cFSOyrG/RGQV3bZ1y16CCDXYwggX0MIID3KADAgECAhMzAAAEBGx0 # Bv9XKydyAAAAAAQEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTE0WhcNMjUwOTExMjAxMTE0WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQC0KDfaY50MDqsEGdlIzDHBd6CqIMRQWW9Af1LHDDTuFjfDsvna0nEuDSYJmNyz # NB10jpbg0lhvkT1AzfX2TLITSXwS8D+mBzGCWMM/wTpciWBV/pbjSazbzoKvRrNo # DV/u9omOM2Eawyo5JJJdNkM2d8qzkQ0bRuRd4HarmGunSouyb9NY7egWN5E5lUc3 # a2AROzAdHdYpObpCOdeAY2P5XqtJkk79aROpzw16wCjdSn8qMzCBzR7rvH2WVkvF # HLIxZQET1yhPb6lRmpgBQNnzidHV2Ocxjc8wNiIDzgbDkmlx54QPfw7RwQi8p1fy # 4byhBrTjv568x8NGv3gwb0RbAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU8huhNbETDU+ZWllL4DNMPCijEU4w # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMjkyMzAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAIjmD9IpQVvfB1QehvpC # Ge7QeTQkKQ7j3bmDMjwSqFL4ri6ae9IFTdpywn5smmtSIyKYDn3/nHtaEn0X1NBj # L5oP0BjAy1sqxD+uy35B+V8wv5GrxhMDJP8l2QjLtH/UglSTIhLqyt8bUAqVfyfp # h4COMRvwwjTvChtCnUXXACuCXYHWalOoc0OU2oGN+mPJIJJxaNQc1sjBsMbGIWv3 # cmgSHkCEmrMv7yaidpePt6V+yPMik+eXw3IfZ5eNOiNgL1rZzgSJfTnvUqiaEQ0X # dG1HbkDv9fv6CTq6m4Ty3IzLiwGSXYxRIXTxT4TYs5VxHy2uFjFXWVSL0J2ARTYL # E4Oyl1wXDF1PX4bxg1yDMfKPHcE1Ijic5lx1KdK1SkaEJdto4hd++05J9Bf9TAmi # u6EK6C9Oe5vRadroJCK26uCUI4zIjL/qG7mswW+qT0CW0gnR9JHkXCWNbo8ccMk1 # sJatmRoSAifbgzaYbUz8+lv+IXy5GFuAmLnNbGjacB3IMGpa+lbFgih57/fIhamq # 5VhxgaEmn/UjWyr+cPiAFWuTVIpfsOjbEAww75wURNM1Imp9NJKye1O24EspEHmb # DmqCUcq7NqkOKIG4PVm3hDDED/WQpzJDkvu4FrIbvyTGVU01vKsg4UfcdiZ0fQ+/ # V0hf8yrtq9CkB8iIuk5bBxuPMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGiMwghofAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAQEbHQG/1crJ3IAAAAABAQwDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIENM/OQS7zpROl3lceodtKjU # lDDOpcdld6ovdPtyWaQaMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAsvA6hHJFOtmSCS6fTAz3HZ3d8a6U5sjpowFnP08qBiYSB6sbDokXBwl9 # ig3UT/erGQXECwSEODZsQQBpFiLLLqY6IjjOk4W6tHrvT9QKn6AIvvpjAbb67jNP # 2weytH/VsYC/Fl7o6gGfWweNtVjYRJehOAx6bZ7KraXuHoVb9Pm02rEwUE+ye8cc # 0gGFokS54QK6B8Obib91SGMC2QIUreIaf0ALx8CN6TDAMWpsX2tyyCvhneYFTKoj # 1GTotY8XFWnh5Mkby186wRsNAgoHBoOdvCmeLBxQsM4YJTbC2sCZlI9zIuFBbDTO # KD6izQcbb2vFS1v1nU1eKh6dU2sxuaGCF60wghepBgorBgEEAYI3AwMBMYIXmTCC # F5UGCSqGSIb3DQEHAqCCF4YwgheCAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFaBgsq # hkiG9w0BCRABBKCCAUkEggFFMIIBQQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCB2tuBKDUpFg+VDPEXAHSNgq5GX+TrkvcD7mqX+/W3MNAIGZ0pekNkd # GBMyMDI0MTIwNDE1MDM1NC4yMThaMASAAgH0oIHZpIHWMIHTMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT # Tjo2NTFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg # U2VydmljZaCCEfswggcoMIIFEKADAgECAhMzAAAB9ZkJlLzxxlCMAAEAAAH1MA0G # CSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4XDTI0 # MDcyNTE4MzEwMVoXDTI1MTAyMjE4MzEwMVowgdMxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9w # ZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjY1MUEt # MDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNl # MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzO90cFQTWd/WP84IT7JM # IW1fQL61sdfgmhlfT0nvYEb2kvkNF073ZwjveuSWot387LjE0TCiG93e6I0HzIFQ # BnbxGP/WPBUirFq7WE5RAsuhNfYUL+PIb9jJq3CwWxICfw5t/pTyIOHjKvo1lQOT # WZypir/psZwEE7y2uWAPbZJTFrKen5R73x2Hbxy4eW1DcmXjym2wFWv10sBH40aj # Jfe+OkwcTdoYrY3KkpN/RQSjeycK0bhjo0CGYIYa+ZMAao0SNR/R1J1Y6sLkiCJO # 3aQrbS1Sz7l+/qJgy8fyEZMND5Ms7C0sEaOvoBHiWSpTM4vc0xDLCmc6PGv03CtW # u2KiyqrL8BAB1EYyOShI3IT79arDIDrL+de91FfjmSbBY5j+HvS0l3dXkjP3Hon8 # b74lWwikF0rzErF0n3khVAusx7Sm1oGG+06hz9XAy3Wou+T6Se6oa5LDiQgPTfWR # /j9FNk8Ju06oSfTh6c03V0ulla0Iwy+HzUl+WmYxFLU0PiaXsmgudNwVqn51zr+B # i3XPJ85wWuy6GGT7nBDmXNzTNkzK98DBQjTOabQXUZ884Yb9DFNcigmeVTYkyUXZ # 6hscd8Nyq45A3D3bk+nXnsogK1Z7zZj6XbGft7xgOYvveU6p0+frthbF7MXv+i5q # cD9HfFmOq4VYHevVesYb6P0CAwEAAaOCAUkwggFFMB0GA1UdDgQWBBRV4Hxb9Uo0 # oHDwJZJe22ixe2B1ATAfBgNVHSMEGDAWgBSfpxVdAF5iXYP05dJlpxtTNRnpcjBf # BgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3Bz # L2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcmww # bAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRwOi8vd3d3Lm1pY3Jvc29m # dC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0El # MjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUF # BwMIMA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAgEAcwxmVPaA9xHf # fuom0TOSp2hspuf1G0cHW/KXHAuhnpW8/Svlq5j9aKI/8/G6fGIQMr0zlpau8jy8 # 3I4zclGdJjl5S02SxDlUKawtWvgf7ida06PgjeQM1eX4Lut4bbPfT0FEp77G76hh # ysXxTJNHv5y+fwThUeiiclihZwqcZMpa46m+oV6igTU6I0EnneotMqFs0Q3zHgVV # r4WXjnG2Bcnkip42edyg/9iXczqTBrEkvTz0UlltpFGaQnLzq+No8VEgq0UG7W1E # LZGhmmxFmHABwTT6sPJFV68DfLoC0iB9Qbb9VZ8mvbTV5JtISBklTuVAlEkzXi9L # IjNmx+kndBfKP8dxG/xbRXptQDQDaCsS6ogLkwLgH6zSs+ul9WmzI0F8zImbhnZh # UziIHheFo4H+ZoojPYcgTK6/3bkSbOabmQFf95B8B6e5WqXbS5s9OdMdUlW1gTI1 # r5u+WAwH2KG7dxneoTbf/jYl3TUtP7AHpyck2c0nun/Q0Cycpa9QUH/Dy01k6tQo # mNXGjivg2/BGcgZJ0Hw8C6KVelEJ31xLoE21m9+NEgSKCRoFE1Lkma31SyIaynbd # YEb8sOlZynMdm8yPldDwuF54vJiEArjrcDNXe6BobZUiTWSKvv1DJadR1SUCO/Od # 21GgU+hZqu+dKgjKAYdeTIvi9R2rtLYwggdxMIIFWaADAgECAhMzAAAAFcXna54C # m0mZAAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UE # CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z # b2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp # Y2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMy # MjVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQH # EwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNV # BAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0B # AQEFAAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51 # yMo1V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY # 6GB9alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9 # cmmvHaus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN # 7928jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDua # Rr3tpK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74 # kpEeHT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2 # K26oElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5 # TI4CvEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZk # i1ugpoMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9Q # BXpsxREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3Pmri # Lq0CAwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUC # BBYEFCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJl # pxtTNRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9y # eS5odG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUA # YgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU # 1fZWy4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2Ny # bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIw # MTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0w # Ni0yMy5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/yp # b+pcFLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulm # ZzpTTd2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM # 9W0jVOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECW # OKz3+SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4 # FOmRsqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3Uw # xTSwethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPX # fx5bRAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVX # VAmxaQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGC # onsXHRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU # 5nR0W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEG # ahC0HVUzWLOhcGbyoYIDVjCCAj4CAQEwggEBoYHZpIHWMIHTMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT # Tjo2NTFBLTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg # U2VydmljZaIjCgEBMAcGBSsOAwIaAxUAJsAKu48NbR5YRg3WSBQCyjzdkvaggYMw # gYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD # VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQsF # AAIFAOr6y6EwIhgPMjAyNDEyMDQxMjM3NTNaGA8yMDI0MTIwNTEyMzc1M1owdDA6 # BgorBgEEAYRZCgQBMSwwKjAKAgUA6vrLoQIBADAHAgEAAgIi6jAHAgEAAgISjjAK # AgUA6vwdIQIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB # AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBCwUAA4IBAQBYhQMogIeFDUrR # IwRU4YMHsPiZCGElvUgmgHgddtSF1sr0JWg6uW+zVTBBOPVsl+IUHxqIx0c72OTL # MYEhF1tr9geRKbk1x7eJmLMBfbipeAwga/pJq6RPG2vnSilmuq/hrp4hqO2vPBYa # Pmiir3aQJKVLJwIjeM7GN/M9nWH0gIiWev64zrGtXt+AyMp6Q3wsJN3xTSZVrHWp # ZduSgBh32boIPCrc4IPPTBbzWEEaGJ0RoC95zkbUd0MF6vT0T9m/FlfiCK3X5l7/ # zSjz3taMFlfojh46I2eEn4MxHWnP/R3Goj+B5qycd5iyGoXYSIJpCodlpF4ISHe+ # QOPU9cJrMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldh # c2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBD # b3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIw # MTACEzMAAAH1mQmUvPHGUIwAAQAAAfUwDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqG # SIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQxIgQgaN4+bTHX/uso # nFXemvmMPLnzRGj+NH7B4ArSuArov8AwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHk # MIG9BCDB1vLSFwh09ISu4kdEv4/tg9eR1Yk8w5x7j5GThqaPNTCBmDCBgKR+MHwx # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1p # Y3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB9ZkJlLzxxlCMAAEAAAH1 # MCIEIJsAF0QtKp+zWOqWDTyk72cE0rYTpB5dhe7Q+cQ+Uo6GMA0GCSqGSIb3DQEB # CwUABIICAKLJNPaoDQ0Jv/zSGKXlWLCC1UV7uftuoyA1DJ2i4lRFTuu2FvMBf5mt # yFallQ9ycpHB/2DmkyPwPs/mcTlsXB8Dnyg1j/mReqUdCf91j8vXpNFnD+kzUBiM # axAwWxizGZ1JU04cvgh+3sodJzBLrXN6A5OAaIhUFMO9aNirUVLHPvE54BIrqiLn # FsZ/hjfQxAz5s2eZdbJgY4Cmy0anBj3vgxEwqIvL8s4WwmHqDI9+BgpqC36xtymS # 2LhQ1sKbO619H28Uk2lGlazax+QKwoNHIXjVm3TCY2jv8NEHDCZzLGQqbQ0+elNk # Eup1aFtPh7HZPut0emcyeugdsiQoMFbZ8ZHAR7S9LsQroySlFatEmvoLEU5XeFcO # AZGfO8cx7P85QerkpBwpr52cjZ9RGVzPPYWToMKg4e1cWcZVIkyPVG6HB4+lNIfY # /FJz1QRPiGQOQfMVQ4VlfafkMkr2boO7v20sndaIk/brxjULIT3PThD6SToNCa78 # 7v1gHDUuMMJwvdIQN29XWR+S8w+t38jQghYuJCnXS6b36cOpPgZaucGk6koRzqeY # l2W+KX0DL01UNQYa8nno+POJXXPgJMd8j+eSS5XDYnpAyPLIMB3ICgWK52NY4N9w # seBbNirjmRR7EXFT0C3a9Eq+/sY4xKxeJTuO0Mgrp3qDP1S0FgYr # SIG # End signature block |