AksHciNcCompanion

0.8

AksHciNcCompanion.psm1

$script:HostVNetNicName = "SDN_VNET"
$script:clusterGroupName = "clustergroup"
$script:MocConfig = $null
$script:MocGroup = $null

$script:InternetUrl = "https://mcr.microsoft.com"
$script:DnsResolutionName = "mcr.microsoft.com"
$script:InternetResponse = "200"

$script:LinuxUserName = "clouduser"
$script:WindowsUserName = "Administrator"

$script:CloudAgentPort = "65000"
$script:CloudAgentCurlResponse = "curl: (52) Empty reply from server"
$script:CurlToSshResponse = 'curl: (1) Received HTTP/0.9 when not allowed'

$script:VmInboundAccessState = @{}

$script:TestOutputDirectory = "C:\Windows\Tracing\AksHci"

$script:RetryWaitSeconds = 5
$script:retryCount = 3

$script:SDNTestId = "sdn-test-be8cf0e4-e5da-42a7-a759-aaf0eeaee1f4"
$script:MocLocation = "MocLocation"

enum AksHciNcResourceType
{
    NetworkInterface
    LoadBalancer
    VirtualNetwork
}

function Write-TestResults($results)
{
    if (-not (Test-Path $script:TestOutputDirectory )) {
        New-Item -ItemType Directory -Path $script:TestOutputDirectory  -Force
    }
    $output = $results | ConvertTo-Json -Depth 10
    $fileName = "Test-AksHciSdnNetwork-$((get-date).Ticks).Json"
    $file = Join-Path $script:TestOutputDirectory $fileName
    $output | Out-File -FilePath $file -Force
    Write-Host "Results written to file $file" -ForegroundColor Green
}

function Write-Fail {

    param (
        [string]$testName,
        [string]$msg,
        [string]$err
    )
    
    #Write-Status -moduleName $moduleName -Verbose -msg "Test Connection for $scenario, Status Succeeded"
    $out = "FAIL: $testName $msg"

    if (-not [string]::IsNullOrWhiteSpace($err))
    {
        $out += "`nERROR $err"
    }

    Write-Host $out -ForegroundColor Red
}
function Write-Pass {
    param (
        [string]$testName,
        [string]$msg
    ) 

    Write-Host "PASS: $testName $msg" -ForegroundColor Green
}
function Write-Msg {
    param (
        [string]$msg,
        [switch]$Warning
    )
    
    #Write-Status -moduleName $moduleName -Verbose -msg "Test Connection for $scenario, Status Succeeded"
    if ($Warning.IsPresent) {
        Write-Warning $msg
        return
    }

    Write-Host $msg
}

function Get-Result {
        param(
        [string]$scenario,
        [bool]$isSuccess,
        [string]$detailedStatus
    )
    
    # return @{
    # "Scenario" = $scenario;
    # "Passed" = $isSuccess;
    # "DetailedStatus" = $detailedStatus;
    # }

    $result = New-Object -TypeName psobject
    $result | Add-Member -MemberType NoteProperty -Name Scenario -Value $scenario
    $result | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $result | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus

    return $result
}

function Get-VMTestResult {
    param(
        [string]$vmName,
        [string]$testName,
        [string]$sshInfo,
        [bool]$isSuccess,
        [string]$detailedStatus,
        [array]$results
    )

    $vmResult = New-Object -TypeName psobject
    $vmResult | Add-Member -MemberType NoteProperty -Name VMName -Value $vmName
    $vmResult | Add-Member -MemberType NoteProperty -Name TestName -Value $testName
    $vmResult | Add-Member -MemberType NoteProperty -Name SSHInfo -Value $sshInfo
    $vmResult | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $vmResult | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus
    $vmResult | Add-Member -MemberType NoteProperty -Name Results -Value $results

    return $vmResult

    # return @{
    # "VMName" = $vmName;
    # "SSHInfo" = $sshInfo;
    # "Passed" = $isSuccess;
    # "DetailedStatus" = $detailedStatus;
    # "TestName" = $testName
    # "Results" = $results;
    # }
}

function Get-TestResult {
    param(
        [string]$testName,
        [bool]$isSuccess,
        [string]$detailedStatus,
        [array]$testResults
    )

    $result = New-Object -TypeName psobject
    $result | Add-Member -MemberType NoteProperty -Name TestName -Value $testName
    $result | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $result | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus
    $result | Add-Member -MemberType NoteProperty -Name TestResults -Value $testResults
    return $result

    # return @{
    # "TestName" = $testName;
    # "Passed" = $isSuccess;
    # "DetailedStatus" = $detailedStatus;
    # "TestResults" = $testResults;
    # }
}

function Get-ClusterTestResult {
        param(
        [string]$clusterName,
        [bool]$isSuccess,
        [string]$detailedStatus,
        [array]$testResults
    )

    $result = New-Object -TypeName psobject
    $result | Add-Member -MemberType NoteProperty -Name ClusterName -Value $clusterName
    $result | Add-Member -MemberType NoteProperty -Name Passed -Value $isSuccess
    $result | Add-Member -MemberType NoteProperty -Name DetailedStatus -Value $detailedStatus
    $result | Add-Member -MemberType NoteProperty -Name TestResults -Value $testResults
    return $result

    # return @{
    # "ClusterName" = $clusterName;
    # "Passed" = $isSuccess;
    # "TestResults" = $testResults;
    # "DetailedStatus" = $detailedStatus;
    # }
}

function Invoke-WebRequestWithRetries {
    param(
        [System.Collections.IDictionary] $Headers,
        [string] $ContentType,
        [Microsoft.PowerShell.Commands.WebRequestMethod] $Method,
        [System.Uri] $Uri,
        [object] $Body,
        [Switch] $DisableKeepAlive,
        [Switch] $UseBasicParsing,
        [Parameter(mandatory=$false)]
        [bool] $shouldRetry = $true
    )
        
    $params = @{
        'Headers'=$headers;
        'ContentType'=$content;
        'Method'=$method;
        'uri'=$uri;
        'ErrorAction'='Stop';
        }
    
    if($Body -ne $null) 
    {
        $params.Add('Body', $Body)
    }

    if($DisableKeepAlive.IsPresent) 
    {
        $params.Add('DisableKeepAlive', $true)
    }

    if($UseBasicParsing.IsPresent) 
    {
        $params.Add('UseBasicParsing', $true)
    }

    if ($script:DefaultCredParaSet -eq $true)
    {
        $params.Add('UseDefaultCredentials', $true)
    }
    elseif($script:NetworkControllerCred -ne [System.Management.Automation.PSCredential]::Empty)
    {
        $params.Add('Credential', $script:NetworkControllerCred)
    }
        
    $retryIntervalInSeconds = 30
    $maxRetry = 6
    $retryCounter = 0
    
    do {
        try {
            $result = $null
            $result = Invoke-WebRequest @params
            break
        }
        catch {

          
            if($_.Exception.Response.StatusCode.value__ -eq 404)
            {
                #Dont retry on Not Found
                break
            }

            $retryCounter++
            if($retryCounter -le $maxRetry) {
                Start-Sleep -Seconds $retryIntervalInSeconds
            }
            else {
                # last retry still fails, so throw the exception
                throw $_
            }
        }
    } while ($shouldRetry -and ($retryCounter -le $maxRetry)) 
    
    return $result       
}

function Initialize-AksHciNcCompanion
{
    $winFeatures = @("RSAT-NetworkController")
    
    foreach ($feature in $winFeatures)
    {
        Write-Verbose "Check install state of $feature"
        if ((Get-WindowsFeature -Name $feature).InstallState -ne "Installed")
        {
            Write-Verbose "Installing $feature"
            Add-WindowsFeature -Name $feature -IncludeAllSubFeature -IncludeManagementTools
        }
    }

}

function Get-NCResource
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $Uri
    )

    $result = Invoke-WebRequestWithRetries -Uri $Uri -DisableKeepAlive -UseBasicParsing -Method "Get"      

    if(-not $result)
    {
        return $null
    }            

    $toplevel = convertfrom-json $result.Content
    if ($toplevel.value -eq $null)
    {
        $jsonOut = $toplevel
    } 
    else
    {
        $jsonOut = $toplevel.value
    }

    return $jsonOut
}

function Get-NCNic
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $resourceId
    )

    Write-Verbose "Fetching nic $resourceId from NC"

    $mocConfig = Get-MocConfigCache     
    $ncRestEndPoint = $mocConfig["networkControllerFqdnOrIpAddress"]
    $uri = "https://$ncRestEndPoint/networking/v1/networkinterfaces/$resourceId"

    $nic = Get-NCResource -Uri $uri

    if ($nic -eq $null)
    {
        $nics = (Invoke-WebRequest -Uri "https://$ncRestEndPoint/networking/v1/networkinterfaces/" -UseBasicParsing).content | ConvertFrom-Json
        $nic = $nics.value | Where-Object {$_.resourceMetadata.resourceName -eq "$resourceId"}
    }

    if ($nic -eq $null)
    {
        Write-Warning "Nic with resource id $resourceId not found in NC($ncRestEndPoint)"
    }
    return $nic
}

function Get-NCNicInstanceId
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $resourceId
    )

    $nic = Get-NCNic -resourceId $resourceId

    return $nic.instanceId
}

function Get-MocConfigCache
{
    if ($null -eq $script:MocConfig)
    {
        $script:MocConfig = Get-MocConfig
    }

    return $script:MocConfig
}

function Get-MocGroupCache
{
    if ($null -eq $script:MocGroup)
    {
        $script:MocGroup = Get-MocGroup -location $script:MocLocation
    }

    return $script:MocGroup
}

function Clear-MocCache
{
    $script:MocConfig = $null
    $script:MocGroup = $null
}

function GetClusterGroup
{
    return $script:clusterGroupName
}

function ValidateState
{ 

    param (
        [ValidateSet("MOC", "AKSHCI", "KVA")]
        [string]$ModuleName
    )
   
    InitializeModule

    $isInstalled = Test-IsAksHciInstalled -ModuleName $ModuleName

    if (-not $isInstalled)
    {
        Write-Error("UnsupportedConfiguration: $ModuleName is not installed") -ErrorAction Stop
    }

    $mocConfig = Get-MocConfigCache

    if (-not $mocConfig["UseNetworkController"])
    {
        Write-Error("UnsupportedConfiguration: SDN integraton is not present") -ErrorAction Stop
    }
}

function CleanupMoc
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    Remove-MocNetworkInterface -name $nicName -group $(GetClusterGroup) -ErrorAction SilentlyContinue
}

function CleanupHost
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    $mocConfig = Get-MocConfigCache    
    $vswitchName = $mocConfig["vswitchName"]

    $nic = Get-VMNetworkAdapter -ManagementOS -Name $nicName -SwitchName $vswitchName -ErrorAction SilentlyContinue

    if ($nic -eq $null)
    {
        return
    }

    Remove-VMNetworkAdapter -ManagementOS -Name $nicName -SwitchName $vswitchName -ErrorAction Stop
}

function CreateHostVNic
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $name,
        [parameter(Mandatory=$true)]
        [string] $mac,
        [parameter(Mandatory=$true)]
        [string] $ipAddress,
        [parameter(Mandatory=$true)]
        [string] $prefix,
        [parameter(Mandatory=$true)]
        [string] $vswitchName
    )

    Write-Verbose "Creating a new host vnic $name, MAC $mac, IP $ipAddress/$prefix on host $(hostname)"
    $nic = Get-VMNetworkAdapter -ManagementOS -Name $name -SwitchName $vswitchName -ErrorAction SilentlyContinue

    if ($nic -ne $null -and $nic.MacAddress -eq $mac)
    {
        Write-Verbose "Reusing existing VNIC "
    }
    else
    {
        CleanupHost -nicName $name
        Start-Sleep 5

        $nic = Add-VMNetworkAdapter -ManagementOS -Name $name -SwitchName $vswitchName -StaticMacAddress $mac -ErrorAction stop
        Start-Sleep 30
    }

    $ifIndex = (Get-NetAdapter -Name "*$name*").ifIndex

    $nic = Get-NetIPAddress -IPAddress $ipAddress -ErrorAction SilentlyContinue

    if ($nic -ne $null)
    {

        if ($nic.InterfaceIndex -ne $ifIndex)
        {
            Write-Error "Fatal error, Cannot connect host, $ipAddress is already present on another interface on the host" -ErrorAction Stop
        }

        return
    }

    Remove-NetIPAddress -InterfaceIndex $ifIndex -Confirm:$false -ErrorAction SilentlyContinue | Out-Null
    New-NetIPAddress -PrefixLength $prefix -IPAddress $ipAddress -InterfaceIndex $ifIndex -ErrorAction Stop | Out-Null
}


function CreateMocNic
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    $mocConfig = Get-MocConfigCache
    $vnetName = $mocConfig["vnetName"]
    $location = $mocConfig["cloudLocation"]
    $retry = $true

    while($retry)
    {
        try
        {
            New-MocNetworkInterface -name $nicName -virtualNetworkName $vnetName -group $(GetClusterGroup) -ErrorAction Stop
            break
        }
        catch
        {
            $e = $_

            $retry = $e.Exception.Message.Contains("PrivateIPAddressInUse") -eq $true

            if (-not $retry)
            {
                Write-Warning "Failed to create Nic in Moc, Error $_"
            }
            else 
            {
                Write-Verbose "IPAddress conflict, retry nic creation"
                throw
            }
        }
    }
}

function Get-MocNic
{
    
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    Write-Verbose "Fetching nic $nicName from MOC"
    $vnetMocNic = Get-MocNetworkInterface -name $nicName -group $(GetClusterGroup) -ErrorAction SilentlyContinue

    if ($vnetMocNic -eq $null)
    {
        Write-Verbose "Creating new nic $nicName in MOC"
        CreateMocNic -nicName $nicName
    }

    return Get-MocNetworkInterface -name $nicName -group $(GetClusterGroup) -ErrorAction SilentlyContinue
}

function SetPortProfile
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName,
        [parameter(Mandatory=$true)]
        [string] $profileId
    )

    Write-Verbose "Setting port profile for $nicName on $(hostName), ProfileId $profileId"

    $vmNic = Get-VMNetworkAdapter -Name $nicName -ManagementOS -ErrorAction Stop

    $FeatureId = "9940cd46-8b06-43bb-b9d5-93d50381fd56"

    $CurrentFeature = Get-VMSwitchExtensionPortFeature -FeatureId $FeatureId -VMNetworkAdapter $vmNic

    if ($CurrentFeature -eq $null)
    {
        $Feature = Get-VMSystemSwitchExtensionPortFeature -FeatureId $FeatureId
        $Feature.SettingData.ProfileId = "{$profileId}"
        $Feature.SettingData.NetCfgInstanceId = "{56785678-a0e5-4a26-bc9b-c0cba27311a3}"
        $Feature.SettingData.CdnLabelString = "TestCdn"
        $Feature.SettingData.CdnLabelId = 1111
        $Feature.SettingData.ProfileName = "Testprofile"
        $Feature.SettingData.VendorId = "{1FA41B39-B444-4E43-B35A-E1F7985FD548}"
        $Feature.SettingData.VendorName = "NetworkController"
        $Feature.SettingData.ProfileData = 1
        Add-VMSwitchExtensionPortFeature -VMSwitchExtensionFeature $Feature -VMNetworkAdapter $vmNic
    }
    else
    {
        $CurrentFeature.SettingData.ProfileId = "{$profileId}"
        $CurrentFeature.SettingData.ProfileData = 1
        Set-VMSwitchExtensionPortFeature -VMSwitchExtensionFeature $CurrentFeature -VMNetworkAdapter $vmNic
    }

    Write-Verbose "Successfully set port profile for $nicName on $(hostName)"
}

function Remove-PortProfile
{
    param
    (
        [parameter(Mandatory=$true)]
        [string] $nicName
    )

    Write-Verbose "Removing port profile on $nicName"

    $vmNic = Get-VMNetworkAdapter -Name $nicName -ManagementOS -ErrorAction SilentlyContinue

    if ($vmNic  -eq $null)
    {
        return
    }

    $FeatureId = "9940cd46-8b06-43bb-b9d5-93d50381fd56"

    $CurrentFeature = Get-VMSwitchExtensionPortFeature -FeatureId $FeatureId -VMNetworkAdapter $vmNic

    if ($CurrentFeature -eq $null)
    {
        return
    }

    Remove-VMSwitchExtensionPortFeature -VMSwitchExtensionFeature $CurrentFeature -VMNetworkAdapterName $vmNic

}

function Test-IsAksHciVersionHigerOrEqual
{
    param
    (
        [parameter(Mandatory=$true)]
        [int] $Major,
        [parameter(Mandatory=$true)]
        [int] $Minor,
        [parameter(Mandatory=$true)]
        [int] $Build
    )

    $version = Get-AksHciVersion
    $verArray = $version.Split(".")

    if ($verArray[0] -lt $Major) {
        return $false
    }

    if ($verArray[0] -gt $Major) {
        return $true
    }

    if ($verArray[1] -lt $Minor) {
        return $false
    }

    if ($verArray[1] -gt $Minor) {
        return $true
    }

    if ($verArray[2] -lt $Build) {
        return $false
    }

    return $true
}

function Connect-AksHciSdnVnet
{
    <#
    .SYNOPSIS
    This command is now deprecated. Connects physical host to the akshci management cluster virtualnetwork in sdn
    .DESCRIPTION
    This command is now deprecated. Adds a vnic to the physical host attaches the vnic to the akshci management cluster vnet in sdn.
    This provides direct access to the akshci vms private ips from the physical hosts.
    #>
    param
    (
        [parameter(Mandatory=$false)]
        [switch] $Force
    )

    ValidateState -ModuleName "AKSHCI" 

    if (-not $Force.IsPresent) {
        if ((Test-IsAksHciVersionHigerOrEqual -Major 1 -Minor 0 -Build 18)) {
            Write-Warning "Connect-AksHciSdnVnet is no longer required for the current akshci version."
            Write-Warning "Log collection and certificate rotation are supported without running this command."
            Write-Warning "To ssh to the vms use Get-AksHciSdnSshEndpoint to find the ssh ip and port to use."
            Write-Warning "To override this warning and connect to the SDN virtualnetwork, rerun with -Force parameter."
            throw "CMD_DEPRECATED"
        }
    }

    Write-Verbose "Connecting host $(hostname) to the Aks-HCI control plane virtual network"
    $nicName = "$script:HostVNetNicName-$(hostname)"

    $mocNic = Get-MocNic -nicName $nicName 
    $ncNic = Get-NCNic -resourceId $nicName

    if ($ncNic  -eq $null)
    {
        Write-Error "Failed to create a VNIC" -ErrorAction Stop
    }

    $vswitchName = (Get-MocConfigCache)["vswitchName"]
    $prefix = $mocNic.properties.ipConfigurations[0].properties.prefixlength
    $ip = $mocNic.properties.ipConfigurations[0].properties.privateIPAddress
    CreateHostVNic -name $nicName -mac $ncNic.properties.privateMacAddress -vswitchName $vswitchName -ipAddress $ip -prefix $prefix

    SetPortProfile -nicName $nicName -profileId $ncNic.instanceId
}

function Disconnect-AksHciSdnVnet
{
    <#
    .SYNOPSIS
    Removes the physical host from the akshci management cluster virutal network in sdn.
    #>
    param([switch] $RetainResources)   

    ValidateState -ModuleName "MOC"

    Write-Verbose "Disconnecting host $(hostname) from the Aks-HCI control plane virtual network"
    $nicName = "$script:HostVNetNicName-$(hostname)"

    if ($RetainResources.IsPresent)
    {
        Remove-PortProfile -nicName $nicName
        Remove-PortProfile -nicName $script:HostVNetNicName
        return
    }
    

    CleanupHost -nicName $nicName
    CleanupMoc -nicName $nicName

    CleanupHost -nicName $script:HostVNetNicName
    CleanupMoc -nicName $script:HostVNetNicName
}

function Remove-AksHciSdnResources
{
    <#
    .SYNOPSIS
    Remove resources created by akshci which are stale/leaked in networkcontroller.
    .DESCRIPTION
    Leaked/stale resources may be present in NetworkController if there are any failures/bugs during uninstall-akshci or removal of a workload cluster. This cmdlet helps to remove those
    leaked resources from networkcontroller.
    .PARAMETER Leaked
    Remove leaked resources. This option can be used on a working akshci installation when any leaked resources are found in NetworkController.
    .PARAMETER All
    Removes stale resources after the uninstallation of akshci. Cannot be run when the akshci installation in present.
    .PARAMETER NCRestEndPoint
    NC rest endpoint(without https:// prefix). Must be specified when using -All option.
    .EXAMPLE
    # Clean up leaked resources when akshci installation is in a working state.
    Remove-AksHciSdnResources -Leaked
    .EXAMPLE
    #Clean up any stale/leaked resources after the uninstallation of akshci.
    Remove-AksHciSdnResources -All -NCRestEndpoint nc.contoso.com
    #>

    param(
        [Parameter(Mandatory = $true, ParameterSetName = 'All')]
        [switch] $All,
        [Parameter(Mandatory = $true, ParameterSetName = 'Leaked')]
        [switch] $Leaked,
        [Parameter(Mandatory = $true, ParameterSetName = 'All')]
        [string] $NCRestEndPoint,
        [switch] $Force
    )

    Initialize-AksHciNcCompanion
    Clear-MocCache

    $isInstalled = Test-IsAksHciInstalled -ModuleName "MOC"
    
    if ($Leaked.IsPresent -and -not $isInstalled){
        throw "AksHci is not installed, unable to enumerate leaked resources"
    }

    if ($All.IsPresent -and $isInstalled) {
        throw "AksHci is installed. Cannot use -All option. Please retry with Remove-AksHciSdnResources -Leaked"
    }
    
    if ($isInstalled) {
        $NCRestEndPoint = (Get-MocConfigCache)["networkControllerFqdnOrIpAddress"]
    }

    # Check that the $NCRestEndPoint is correct. If it is, the test will populate the uri variable
    $uri = "https://$NCRestEndPoint"
    if (-not (Test-NetworkControllerFqdnOrIpAddress $uri)) {
        throw "Cannot connect to Network Controller $NCRestEndPoint"
    }

    if ($Leaked.IsPresent) {
    
        # Check if MocGroup is configured
        if ((Get-MocGroupCache).Length -eq 0) {
            Write-Warning "No MocGroup configured, cannot enumerate leaked resources"
            return
        }

        Remove-AksHciNcResourcesLeaked -ConnectionUri $uri -Force:$Force

        return
    }

    if ($All.IsPresent) {      
       
        Remove-AksHciNcResourcesAll -ConnectionUri $uri -Force:$Force
    }

}

function Get-MocResourceName
{
    param(
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [AksHciNcResourceType] $Type
    )

    # Get all the moc groups
    $mocGroup = Get-MocGroupCache
    # Return an array of string with the group names
    $mocGroup = $mocGroup | Select-Object -ExpandProperty name

    $mocResources = @()
    switch ($Type)
    {
        ([AksHciNcResourceType]::NetworkInterface)
        {
            foreach ($group in $mocGroup)
            {
                $mocGroupResource = Get-MocNetworkInterface -group $group -ErrorAction Stop
                # Extract only the name property for each resource in the group and save it
                $mocGroupResource = $mocGroupResource | Select-Object -ExpandProperty name
                $mocResources += $mocGroupResource
            }
        }
        ([AksHciNcResourceType]::LoadBalancer)
        {
            foreach ($group in $mocGroup)
            {
                $mocGroupResource = Get-MocLoadBalancer -group $group -ErrorAction Stop
                # Extract only the name property for each resource in the group and save it
                $mocGroupResource = $mocGroupResource | Select-Object -ExpandProperty name
                $mocResources += $mocGroupResource
            }
        }
        ([AksHciNcResourceType]::VirtualNetwork)
        {
            foreach ($group in $mocGroup)
            {
                $mocGroupResource = Get-MocVirtualNetwork -group $group -ErrorAction Stop
                # Extract only the name property for each resource in the group and save it
                $mocGroupResource = $mocGroupResource | Select-Object -ExpandProperty name
                $mocResources += $mocGroupResource
            }
        }
        Default
        {
            $mocResources = @()
        }
    }
    
    return $mocResources
}

function Get-AksHciNcResourceNameAndId
{
    param(
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [AksHciNcResourceType] $Type,

        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri
    )

    # Query all Nc resources from the rest api
    switch ($Type) {
        ([AksHciNcResourceType]::NetworkInterface)
        {
            $ncResponse = Get-NetworkControllerNetworkInterface -ConnectionUri $ConnectionUri
        }
        ([AksHciNcResourceType]::LoadBalancer)
        {
            $ncResponse = Get-NetworkControllerLoadBalancer -ConnectionUri $ConnectionUri
        }
        ([AksHciNcResourceType]::VirtualNetwork)
        {
            $ncResponse = Get-NetworkControllerVirtualNetwork -ConnectionUri $ConnectionUri
        }
        Default
        {
            $ncResponse = @()
        }
    }

    # Remove entries that don't have ResourceMetadata
    $ncResponse = $ncResponse | Where-Object -Property ResourceMetadata -ne nil
    # Re-format object to extract ResourceId, ResourceMetadata.Client, ResourceMetadata.ResourceName
    $ncResponse = $ncResponse | Select-Object ResourceId, @{N="Client";E={$_.ResourceMetadata.Client}}, @{N="ResourceName";E={$_.ResourceMetadata.ResourceName}}
    # Only select the AksHci Client Objects
    $ncResponse = $ncResponse | Where-Object -Property Client -eq "AksHci"

    return $ncResponse
}

function Remove-NCResource
{
    param(
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [AksHciNcResourceType] $Type,

        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri,
        
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ResourceId
    )

    switch ($Type) {
        ([AksHciNcResourceType]::NetworkInterface)
        {
            Remove-NetworkControllerNetworkInterface -ConnectionUri $ConnectionUri -ResourceId $ResourceId -ErrorAction SilentlyContinue -Force
        }
        ([AksHciNcResourceType]::LoadBalancer)
        {
            Remove-NetworkControllerLoadBalancer -ConnectionUri $ConnectionUri -ResourceId $ResourceId -ErrorAction SilentlyContinue -Force
        }
        ([AksHciNcResourceType]::VirtualNetwork)
        {
            Remove-NetworkControllerVirtualNetwork -ConnectionUri $ConnectionUri -ResourceId $ResourceId -ErrorAction SilentlyContinue -Force
        }
        Default
        {
            Write-Error "Network Controller Resource type unknown, cannot delete"
        }
    }
}

function Remove-AksHciNcResourcesLeaked
{
    param(
        [switch] $Force,
        [Parameter(Mandatory = $true)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri
    )

    # go over each resource type and compare Moc to NC
    foreach ($resType in [AksHciNcResourceType].GetEnumValues())
    {
        $mocResources = Get-MocResourceName -Type $resType
        if ($mocResources.Length -eq 0)
        {
            Write-Error "No $resType resources found in Moc, cannot compare with NetworkController"
            continue
        }

        $ncResources = Get-AksHciNcResourceNameAndId -ConnectionUri $ConnectionUri -Type $resType
        if ($ncResources.Length -eq 0)
        {
            Write-Error "No $resType resources found in NetworkController, nothing to remove"
            continue
        }

        # Compare the objects
        $diff = Compare-Object -ReferenceObject $mocResources -DifferenceObject ($ncResources | Select-Object -ExpandProperty ResourceName)

        # NetworkController resources not found in Moc
        $ncOnly = $diff | Where-Object -Property SideIndicator -eq "=>" | Select-Object -ExpandProperty InputObject
        if ($ncOnly.Length -gt 0)
        {
            $ncOnlyResourceName = $ncOnly
            # Expand ncOnly from ResourceName to include ResourceId. This may be a slow operation, for each resource in $ncOnly,
            # it will search through $ncNetworkInterfaces to find the full object
            $ncOnly = $ncOnly | ForEach-Object { $ncResources | Where-Object -Property ResourceName -eq $_ }

            # Prompt user to delete
            $decision = 1
            if (!$Force.IsPresent)
            {
                $title = "Delete NetworkController$($resType) Resources"
                $question = "Are you sure you want to delete the following resources:`n$($ncOnlyResourceName -join "`n")"
                $choices = @('&Yes', '&No', '&Suspend')
                $decision = $Host.UI.PromptForChoice($title, $question, $choices, 1)
            }

            # Delete the Network Controller resources
            if ($decision -eq 0 -or $Force.IsPresent)
            {
                foreach ($n in $ncOnly)
                {
                    Remove-NCResource -ConnectionUri $ConnectionUri -Type $resType -ResourceId $n.ResourceId
                }
            }
            # Suspend, exit immediatly
            elseif ($decision -eq 2)
            {
                return
            }
        }
        else
        {
            Write-Host "No Leaked NetworkController$($resType) resources found"
        }
    }
}

function Remove-AksHciNcResourcesAll
{
    param(
        [switch] $Force,
        [Parameter(Mandatory)]
        [ValidateNotNullOrEmpty()]
        [string] $ConnectionUri
    )

    foreach ($resType in [AksHciNcResourceType].GetEnumValues())
    {
        $ncResources = Get-AksHciNcResourceNameAndId -ConnectionUri $ConnectionUri -Type $resType

        # Prompt user to delete
        $decision = 1
        if (!$Force.IsPresent -and $ncResources.Length -ne 0)
        {
            $title = "Delete NetworkController$($resType) Resources"
            $question = "Are you sure you want to delete the following resources:`n$(($ncResources | Select-Object -ExpandProperty ResourceName) -join "`n")"
            $choices = @('&Yes', '&No', '&Suspend')
            $decision = $Host.UI.PromptForChoice($title, $question, $choices, 1)
        }

        # Delete the Network Controller resources
        if ($decision -eq 0 -or $Force.IsPresent)
        {
            foreach ($n in $ncResources)
            {
                Remove-NCResource -ConnectionUri $ConnectionUri -Type $resType -ResourceId $n.ResourceId
            }
        }
        # Suspend, exit immediatly
        elseif ($decision -eq 2)
        {
            return
        }
    }
}

function Test-AksHciSdnConfig
{
    # By default, we will use the values in Moc to test NC
    # If 1 parameter is manually set, all parameters must be supplied and moc will not be used
    [CmdletBinding(DefaultParameterSetName = 'Moc')]
    param(
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [string] $networkControllerFqdnOrIpAddress,
        
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [string] $networkControllerLbSubnetRef,
        
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [string] $networkControllerLnetRef,
        
        [Parameter(Mandatory = $true, ParameterSetName = 'Manual')]
        [ValidateNotNullOrEmpty()]
        [VipPoolSettings] $vipPool
    )

    # ----- Setup -----
    $useMoc = $PSCmdlet.ParameterSetName -eq "Moc"
    $ret = $true
    Clear-MocCache

    # Read all values from Moc
    if ($useMoc)
    {
        $mocConfig = Get-MocConfigCache

        # Check that Moc has been initialized
        if ($mocConfig["installState"] -eq [InstallState]::NotInstalled)
        {
            Write-Error "Moc is not in an installed state. Cannot read NetworkController config in Moc"
            $ret = $false
            # Cannot continue any more tests without Moc
            return $ret
        }

        # Get networkControllerFqdnOrIpAddress
        $networkControllerFqdnOrIpAddress = $mocConfig["networkControllerFqdnOrIpAddress"]
        # Get networkControllerLbSubnetRef
        $networkControllerLbSubnetRef = $mocConfig["networkControllerLbSubnetRef"]
        # Get networkControllerLnetRef
        $networkControllerLnetRef = $mocConfig["networkControllerLnetRef"]
        # Get vipPool
        $vipPoolDefaultName = $mocConfig["defaultvippoolname"]
        $vipPoolDefault = Get-MocVipPool -location $script:MocLocation -name $vipPoolDefaultName
        $vipPool = [VipPoolSettings]::new($vipPoolDefault.name, $vipPoolDefault.properties.startIp, $vipPoolDefault.properties.endIp)
    }

    # ----- Tests -----
    # Check networkControllerFqdnOrIpAddress
    Write-Verbose "Check networkControllerFqdnOrIpAddress"
    $uri = "https://$networkControllerFqdnOrIpAddress"
    $result = Test-NetworkControllerFqdnOrIpAddress $uri
    if (-not $result)
    {
        Write-Error "Cannot connect to NetworkController with the networkControllerFqdnOrIpAddress $networkControllerFqdnOrIpAddress"
        $ret = $false
        # Cannot test any more without the Network Controller Uri
        return $ret
    }

    # Check networkControllerLnetRef
    Write-Verbose "Check networkControllerLnetRef"
    $result = Test-NetworkControllerReference $uri $networkControllerLnetRef
    if (-not $result)
    {
        Write-Error "networkControllerLnetRef ($networkControllerLnetRef) cannot be found in the Network Controller"
        $ret = $false
    }

    # Check networkControllerLbSubnetRef
    Write-Verbose "Check networkControllerLbSubnetRef"
    $lbSubnetWebResult = $null
    $result = Test-NetworkControllerReference $uri $networkControllerLbSubnetRef ([ref]$lbSubnetWebResult)
    if (-not $result)
    {
        Write-Error "networkControllerLbSubnetRef ($networkControllerLbSubnetRef) cannot be found in the Network Controller"
        $ret = $false
    }

    # Check the vipPool -- uses previous networkControllerLbSubnetRef result
    Write-Verbose "Check vippool setting"
    if ($null -ne $lbSubnetWebResult)
    {
        $result = Test-NetworkControllerDefaultVipPool ($lbSubnetWebResult.Content) $vipPool
        if (-not $result)
        {
            Write-Error "AksHci Vippool ip range is not within the NetworkController Vippool range"
            $ret = $false
        }
    }
    else
    {
        Write-Warning "Skipping Vippool Test, lb Subnet Ref not found in Network Controller"
    }

    return $ret
}

function Test-NetworkControllerFqdnOrIpAddress([string]$ConnectionUri)
{
    try {
        $result = Invoke-WebRequest $ConnectionUri/networking/discovery -UseBasicParsing
        if ($result.StatusCode -eq 200) {
            return ($result.Content | ConvertFrom-Json).resourceId -eq "discovery"
        }
    }
    catch {
    }

    return $false

}

function Test-NetworkControllerReference([string]$ConnectionUri, [string]$subRef, [ref]$webResult)
{
    $uri = "$ConnectionUri/networking/v1$subRef"
    $result = Invoke-WebRequest -UseBasicParsing -Uri $uri
    if ($null -eq $result -or $result.StatusCode -ne 200)
    {
        return $false
    }

    # set the webResult if it was passed
    if ($null -ne $webResult)
    {
        $webResult.Value = $result
    }
    return $true
}

function Test-NetworkControllerDefaultVipPool([string]$lbSubnetContent, [VipPoolSettings]$mocVipPool)
{
    $ncVippool = (ConvertFrom-Json -InputObject $lbSubnetContent).Properties.ipPools.Properties
    if ([string]::IsNullOrWhiteSpace($ncVippool))
    {
        return $false
    }

    # check that the IP addresses are in the following order
    # NCVippoolStart -> MocVippoolStart -> MocVippoolEnd -> NCVippoolEnd
    return (
        [AKSHCI.IPUtilities]::ValidateRange($ncVippool.startIpAddress, $vipPool.VipPoolStart) -and
        [AKSHCI.IPUtilities]::ValidateRange($vipPool.VipPoolStart, $vipPool.VipPoolEnd) -and
        [AKSHCI.IPUtilities]::ValidateRange($vipPool.VipPoolEnd, $ncVippool.endIpAddress)
    )
}
function Get-FirstNcVmName()
{
    $networkControllerVMName = $null
    $isMultiNode = $false

    try {
        $failoverCluster = Get-FailoverCluster -ErrorAction SilentlyContinue
        $isMultiNode = $failoverCluster -ne $null
    }
    catch {}

    try 
    {
        
        if ($isMultiNode)
        {
            Get-ClusterNode -ErrorAction Stop | ForEach-Object {
                
                if ([string]::IsNullOrWhiteSpace($networkControllerVMName))
                {
                    Write-Msg "Checking $_"
                    $networkControllerVMName = Invoke-Command -ComputerName $_ -ScriptBlock {
                        (Get-VM | Get-VMNetworkAdapter | where-object { $_.IPAddresses.Contains($((Get-NetTCPConnection -RemotePort 8571).RemoteAddress)) }).VMName
                    }
                }
            }
        }
        else 
        {
            $networkControllerVMName = (Get-VM | Get-VMNetworkAdapter | where-object { $_.IPAddresses.Contains($((Get-NetTCPConnection -RemotePort 8571).RemoteAddress)) }).VMName            
        }        
    }
    catch 
    {
        Write-Error("Unable to detect NC VM name")
    }

    return $networkControllerVMName
}

function Get-NatRuleMap
{
    <#
    .DESCRIPTION
        Fetches the ssh nat lb rules for the group and caches it.
        Returns a map of all ssh nat rules for the given group
    .PARAMETER groupName
        Group name
    #>

     param (
        [Parameter()]
        [String] $groupName
    )

    if ($script:natRuleCache -ne $null)
    {
        $ruleMap = $script:natRuleCache[$groupName]
        if ($ruleMap -ne $null) 
        {
            return $ruleMap
        }
    }
    else 
    {
        $script:natRuleCache = @{}
    }

    $ruleMap = @{}
    $lbs = Get-MocLoadBalancer -group $groupName

    if ($lbs -eq $null) 
    {
        $script:natRuleCache[$groupName] = $ruleMap
        return $ruleMap
    }
    
    
    $inboundLb = $lbs | where-object { $_.properties.inboundNatRules -ne $null }
    foreach($lb in $inboundLb) 
    {
        $sshNatRules = $lb.properties.inboundNatRules | `
            Where-Object { $_.Properties.backendPort -eq 22 }
        if($sshNatRules -ne $null)
        {
            $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress                
            foreach($natRule in $sshNatRules)
            {
                $port = $natRule.properties.frontendPort
                $sshData = @{}
                $sshData["ip"] = $feIP
                $sshData["port"] = $port
                $ruleMap[$natRule.Name] = $sshData
            }
        }
    }

    $natRuleCache[$groupName] = $ruleMap
    return $ruleMap
}
function InitializeModule
{
    $script:natRuleCache = $null
    Clear-MocCache
    $script:retryCount = 3
}

function Get-AksHciSdnSshEndpoint
{
    <#
    .SYNOPSIS
        Fetches the ssh ip and port for the given VM.
    .DESCRIPTION
        When SDN integration is configured, ssh access to the vm is via the inbound nat rule. This cmdlet fetches the frontend nat ip address
        and port from the loadbalancer.
    .PARAMETER VMName
        Name of the VM as shown in hyper-v for which the ssh configuration should be fetched. If VMName is omitted, ssh information for all the vms in the given cluster is fetched.
    .PARAMETER ClusterName
        Name of the targetcluster where the vm belongs
    .PARAMETER Management
        Denotes that the information should be fetched for the management cluster.
    .EXAMPLE
    # Fetch ssh endpoint of the management cluster VMs
    Get-AksHciSdnSshEndpoint -Management
    .EXAMPLE
    # Fetch ssh endpoint of the given VM from the given target cluster
    Get-AksHciSdnSshEndpoint -VMName testcluster-control-plansfdsfe-c784t-d0e6fasdffe2 -clustername testcluster
    .EXAMPLE
    # Fetch ssh endpoints for all the VM from the given target cluster
    Get-AksHciSdnSshEndpoint -clustername testcluster
    #>
    
     param (
        [Parameter(Mandatory = $false)]
        [String] $VMName,
        [Parameter(Mandatory = $true, ParameterSetName = 'Cluster')]
        [String] $ClusterName,
        [Parameter(Mandatory = $true, ParameterSetName = 'Management')]
        [Switch] $Management,
        [Parameter(Mandatory = $true, ParameterSetName = 'Test')]
        [Switch] $Test
    )

    ValidateState -ModuleName "MOC"

    if (-not $(Test-IsAksHciVersionHigerOrEqual -Major 1 -Minor 0 -Build 18)) {
        Write-Warning "Get-AksHciSdnSshEndpoint is only supported with AksHci version 1.0.18 or higher"
        throw "NOT_SUPPORTED"
    }    

    $groupName = "clustergroup"
    if ($Test.IsPresent) {
        $groupName = $script:SDNTestId
    }
    elseif (-not $Management.IsPresent) {
        $groupName += "-$ClusterName"
    }

    $group = Get-MocGroup -name $groupName -Location $script:MocLocation -ErrorAction SilentlyContinue
    if ($group -eq $null) {
        Write-Warning "No resources found"
        return
    }

    if (-not [String]::IsNullOrWhiteSpace($VMName)) {
        $vmAr = $VMName.Split("-")
        $mocVmName = ($vmAr | select -First ($vmAr.count - 1)) -join "-"
        $mocVms = Get-MocVirtualMachine -Name $mocVmName -group $groupName
    } else {
        $mocVms = Get-MocVirtualMachine -group $groupName
    }    

    $eps = @()
    $mocVms | Foreach-Object {
        $nicName = $_.virtualmachineproperties.networkprofile.networkinterfaces[0].id
        $ep = Get-NatEndpoint -nicName $nicName -groupName $groupName

        $result = New-Object -TypeName psobject
        $result | Add-Member -MemberType NoteProperty -Name VMName -Value $_.Name
        $result | Add-Member -MemberType NoteProperty -Name PrivateIP -Value $ep.privateip
        $result | Add-Member -MemberType NoteProperty -Name SSH-IP -Value $ep.ip
        $result | Add-Member -MemberType NoteProperty -Name SSH-Port -Value $ep.port

        $eps += $result
    }

    return $eps
   
}
function Get-NatEndpoint
{
    <#
    .DESCRIPTION
        Fetches the ssh frontend nat ip and port for the given nic. When SDN integration is configured,
        ssh access to the node is via the inbound nat rule
 
    .PARAMETER nicName
        Networkinterface name
    .PARAMETER groupName
        Group name
    #>
     param (
        [Parameter()]
        [String] $nicName,

        [Parameter()]
        [String] $groupName
    )
   
    $nic = Get-MocNetworkInterface -group $groupName -name $nicName
    $natRuleName = $nic.properties.ipConfigurations[0].properties.loadBalancerInboundNatRules[0].name

    if ($natRuleName -eq $null -or $natRuleName[0] -eq $null)
    {
        return $null
    }

    $ruleMap = Get-NatRuleMap -groupName $groupName
    $ep = $ruleMap[$natRuleName]
    $ep["privateip"] = $nic.properties.ipConfigurations[0].properties.privateIPAddress
    return $ep
}

function Get-AksHciSdnLogs
{

    <#
    .DESCRIPTION
        Collects SDN logs.
    .PARAMETER networkControllerVMName
        NetworkController VM name. If there are more than one VM, use any one of the VM name.
    #>

     param (
        [Parameter()]
        [String] $NetworkControllerVMName,
        [Parameter()]
        [switch] $SkipAksHciLogs,
        [Parameter()]
        [switch] $VirtualMachineLogs,
        [Parameter()]
        [switch] $AgentLogs,
        [Parameter()]
        [switch] $EventLogs,
        [Parameter()]
        [switch] $KvaLogs,
        [Parameter()]
        [switch] $DownloadSdkLogs,
        [Parameter()]
        [switch] $BillingRecords
    )
   
    if ([string]::IsNullOrWhiteSpace($NetworkControllerVMName))
    {
        $NetworkControllerVMName = Get-FirstNcVmName
    }

    if ([string]::IsNullOrWhiteSpace($NetworkControllerVMName))
    {
        Write-Error("Unable to detect NC VM name, please use -networkControllerVMName and provide one of the NC vm name") -ErrorAction Stop
    }

    if(-not $skipAksHciLogs.IsPresent) {
        $zipName = Get-AksHciLogs -EventLogs:$EventLogs.IsPresent -KvaLogs:$KvaLogs.IsPresent -DownloadSdkLogs:$DownloadSdkLogs.IsPresent `
            -BillingRecords:$BillingRecords.IsPresent -VirtualMachineLogs:$VirtualMachineLogs.IsPresent -AgentLogs:$AgentLogs.IsPresent
    }

    $environmentDetails = Get-SdnInfrastructureInfo -NetworkController $NetworkControllerVMName
    Install-SdnDiagnostics -ComputerName ($environmentDetails.FabricNodes | where {$_ -ne $null})
    Start-SdnDataCollection -NetworkController $environmentDetails.NetworkController[0] -Role Gateway,NetworkController,Server,LoadBalancerMux -IncludeLogs -IncludeNetView
    Write-Host "AKS-HCI Log location $zipName" -ForegroundColor Green

}

function Test-IsAksHciInstalled
{
    param (
        [ValidateSet("MOC", "AKSHCI", "KVA")]
        [string]$ModuleName
    )

    try {
        if ($ModuleName -eq "MOC") {
            $config = Get-MocConfig -ErrorAction Stop
        } elseif ($ModuleName -eq "AKSHCI") {
            $config = (Get-AksHciConfig -ErrorAction Stop).AksHci
        } elseif ($ModuleName -eq "KVA") {
            $config = Get-KvaConfig -ErrorAction Stop
        }

    } catch {
        
        $isNotInstalled = $_.InvocationInfo.Line.Contains("generic_cannot_deploy")

        if ($isNotInstalled) {
            return $false
        }

        if ($_.FullyQualifiedErrorId -eq "CommandNotFoundException") {
            return $false
        }

        throw
    }

    $currentState = $config.installState

    if (-not $currentState)
    {
        return $false
    }

    switch($currentState)
    {
        $([InstallState]::NotInstalled) { return $false }
        $([InstallState]::InstallFailed) { return $false }
    }

    return $true
}

function Write-ConnectionResult([string] $scenario, $result)
{
    $status = $true
    $result | Foreach-Object {
        Write-Msg "Test Connection Result - IP $($_.IP), Port $($_.Port), Status $($_.Status)"
        if ($($_.Status) -eq "Failed") {
            $status = $false
        }
    }  

    if ($status) {
        Write-Msg "Test Connection for $scenario, Status Succeeded"
    }

    return $status
}
function Test-Connectivity([string]$ip, [string]$port)
{
    $ProgressPreference = "SilentlyContinue"
    $retry = $script:retryCount
    try {
        while($retry -ge 0) {

                $result = Test-NetConnection -ComputerName $ip -Port $port -ErrorAction SilentlyContinue
                Write-Msg -msg "$($MyInvocation.MyCommand) IP [$ip] Port [$port]"
                if ($result -eq $null -or $result.TcpTestSucceeded -ne $true)
                {
                    $retry -= 1
                    if ($retry -ge 0) {
                        Write-Msg -msg "$($MyInvocation.MyCommand) IP [$ip] Port [$port] Failed, retrying"
                        Start-Sleep $script:RetryWaitSeconds
                        continue
                    } 

                    return $false
                } else {
                    return $true
                }
            }
        
        }
    finally {
        $ProgressPreference = "Continue"
    }

    return $false
}
function Test-AllLoadbalancingRules([string]$lbName, [string]$group, [bool]$includeSsh)
{
    $testResult = Get-TestResult -testName "LBConnectivity" -testResults @()

    try {
        $lb = Get-MocLoadBalancer -name "$lbName" -group $group
        $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress
        $fePorts = @()
        $lb.properties.loadBalancingRules | foreach-object { 
            $fePorts += $_.Properties.FrontendPort
        }

        if ($includeSsh)
        {
            $fePorts += "22"
        }
        $overallStatus = $true
        $fePorts | foreach-object {            
            $status = Test-Connectivity -ip $feIP -port $_   
            $detailedStatus = "LB [$lbName] feIP [$feIP] fePort [$_] status "
            if ($status) {
                $detailedStatus += "[Success]"
            } else {
                $detailedStatus += "[Failed]"
            }

            $result = Get-Result -scenario "LBConnectivity" -isSuccess $status -detailedStatus $detailedStatus
            $testResult.TestResults += $result
            $overallStatus =  $overallStatus -and $result.Passed
        }
    } catch {
        $testResult.DetailedStatus = Convert-Error $_
        $overallStatus = $false
    }

    $testResult.Passed = $overallStatus
    return $testResult
}

function Test-ManagementControlPlaneConnectivity()
{
    $testResult = Get-TestResult -testName "ManagementControlPlaneConnectivity" -testResults @()
    try {        
        Write-Msg -msg "$($MyInvocation.MyCommand)"
        $kvaName = (Get-KvaConfig).kvaname
        $lbName = "$kvaName-load-balancer"
        $lbTestResult = Test-AllLoadbalancingRules -lbName $lbName -group "clustergroup" -includeSsh $false
        $status = $lbTestResult.Passed
        $testResult.TestResults += $lbTestResult
        $testResult.Passed = $lbTestResult.Passed

        if (-not $status) {
            throw "Management controlplane connectivity test failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand)"
    }
    catch {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -err $_
        $testResult.Passed = $false
        $testResult.DetailedStatus = Convert-Error $_
    }

    return $testResult
}


function Test-TargetControlPlaneConnectivity()
{
    param (
        [string]$clusterName
    )

    $testName = "$($MyInvocation.MyCommand)"
    $testResult = Get-TestResult -testName $testName -testResults @() -isSuccess $true
    try {
        Write-Msg -msg "$($MyInvocation.MyCommand)"
        $status = $true
        $clusters = Get-AksHciCluster -ErrorAction Stop -Name $clusterName
        $clusters | ForEach-Object {
            $name = $($_.Name)
            Write-Msg -msg "$($MyInvocation.MyCommand) Cluster [$name]"
            $lbName = "$($_.Name)-load-balancer"
            $lbTestResult = Test-AllLoadbalancingRules -lbName $lbName -group "clustergroup-$name" -includeSsh $false
            $lbTestResult.TestName = "$testName-[$name]"
            $status = $status -and $lbTestResult.Passed
            $testResult.TestResults += $lbTestResult
            $testResult.Passed =  $testResult.Passed -and $lbTestResult.Passed
        }

        if (-not $status) {
            throw "TargetControlPlaneConnectivity test failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand)"
    } catch {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -err $_
        $testResult.Passed = $false
        $testResult.DetailedStatus = Convert-Error $_
    }

    return $testResult
}

function Reset-Key
{
     param (
        [string]$sshIP,
        [string]$sshPort
     )

    if ($sshPort -eq "22") {
        $keyGenAddress = $sshIP    
    } else {
        $keyGenAddress = "[$sshIP]`:$sshPort"    
    }

    (& ssh-keygen -R $keyGenAddress) 2>&1>$null
}

function Run-SshCommand
{
       param (
        [string]$sshIP,
        [string]$sshPort,
        [string]$command,
        [bool]$windows = $false
    )
    
    Reset-Key -sshIP $sshIP -sshPort $sshPort 
    $sshKey = (Get-MocConfigCache).sshPrivateKey
    if ($windows) {
        $bytes = [System.Text.Encoding]::Unicode.GetBytes($command)
        $encodedCommand = [Convert]::ToBase64String($bytes)
        $response = (& ssh -oBatchMode=yes -i $sshKey -o LogLevel=ERROR -o StrictHostKeyChecking=no "$script:WindowsUserName@$sshIP" -p $sshPort powershell.exe -NonInteractive -encodedCommand $encodedCommand 2>$null)

    } else {
        $response = (& ssh -oBatchMode=yes -i $sshKey -o LogLevel=ERROR -o StrictHostKeyChecking=no "$script:LinuxUserName@$sshIP" -p $sshPort $command)
    }
    
    return $response
}

function Test-CloudAgentConnectivity
{
    param (
        [string]$sshIP,
        [string]$sshPort,
        [bool]$windows = $false
    )
    
    Reset-Key -sshIP $sshIP -sshPort $sshPort

    $sshKey = (Get-MocConfigCache).sshPrivateKey
    $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
    $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
    $status = $false
    $retry = $script:retryCount
    while($retry -ge 0) {
        if ($windows) {
            $command ="
            invoke-command -scriptblock {
                try {
                        (Test-NetConnection $cloudAgentAddress -port $script:CloudAgentPort -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).TcpTestSucceeded
                    }
                    catch {
                    }
                }"       
            $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
            $status = ($response -ieq "True")
        } else {
            $cloudAgentEndpoint = $cloudAgentAddress + ":" + $script:CloudAgentPort
            $command = "curl --connect-timeout 5 $cloudAgentEndpoint --stderr - `| grep curl:"
            $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
            $status = $response -ieq $script:CloudAgentCurlResponse
        }  

        if ($status) {
            $msg = "Cloudagent connectivity to [$cloudAgentAddress] succeeded"
            break
        } else {
            $msg = "Cloudagent connectivity to [$cloudAgentAddress] failed, response $response"
            Write-Msg -msg "$($MyInvocation.MyCommand) $msg, retrying"
            $retry -= 1
            if ($retry -ge 0) {
                Start-Sleep $script:RetryWaitSeconds
            }
        }
    }
    
    $result = Get-Result -scenario "cloudagent" -isSuccess $status -detailedStatus $msg
    return $result
}

function Test-OutboundConnectivity
{
    param (
        [string]$scenario,
        [string]$sshIP,
        [string]$sshPort,
        [string]$outboundIP,
        [string]$outboundPort,
        [string]$expectedLinuxResponse,
        [bool]$windows = $false
    )
    
    Reset-Key -sshIP $sshIP -sshPort $sshPort

    $sshKey = (Get-MocConfigCache).sshPrivateKey
    $status = $false
    $url = $outboundIP + ":" + $outboundPort
    $retry = $script:retryCount
    while($retry -ge 0) {
        if ($windows) {
            $command ="
            invoke-command -scriptblock {
                try {
                        (Test-NetConnection $outboundIP -port $outboundPort -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).TcpTestSucceeded
                    }
                    catch {
                    }
                }"       
            $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
            $status = ($response -ieq "True")
        } else {
            $command = "curl --connect-timeout 5 $url --stderr - `| grep curl:"
            $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
            $status = $response -ieq $expectedLinuxResponse
        }  

        if ($status) {
            $msg = "[$scenario] Outbound connectivity to [$url] succeeded"
            break
        } else {
            $msg = "[$scenario] Outbound connectivity to [$url] failed, response $response"
            Write-Msg -msg "$($MyInvocation.MyCommand) $msg, retrying"
            $retry -= 1
            if ($retry -ge 0) {
                Start-Sleep $script:RetryWaitSeconds
            }
        }
    }
    
    $result = Get-Result -scenario $scenario -isSuccess $status -detailedStatus $msg
    return $result
}

function Test-InternetConnectivity
{
    param (
        [string]$sshIP,
        [string]$sshPort,
        [bool]$windows = $false
    )
    Reset-Key -sshIP $sshIP -sshPort $sshPort
    $sshKey = (Get-MocConfigCache).sshPrivateKey

    $status = $false
    $response = $null
    $retry = $script:retryCount
    while($retry -ge 0) {
        if ($windows) {
            $command ="
            invoke-command -scriptblock {
                try {
                        (Invoke-WebRequest -UseBasicParsing -Uri $($script:InternetUrl) -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).StatusCode
                    }
                    catch {
                        
                    }
                }"
        } else {
            #$command = "curl --connect-timeout 5 $($script:InternetUrl) -s -f"
            $command = "curl -o /dev/null -s -w '%{http_code}' $($script:InternetUrl)"
        }

        $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
        $status = $response -ieq $script:InternetResponse

        if ($status) {
            $msg = "Internet connectivity to [$script:InternetUrl] succeeded"
            break
        } else {
            $msg = "Internet connectivity to [$script:InternetUrl] failed, response $response"
            Write-Msg -msg "$($MyInvocation.MyCommand) $msg, retrying"
            $retry -= 1
            if ($retry -ge 0) {
                Start-Sleep $script:RetryWaitSeconds
            }
        }
    }
    
    $result = Get-Result -scenario "internet" -isSuccess $status -detailedStatus $msg
    return $result 
}

function Test-DnsResolution
{
        param (
        [string]$sshIP,
        [string]$sshPort,
        [bool]$windows = $false
    )

    $retry = $script:retryCount
    while($retry -ge 0) {
        Write-Verbose "$($MyInvocation.MyCommand) Query [$script:DnsResolutionName]"
        if ($windows) {
            $command ="
            invoke-command -scriptblock {
                try {
                        (Resolve-DnsName -Type A -DnsOnly $script:DnsResolutionName -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).IPAddress
                    }
                    catch {
                    }
                }"
        } else {
            $command = "dig +short A $script:DnsResolutionName | grep -v ';' "
        }

        $response = Run-SshCommand -sshIP $sshIP -sshPort $sshPort -command $command -windows $windows
        Write-Verbose "$($MyInvocation.MyCommand) Response [$response]"
        $status = (-not [String]::IsNullOrWhiteSpace($response)) -and $response.Count -gt 0 

        if ($status) {
            $msg = "DNS resolution to [$script:DnsResolutionName] succeeded"
            break
        } else {
            $msg = "DNS resolution to [$script:DnsResolutionName] failed, response $response"
            Write-Msg -msg "$($MyInvocation.MyCommand) $msg, retrying"
            $retry -= 1
            if ($retry -ge 0) {
                Start-Sleep $script:RetryWaitSeconds
            }
        }
    }
    
    $result = Get-Result -scenario "dns" -isSuccess $status -detailedStatus $msg
    return $result 
}

function Test-ManagementClusterConnectivity
{
    param (
        [ValidateSet("cloudagent", "dns", "internet", "inbound", "outboundvip")]
        [string]$scenario
    )

    try {
        
        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario]"
        $kvaName = (Get-KvaConfig).kvaname
        $lbName = "$kvaName-load-balancer"
        $lb = Get-MocLoadBalancer -name "$lbName" -group $(GetClusterGroup)
        $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress

        if ($scenario -ne "inbound" -and $script:VmInboundAccessState["managementControlPlane"] -eq $false) {
            Write-Msg -msg "$($MyInvocation.MyCommand) [SKIP NO INBOUND ACCESS] Scenario [$scenario]" -Warning
            throw "Test skipped, no inbound access to the management control plane vm"
        }

        switch ($scenario) {
            "inbound" {
                $status = $(Test-Connectivity  -ip $feIP -port 22)
                $script:VmInboundAccessState["managementControlPlane"] = $result
                if ($status) {
                    $msg = "Inbound connectivity to $feIP succeeded"
                } else {
                    $msg = "Inbound connectivity to $feIP failed"
                }

                $result = Get-Result -scenario "inbound" -isSuccess $status -detailedStatus $msg

                break
            }
            "cloudagent" {
                $result = $(Test-CloudAgentConnectivity -sshIP $feIP  -sshPort "22" -windows $false)
                break
            }
            "dns" {
                $result = $(Test-DnsResolution -sshIP $feIP  -sshPort "22" -windows $false)
                break                
            }
            "internet" {
                $result  = $(Test-InternetConnectivity -sshIP $feIP -sshPort "22" -windows $false)
                break
            }
            "outboundvip" {
                $result = $(Test-OutboundConnectivity -sshIP $feIP -sshPort 22 `
                    -outboundIP $feIP -outboundPort "22" `
                    -windows $false -scenario $scenario -expectedLinuxResponse $script:CurlToSshResponse)
                break
            }
        }                             
        $status = $result.Passed
        if (-not $status) {
            throw "Test-ManagementClusterConnectivity Scenario [$scenario] failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario]"
        return $result
    } catch {
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario]" -err $_
        return $false
    }
}

function Convert-Error($e)
{
    return $e | Select-Object @{N="Message"; E={$_.Exception.Message}}, @{N="Stack"; E={$_.ScriptStackTrace.Split("`n")[0]}}
}
function Test-VMConnectivity
{
    param (
        [ValidateSet("inbound","cloudagent", "dns", "internet", "outboundvip")]
        [string]$scenario,
        [string]$vmName,
        [string]$group,
        [string]$lbName
    )

    $result = $false

    try {
        $vmTestResult = Get-VMTestResult -vmName $vmName -isSuccess $result -testName $scenario

        if ($scenario -ne "inbound" -and $script:VmInboundAccessState["$vmName"] -eq $false) {
            $msg = "$($MyInvocation.MyCommand) [SKIP NO INBOUND ACCESS] Scenario [$scenario] Group [$group] VM [$vmName]"
            Write-Msg -msg $msg -Warning
            $vmTestResult.DetailedStatus = $msg
            return $vmTestResult
        }

        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName]"
        $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
        $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
        $sshKey = $(Get-MocConfigCache).sshPrivateKey
        $lb = Get-MocLoadBalancer -name $lbName -group $group
        $feIP = $lb.properties.frontendIPConfigurations[0].properties.ipAddress
        $vm = Get-MocVirtualMachine -group $group -name $vmName
        $nicName = $_.virtualmachineproperties.networkprofile.networkinterfaces[0].id
        $isWindows = $_.virtualmachineproperties.osprofile.osType -eq "Windows"
        $endpoint = Get-NatEndpoint -groupName $group -nicName $nicName
            
        if ($endpoint -eq $null) {
            $msg = "$($MyInvocation.MyCommand) Unable to find ssh endpoint for vm $($_.Name) in cluster $name"
            $vmTestResult.DetailedStatus = $msg
            Write-Fail -msg $msg
            return
        }

        $sshInfo = "[$($endpoint.ip)`:$($endpoint.port)]"
        $vmTestResult.SSHInfo = $sshInfo

        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName], SSH [$($endpoint.ip)`:$($endpoint.port)]" 
        switch ($scenario) {
            "inbound" {               
                $status = $(Test-Connectivity  -ip $($endpoint.ip) -port $($endpoint.port))
                $script:VmInboundAccessState["$vmName"] = $status
                if ($status) {
                    $msg = "Inbound connectivity to $sshInfo succeeded"
                } else {
                    $msg = "Inbound connectivity to $sshInfo failed"
                }

                $result = Get-Result -scenario "inbound" -isSuccess $status -detailedStatus $msg
                break
            }
            "cloudagent" {
                $result = $(Test-CloudAgentConnectivity -sshIP $($endpoint.ip) -sshPort $($endpoint.port) -windows $isWindows)
                break
            }
            "dns" {
                $result = $(Test-DnsResolution -sshIP $($endpoint.ip) -sshPort $($endpoint.port) -windows $isWindows)
                break                
            }
            "internet" {
                $result = $(Test-InternetConnectivity -sshIP $($endpoint.ip) -sshPort $($endpoint.port) -windows $isWindows)
                break
            }
            "outboundvip" {
                $result = $(Test-OutboundConnectivity -sshIP $($endpoint.ip) -sshPort $($endpoint.port) `
                    -outboundIP $($endpoint.ip) -outboundPort "22" `
                    -windows $isWindows -scenario $scenario -expectedLinuxResponse $script:CurlToSshResponse)
                break
            }
        }              

        $vmTestResult.Results = @($result)
        $vmTestResult.Passed = $result.Passed

        if (-not $result.Passed) {
            throw "Test-VMConnectivity Scenario [$scenario] VM [$vmName] failed "
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName]"
    } catch {
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario] Group [$group] VM [$vmName]" -err $_
        $vmTestResult.DetailedStatus = Convert-Error $_
        $vmTestResult.Passed = $false
    }

    return $vmTestResult
}

function Test-TargetClusterConnectivity
{
    param (
        [ValidateSet("inbound", "cloudagent", "dns", "internet", "outboundvip")]
        [string]$scenario,
        [string]$clusterName
    )

    $testName = "$($MyInvocation.MyCommand)[$scenario]"
    $testResult = Get-TestResult -testName $testName -isSuccess $true -testResults @()
    try {
        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario]"
        $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
        $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
        $sshKey = $(Get-MocConfigCache).sshPrivateKey
        $status = $true

        $clusters = Get-AksHciCluster -ErrorAction Stop -Name $clusterName
        $clusters | ForEach-Object {
            $name = $_.Name
            $group = "clustergroup-$name"
            Write-Msg -msg "$($MyInvocation.MyCommand) Cluster [$name]"
            $lbName = "$name-load-balancer"
            $vms = Get-MocVirtualMachine -group $group
            $vms | ForEach-Object {
                $result = Test-VMConnectivity -scenario $scenario -vmName $_.Name -group $group -lbName $lbName
                $testResult.TestResults += $result
                $status = $status -and $result.Passed
            }
        }

        if (-not $status) {
            throw "Test-TargetClusterConnectivity [$scenario] failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario]"
    }
    catch  {
        $status = $false
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario]" -err $_
        $testResult.DetailedStatus = Convert-Error $_
    } finally  {
        $testResult.Passed = $status
    }
    return $testResult
}

function Test-ManagementClusterNetwork()
{
    param (
        [ValidateSet("cloudagent", "dns", "internet", "inbound", "all", "outboundvip")]
        [string]$scenario = "all"
    )

    $status = $true
    $mgmtClusterResult = Get-ClusterTestResult -clusterName "Management" -testResults @()
    try {
        switch($scenario) {
            {$_ -in "inbound", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "inbound"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }
            {$_ -in "controlplane", "all"} {
                $result = Test-ManagementControlPlaneConnectivity
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "cloudagent", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "cloudagent"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "dns", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "dns"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "internet", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "internet"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }
            
            {$_ -in "outboundvip", "all"} {
                $result = Test-ManagementClusterConnectivity -scenario "outboundvip"
                $mgmtClusterResult.TestResults += $result
                $status = $status -and $result.Passed
            }
        }
    } catch {
        $status = $false
        $mgmtClusterResult.DetailedStatus = Convert-Error $_
    }

    $mgmtClusterResult.Passed = $status
    return $mgmtClusterResult
}

function Test-TargetClusterNetwork()
{
    param (
        [ValidateSet("controlplane","cloudagent", "dns", "internet", "inbound", "all", "outboundvip")]
        [string]$scenario="all",
        [string]$clusterName
    )

    $status = $true
    $testName = "$($MyInvocation.MyCommand)[$scenario]"
    $testResult = Get-TestResult -testName $testName -isSuccess $true -testResults @()

    try {
        $clusters = Get-AksHciCluster -ErrorAction Stop -Name $clusterName
        $clusters | ForEach-Object {
            $name = $_.Name
            $clusterTestResult = Get-ClusterTestResult -clusterName $name -isSuccess $true -testResults @()
            $testResult.TestResults += $clusterTestResult
            switch($scenario) {
                {$_ -in "inbound", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "inbound"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }
                {$_ -in "controlplane", "all"} {
                    $result = Test-TargetControlPlaneConnectivity -clusterName $name 
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }

                {$_ -in "cloudagent", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "cloudagent"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }

                {$_ -in "dns", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "dns"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }

                {$_ -in "internet", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "internet"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }
                {$_ -in "outboundvip", "all"} {
                    $result = Test-TargetClusterConnectivity -clusterName $name -scenario "outboundvip"
                    $clusterTestResult.TestResults += $result
                    $status = $status -and $result.Passed
                }
            }
        }
    } catch {
        $status = $false
        $testResult.DetailedStatus = Convert-Error $_
    } finally {
        $testResult.Passed = $status
    }

    return $testResult

}

function Test-AksHciSdnNetwork()
{
    <#
    .SYNOPSIS
    Validates various network connectivity when SDN integration with AKS-HCI is present.
    .DESCRIPTION
    Performs the following validations for management and target clusters
    1. DNS resolution for mcr.microsoft.com
    2. Internet connectivity by connecting to "https://mcr.microsoft.com"
    3. Connectivity from the vm to cloudagent endpoint on the physical host
    4. Inbound ssh connectivity to the VMs from the physical host
    #>

    param(
        [Parameter(Mandatory = $false, ParameterSetName = 'Management')]
        [Switch] $Management
    )

    ValidateState -ModuleName "AKSHCI"

    if (-not $(Test-IsAksHciVersionHigerOrEqual -Major 1 -Minor 0 -Build 18)) {
        Write-Warning "Test-AksHciSdnNetwork is only supported with AksHci version 1.0.18 or higher"
        throw "NOT_SUPPORTED"
    }

    $status = $true
    $results = @()

    $result = Test-ManagementClusterNetwork
    $results += $result
    $status = $status -and $result.Passed

    $testTargetCluster = -not $Management.IsPresent

    if ($testTargetCluster) {
        $result = Test-TargetClusterNetwork
        $results += $result
        $status = $status -and $result.Passed
    }

    if ($status) {
        Write-Pass -testName "$($MyInvocation.MyCommand)"
    }
    else {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -msg "One or more tests failed"
    }

    Write-TestResults $results

    return $results
}

function Get-MocVnetYaml {
    param (
        $name,
        [Array]
        $dnsServers,
        $addressPrefix,
        $poolStart,
        $poolEnd,
        $switchName,
        $location
    )
    
    $mocVnet = @"
name: $name
virtualnetworkpropertiesformat:
    dhcpoptions:
        dnsservers:
"@
$dnsServers | ForEach-Object {
    $mocVnet += @"
`n - $($_)
"@
}

    $mocVnet += @"
`n subnets:
        - name: $("$name-subnet")
          subnetpropertiesformat:
            addressprefix: $addressPrefix
            ipallocationmethod: Static
            ippools:
                - end: $poolEnd
                  ippooltype: vm
                  name: vmpool0
                  start: $poolStart
tags:
    VSwitch-Name: $switchName
type: Transparent
location: $location
"@

    return $mocVnet
}
function Invoke-MocCtl {
    param (
        $Arguments
    )
    
    $Script:MocModuleData = Import-Module -Name 'Moc' -PassThru
    & $Script:MocModuleData { param($Arguments) Invoke-MocCommand -Arguments $Arguments } -Arguments $Arguments
}

function Get-MocLbYaml {
    param (
        $name,
        $subnet
    )
    $lbYaml = @"
name: $name
loadbalancerpropertiesformat:
    backendaddresspools:
      - name: $name
    frontendipconfigurations:
        - frontendipconfigurationpropertiesformat:
            subnet:
                id: $subnet
    loadbalancingrules:
        - loadbalancingrulepropertiesformat:
            backendport: 22
            frontendport: 6443
            protocol: Tcp
tags:
    Role: AKSHCI_APISERVER
"@

    return $lbYaml
}

function Get-MocVmYaml {
    param (
        $name,
        $image,
        $nicName,
        $sshKey
    )
    
    $yaml = @"
name: $name
virtualmachineproperties:
  disableHighAvailability: false
  hardwareprofile:
    vmsize: Default
  storageprofile:
    imagereference:
      name: $image
  osprofile:
    computername: $name
    adminusername: "clouduser"
    adminpassword: ""
    customdata: ""
    windowsconfiguration: null
    linuxconfiguration:
      ssh:
        publickeys:
        - keydata: $sshKey
      disablepasswordauthentication: true
    osType: Linux
    osbootstrapengine: CloudInit
  securityProfile:
    enableTPM: false
    uefiSettings:
    secureBootEnabled: true
  networkprofile:
    networkinterfaces:
     - id: $nicName
"@
  return $yaml
}

function TryGetMocResource {
    param(
        $scriptBlock
    )

    try {
        $result = Invoke-Command -ScriptBlock $scriptBlock
        return $result
    }
    catch {
        if (-not $_.Exception.Message.Contains("NotFound")) {
            throw
        }
    }
}

function New-AksHciSdnTestVm {
    
    param(
        $name
    )

    ValidateState -ModuleName MOC

    $location = $script:MocLocation
    $sdnCommonName = $name
    $groupName = $script:SDNTestId
    
    $getGroup = {Get-MocGroup -name $groupName -location $location -ErrorAction Stop}
    $group = TryGetMocResource -scriptBlock $getGroup 

    if ($group -eq $null) {
        $group = new-mocgroup -name $groupName -location $location -ErrorAction Stop
    }

    $config = Get-MocConfigCache
    $lbName = $script:SDNTestId + "-load-balancer"

    $getLb = {Get-MocLoadBalancer -name $lbName -group $groupName -ErrorAction Stop}
    $lb = TryGetMocResource -scriptBlock $getLb 

    if ($lb -eq $null) {
        $lbYaml = Get-MocLbYaml -name $lbName -subnet $config.vnetName
        $lbfile = New-TemporaryFile
        $lbYaml | out-file $lbfile
        $lbCreateArgs = "network loadbalancer create --config $($lbfile.FullName) --group $groupName"
        Invoke-MocCtl -Arguments $lbCreateArgs
    }
    
    $getVnet = {Get-MocVirtualNetwork -name $config.vnetName -group $groupName -ErrorAction Stop}
    $vnet = TryGetMocResource -scriptBlock $getVnet 

    if ($vnet -eq $null) {
        $yaml = Get-MocVnetYaml -name $config.vnetName -dnsservers $config.dnsservers -switchName $config.vswitchName `
            -poolStart $config.k8snodeippoolstart -poolEnd $config.k8snodeippoolend `
            -addressPrefix $config.ipaddressprefix -location $script:MocLocation

        $file = New-TemporaryFile
        $yaml | out-file $file
        $vnetCreateArgs = "network vnet create --config $($file.FullName) --group $groupName"
        Invoke-MocCtl -Arguments $vnetCreateArgs
    }

    $nicName = $name + "-nic"
    
    try {
        $nic = Get-MocNetworkInterface -name $nicName -group $groupName -ErrorAction SilentlyContinue
    }    
    catch {
        if (-not $_.Exception.Message.Contains("NotFound")) {
            throw
        }
    }

    $getNic = {Get-MocNetworkInterface -name $nicName -group $groupName -ErrorAction Stop}
    $nic = TryGetMocResource -scriptBlock $getNic

    if ($nic -eq $null) {
        New-MocNetworkInterface -name $nicName -group $groupName -virtualNetworkName $config.vnetName -loadBalancerBackend $lbName
    }

    $getImage = {Get-MocGalleryImage -name $script:SDNTestId -location $script:MocLocation -ErrorAction Stop}
    $image = TryGetMocResource -scriptBlock $getImage

    if ($image -eq $null) {
        $kvaConfig = Get-KvaConfig
        $imageRelease = Get-ImageReleaseManifest -imageVersion $kvaConfig.version `
            -operatingSystem "Linux"  -k8sVersion $kvaConfig.kvaK8sVersion.TrimStart("v") -moduleName "Moc"
        $result = Get-ImageRelease -imageRelease $imageRelease -imageDir $kvaConfig.imageDir -moduleName "Moc" -releaseVersion $kvaConfig.version

        New-MocGalleryImage -name $script:SDNTestId -location $script:MocLocation -imagePath $result -container "MocStorageContainer"
    }

    $getVM = {Get-MocVirtualMachine -name $name -group $groupName -ErrorAction Stop}
    $vm = TryGetMocResource -scriptBlock $getVM

    if ($vm -eq $null) {
        $vmYaml = Get-MocVmYaml -name $name -image $script:SDNTestId -nicName $nicName -sshKey $(Get-content $config.sshPublicKey)
        $vmFile = New-TemporaryFile
        $vmYaml | out-file $vmFile
        $vmCmd = "compute vm create --config $($vmFile.FullName) --group $groupName"
        Invoke-MocCtl -Arguments $vmCmd
    }
}

function Cleanup-AksHciSdnTestResources {
    
    <#
    .SYNOPSIS
    Cleans up any test resources created for SDN pre deployment validation.
    #>

    $group = Get-MocGroup  -location $script:MocLocation -ErrorAction SilentlyContinue -Name $script:SDNTestId
    if ($group -ne $null) {
        Remove-MocGroup -Name $script:SDNTestId -location $script:MocLocation
    }
    
    Remove-MocGalleryImage -name $script:SDNTestId -location $script:MocLocation -ErrorAction SilentlyContinue
}

function Remove-AksHciSdnTestVm {
    param(
        $name
    )

    $groupName = $script:SDNTestId
    $vm = Get-MocVirtualMachine -name $name -group $groupName -ErrorAction SilentlyContinue
    if ($vm -ne $null) {
        Remove-MocVirtualMachine -name $name -group $groupName -ErrorAction SilentlyContinue
    }
    $nicName = $name + "-nic"
    $nic = Get-MocNetworkInterface -name $nicName -group $groupName -ErrorAction SilentlyContinue
    if ($nic -ne $null) {
        Remove-MocNetworkInterface -name $nicName -group $groupName -ErrorAction SilentlyContinue
    }
}

function Test-AksHciSdnReadiness()
{
    <#
    .SYNOPSIS
    Validates that the SDN deployment is ready to install AKS-HCI. Run after Set-AksHciConfig and before Install-AksHci.
    .DESCRIPTION
    Creates a new VM in SDN VNET using MOC and runs various validations to test if the deployment meets the requirements for AksHCI SDN integration. Includes the below tests,
    1. Validates the various SDN parameters provided as part of Set-AksHciConfig against NetworkController.
    2. Inbound connectivity to the VM public IP address from the host. Tested using ssh.
    3. Outbound connectivity from the VM to the cloudagent on the host. Tested by connecting to cloudagent on port 65000 from the vm.
    4. Outbound connectivity from the VM to a public IP address in SDN. Tested by connecting to the VMs public IP:22 from inside the VM.
    5. DNS resolution. Tested by resolving mcr.microsoft.com
    6. Internet connectivity. Tested by connecting to https://mcr.microsoft.com
    This command requires MOC to be already installed and must be executed after Set-AksHciConfig, before Install-AksHci.
    IMPORTANT If '-skipCleanup' parameter is specified, the test resources should be manually cleaned up using Cleanup-AksHciSdnTestResources before proceeding with Install-AksHCI.
    Failure to do so may cause Install-AksHCI to fail due to resource conflicts.
    .PARAMETER scenario
    Specify a test scenario. By default executes all scenarios.
    .PARAMETER skipCleanup
    Skips the cleanup of test vm and associated resources after the test. Use if any tests fails and VM is required to further debug the failure.
    IMPORTANT If this parameter is specified, the test resources should be manually cleaned up using Cleanup-AksHciSdnTestResources before proceeding with Install-AksHCI.
    Failure to do so may cause Install-AksHCI to fail due to resource conflict.
    #>

    param (
        [ValidateSet("cloudagent", "dns", "internet", "inbound", "all")]
        [string]$scenario="all",        
        [switch]$skipCleanup
    )

    $groupName = $script:SDNTestId

    if ($skipCleanup.IsPresent) {
        Write-Warning "SkipCleanup specified. `nManual cleanup of test resources using 'Cleanup-AksHciSdnTestResources' is required before proceeding with Install-AksHci" -WarningAction Inquire
    }

    ValidateState -ModuleName "MOC"
    $script:retryCount = 10
    $isInstalled = Test-IsAksHciInstalled -ModuleName "AKSHCI"

    if ($isInstalled) {
        Write-Warning "Test-AksHciSdnReadiness skipped, cannot be used after akshci is installed. Use Test-AksHciSdnNetwork instead."
        throw "NOT_SUPPORTED"
    }

    if (-not $(Test-IsAksHciVersionHigerOrEqual -Major 1 -Minor 0 -Build 18)) {
        Write-Warning "Test-AksHciSdnReadiness is only supported with AksHci version 1.0.18 or higher"
        throw "NOT_SUPPORTED"
    }

    $status = $true
    $testName = "$($MyInvocation.MyCommand)[$scenario]"
    $testResult = Get-TestResult -testName $testName -isSuccess $true -testResults @()

    try {
        
        if (-not $(Test-AksHciSdnConfig)) {
            throw "SDN Configuration check failed"
        }

        New-AksHciSdnTestVm -name "sdn-predeployment"
        $clusterTestResult = Get-ClusterTestResult -clusterName $groupName -isSuccess $true -testResults @()
        $testResult.TestResults += $clusterTestResult
        switch($scenario) {
            {$_ -in "inbound", "all"} {
                $result = Test-SdnTestNetworkConnectivity -groupName $groupName -scenario "inbound"
                $clusterTestResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "cloudagent", "all"} {
                $result = Test-SdnTestNetworkConnectivity -groupName $groupName -scenario "cloudagent"
                $clusterTestResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "dns", "all"} {
                $result = Test-SdnTestNetworkConnectivity -groupName $groupName -scenario "dns"
                $clusterTestResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "internet", "all"} {
                $result = Test-SdnTestNetworkConnectivity -groupName $groupName -scenario "internet"
                $clusterTestResult.TestResults += $result
                $status = $status -and $result.Passed
            }

            {$_ -in "outboundvip", "all"} {
                $result = Test-SdnTestNetworkConnectivity -groupName $groupName -scenario "outboundvip"
                $clusterTestResult.TestResults += $result
                $status = $status -and $result.Passed
            }
        }
    } catch {
        $status = $false
        $testResult.DetailedStatus = Convert-Error $_
    } finally {
        $testResult.Passed = $status
        if (-not $skipCleanup.IsPresent) {
            Cleanup-AksHciSdnTestResources
        }
    }

    if ($testResult.Passed) {
        Write-Msg "$($MyInvocation.MyCommand) All SDN pre-deployments tests passed"
        Write-Pass -testName "$($MyInvocation.MyCommand)"
    }
    else {
        Write-Fail -testName "$($MyInvocation.MyCommand)" -msg "One or more tests failed"
    }

    Write-TestResults $testResult

    if ($skipCleanup.IsPresent) {
        Write-Warning "SkipCleanup switch was specified. Manually cleanup the test resources using Cleanup-AksHciSdnTestResources before Install-AksHci"
    }

    return $testResult

}

function Test-SdnTestNetworkConnectivity
{
    param (
        [ValidateSet("inbound", "cloudagent", "dns", "internet", "outboundvip")]
        [string]$scenario,
        [string]$groupName
    )

    $testName = "$($MyInvocation.MyCommand)[$scenario]"
    $testResult = Get-TestResult -testName $testName -isSuccess $true -testResults @()
    try {
        Write-Msg -msg "$($MyInvocation.MyCommand) Scenario [$scenario]"
        $cloudFqdn = $(Get-MocConfigCache).cloudFqdn
        $cloudAgentAddress = (Resolve-dnsName $cloudFqdn -Type A -DnsOnly -ErrorAction Stop).IPAddress
        $sshKey = $(Get-MocConfigCache).sshPrivateKey
        $status = $true

        $name = "SDNTestNetwork"
        Write-Msg -msg "$($MyInvocation.MyCommand) Cluster [$name]"
        $lbName = "$groupName-load-balancer"
        $vms = Get-MocVirtualMachine -group $groupName

        $vms | ForEach-Object {
            $result = Test-VMConnectivity -scenario $scenario -vmName $_.Name -group $groupName -lbName $lbName
            $testResult.TestResults += $result
            $status = $status -and $result.Passed
        }

        if (-not $status) {
            throw "Test-SdnTestNetworkConnectivity [$scenario] failed"
        }

        Write-Pass -testName "$($MyInvocation.MyCommand) Scenario [$scenario]"
    }
    catch  {
        $status = $false
        Write-Fail -testName "$($MyInvocation.MyCommand) Scenario [$scenario]" -err $_
        $testResult.DetailedStatus = Convert-Error $_
    } finally  {
        $testResult.Passed = $status
    }
    return $testResult
}