PureStorage.CBS.AVS

1.1.403-marelli

PureStorage.CommonUtil.ps1

<#
Common Utility functions.
#>

$DEFAULT_PER_HOST_TIMEOUT_IN_SECONDS = 30
$MIN_TIMEOUT_IN_MINUTES = 10
$MAX_TIMEOUT_IN_MINUTES = 60


function Get-AzContextWrapper {
    # Check if the environment variable is set to AzureCLI
    if ($env:CBS_AVS_EXEC_MODE -eq "AzureCLI") {
        try {
            $AzAccount = az account show --query "{SubscriptionId:id, TenantId:tenantId, Name:name, User:user}" -o json | ConvertFrom-Json

            if (-not $AzAccount) {
                throw "Failed to retrieve Azure CLI context. Did you do 'az login'?"
            }

            # Construct an equivalent AzContext-like object with additional AuthSource property
            $AzContext = [PSCustomObject]@{
                Account        = $AzAccount.User.name
                Environment    = "AzureCloud"
                Subscription   = [PSCustomObject]@{
                    Id       = $AzAccount.SubscriptionId
                    Name     = $AzAccount.Name
                    State    = "Enabled"  # Assuming the subscription is enabled
                    TenantId = $AzAccount.TenantId
                }
                Tenant         = [PSCustomObject]@{
                    Id = $AzAccount.TenantId
                }
                SubscriptionId = $AzAccount.SubscriptionId
                TenantId       = $AzAccount.TenantId
                AuthSource     = "AzureCLI"  # New field to indicate source
            }

            return $AzContext
        } catch {
            Write-Error "Failed to retrieve context from Azure CLI: $_"
            return $null
        }
    }
    else {
        try {
            $AzContext = Get-AzContext
            if (-not $AzContext) {
                throw "Failed to retrieve AzContext using Get-AzContext."
            }

            # Add AuthSource property to indicate the source
            $AzContext | Add-Member -MemberType NoteProperty -Name "AuthSource" -Value "PowerShell" -Force
            return $AzContext
        } catch {
            Write-Error "Failed to retrieve context using Az PowerShell module: $_"
            return $null
        }
    }
}

function Get-PfaHostFromVmHost {
    <#
    .SYNOPSIS
      Gets a FlashArray host object from a ESXi vmhost object
    .DESCRIPTION
      Takes in a vmhost and returns a matching FA host if found
    #>

    Param(
        [Parameter(Mandatory = $true, ValueFromPipeline = $True)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.VMHost]$Esxi,

        [Parameter(Mandatory = $true, ValueFromPipeline = $True)]
        $Flasharray
    )

    $fahosts = Get-Pfa2Host -Array $FlashArray | Where-Object { $_.IsLocal -eq $True }
    $ArrayName = Get-ArrayName -FlashArray $FlashArray
    $IsNvme = Test-ArrayHasNVmeInterface -FlashArray $FlashArray

    if ($IsNvme) {
        $adapter = $esxi | Get-VMHostHba | Where-Object { $_.Model -like "*NVMe over TCP*" }
        if ($null -ne $adapter) {
            $nqn = Get-VmHostNqn -Esxi $Esxi
            if (-not $nqn) {
                throw "Failed to get NQN for $($Esxi.Name)"
            }
            foreach ($fahost in $fahosts) {
                if ($fahost.nqns.count -ge 1) {
                    foreach ($fahostnqn in $fahost.nqns) {
                        if ($nqn.ToLower() -eq $fahostnqn.ToLower()) {
                            $faHostMatch = $fahost
                            break
                        }
                    }
                }
            }
        }
    }
    else {
        $iscsiadapter = $esxi | Get-VMHostHBA -Type iscsi | Where-Object { $_.Model -eq "iSCSI Software Adapter" }
        if ($null -ne $iscsiadapter) {
            $iqn = $iscsiadapter.ExtensionData.IScsiName
            foreach ($fahost in $fahosts) {
                if ($fahost.iqns.count -ge 1) {
                    foreach ($fahostiqn in $fahost.iqns) {
                        if ($iqn.ToLower() -eq $fahostiqn.ToLower()) {
                            $faHostMatch = $fahost
                            break
                        }
                    }
                }
            }
        }
    }
    if ($null -ne $faHostMatch) {
        return $faHostMatch
    }
    else {
        throw "No matching host for $($esxi.Name) could be found on the Pure Cloud Block Store $ArrayName"
    }
}

function Get-PfaHostGroupfromVcCluster {
    <#
    .SYNOPSIS
      Retrieves a FA host group from an ESXi cluster
    .DESCRIPTION
      Takes in a vCenter Cluster and retrieves corresonding host group
    #>

    Param(
        [Parameter(Mandatory = $true, ValueFromPipeline = $True)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster,

        [Parameter(Mandatory = $True)]
        $Flasharray
    )

    $esxiHosts = $cluster | Get-VMHost
    $faHostGroups = @()
    $faHostGroupNames = @()
    $ArrayName = Get-ArrayName -FlashArray $FlashArray
    foreach ($esxiHost in $esxiHosts) {
        try {
            $faHost = $esxiHost | Get-PfaHostFromVmHost -flasharray $flasharray
            if ($null -ne $faHost.HostGroup.Name) {
                if ($faHostGroupNames.contains($faHost.HostGroup.Name)) {
                    continue
                }
                else {
                    $faHostGroupNames += $faHost.HostGroup.Name
                    $faHostGroup = Get-Pfa2HostGroup -Array $Flasharray -Name $($faHost.HostGroup.Name)  -ErrorAction stop | Where-Object { $_.IsLocal -eq $True }
                    $faHostGroups += $faHostGroup
                }
            }
        }
        catch {
            continue
        }
    }
    if ($null -eq $faHostGroup) {
        throw "No host group found for cluster $($Cluster.Name) on $ArrayName."
    }
    if ($faHostGroups.count -gt 1) {
        Write-Warning -Message "Cluster $($Cluster.Name) spans more than one host group. The recommendation is to have only one host group per cluster"
    }
    return $faHostGroups
}


function Get-ArrayName {
    Param(
        [Parameter(Mandatory = $true)]
        $FlashArray
    )
    $ArrayName = (Get-Pfa2Array -Array $Flasharray).Name
    return $ArrayName
}

function Get-AzureAuthHeader {
    param (
        [Parameter(Mandatory=$true)]
        $AzContext
    )

    # Determine the authentication source (default to PowerShell if AuthSource is missing)
    $AuthSource = if ($AzContext.PSObject.Properties.Name -contains "AuthSource") {
        $AzContext.AuthSource
    } else {
        "PowerShell"
    }

    if ($AuthSource -eq "AzureCLI") {
        # Get access token using az cli
        try {
            $AzToken = az account get-access-token --resource "https://management.azure.com" --query "accessToken" -o tsv
            if (-not $AzToken) {
                throw "Failed to retrieve Azure access token using Azure CLI."
            }
        } catch {
            throw "Error retrieving Azure access token from Azure CLI: $_"
        }
    } else {
        # Default to PowerShell method
        try {
            $AzProfile = [Microsoft.Azure.Commands.Common.Authentication.Abstractions.AzureRmProfileProvider]::Instance.Profile
            $ProfileClient = New-Object -TypeName Microsoft.Azure.Commands.ResourceManager.Common.RMProfileClient -ArgumentList ($AzProfile)
            $Token = $profileClient.AcquireAccessToken($AzContext.Subscription.TenantId)

            if (-not $Token) {
                throw "Failed to acquire Azure access token for tenant $($AzContext.Subscription.TenantId) using Az PowerShell."
            }
            $AzToken = $Token.AccessToken
        } catch {
            throw "Error retrieving Azure access token from Az PowerShell module: $_"
        }
    }

    # Construct authentication header
    $AuthHeader = @{
        'Content-Type'  = 'application/json'
        'Authorization' = 'Bearer ' + $AzToken
    }

    return $AuthHeader
}

function Get-AVSvCenterEndpoint {
    param (
        [Hashtable] $AuthHeader,
        [String] $SubscriptionId,
        [String] $AvsResourceGroupName,
        [String] $AvsPrivateCloudName
    )
    $RestUri = "https://management.azure.com/subscriptions/$($subscriptionId)/resourceGroups/$($avsResourceGroupName)/providers/Microsoft.AVS/privateClouds/$($avsPrivateCloudName)?api-version=2022-05-01"
    $Response = Invoke-RestMethod -Uri $restUri -Method Get -Headers $authHeader
    if (-not $Response) {
        throw "Failed to get AVS vCenter endpoint. Please make sure you have the right permission to access the AVS instance with subscriptionId $subscriptionId, resource group $avsResourceGroupName and private cloud $avsPrivateCloudName"
    }
    $VcsaEndpoint = $response.properties.endpoints.vcsa
    $VcsaIPAddress = [System.Uri]::new($vcsaEndpoint).Host

    return $vcsaIPAddress
}

function Get-AVSvCenterCredential {
    param (
        [Hashtable] $AuthHeader,
        [String] $SubscriptionId,
        [String] $AvsResourceGroupName,
        [String] $AvsPrivateCloudName
    )
    $RestUri = "https://management.azure.com/subscriptions/$($subscriptionId)/resourceGroups/$($avsResourceGroupName)/providers/Microsoft.AVS/privateClouds/$($avsPrivateCloudName)/listAdminCredentials?api-version=2022-05-01"
    $Response = Invoke-RestMethod -Uri $restUri -Method Post -Headers $AuthHeader
    if (-not $Response) {
        throw "Failed to get AVS vCenter credential. Please make sure you have the right permission to access the AVS instance with subscriptionId $subscriptionId, resource group $avsResourceGroupName and private cloud $avsPrivateCloudName"
    }
    return $Response
}

function Connect-AVSvCenter {
    param (
        [String] $AVSResourceGroupName,
        [String] $AVSPrivateCloudName
    )
    $AzContext = Get-AzContextWrapper
    $AzureSubscriptionId = $AzContext.Subscription.Id
    Write-Debug "The default subscriptionId $AzureSubscriptionId of the current Azure context will be used."
    $AuthHeader = Get-AzureAuthHeader -AzContext $AzContext
    $AVSvCenterCredential = Get-AVSvCenterCredential -AuthHeader $AuthHeader -SubscriptionId $AzureSubscriptionId -AVSResourceGroupName $AVSResourceGroupName -AVSPrivateCloudName $AVSPrivateCloudName
    $AVSvCenterEndpoint = Get-AVSvCenterEndpoint  -AuthHeader $AuthHeader -SubscriptionId $AzureSubscriptionId -AVSResourceGroupName $AVSResourceGroupName -AVSPrivateCloudName $AVSPrivateCloudName
    $Credential = New-Object System.Management.Automation.PSCredential -ArgumentList ($AVSvCenterCredential.vCenterUsername, $(ConvertTo-SecureString $AVSvCenterCredential.vCenterPassword -AsPlainText -Force))
    # Try to connect to vcenter everytime when a cmdlet is invoked by the user to make sure right avs instance is connected
    $vCenterServer = Connect-VIserver -server $AVSvCenterEndpoint -Credential $Credential -ErrorAction Stop
    return $vCenterServer
}

function Connect-PureCloudBlockStore {
    Param (
        [Parameter(Mandatory = $false)]
        $PureCloudBlockStoreConnection
    )

    if (-not $PureCloudBlockStoreConnection) {
        $fa = $global:PURE_AUTH_2_X
        if (-not $fa) {
            throw "Please login to Cloud Clock Store using: Connect-Pfa2Array"
        }
    }
    else {
        $fa = $PureCloudBlockStoreConnection
    }

    $ClientName = "PureStorage.CBS.AVS"
    $module = Get-Module -Name $ClientName
    if ($module) {
        $Version = $module.Version.ToString()
    }
    else {
        $Version = "0.0.0.0"
    }

    # if was able to connect, also update the user agent to track backup sdk telemetry
    $success = $fa.UpdateUserAgent($ClientName, $Version );

    # this should not be a show stopper
    if (-not $success) {
        Write-Warning -Level WARN -FunctionName $FunctionName -Msg "Failed to set useragent"
    }
    return $fa
}

function Purge-AzureSecretWithRetry {
    Param (
        [String] $KeyVaultName,
        [String] $SecretName
    )
    $errorOccurred = $true
    $retryCount = 0
    # Unfortunately we have to do retry because Remove-AzKeyVaultSecret is asyncronous. If purge happens directly after delete, "Secret is currently being deleted" error is thrown
    do {
        try {
            # To purge, use -InRemovedState parameter
            $retryCount = $retryCount + 1
            Remove-AzkeyVaultSecret -VaultName $KeyVaultName -Name $SecretName -InRemovedState -Force -ErrorAction Stop
            $errorOccurred = $false
        }
        catch {
            $errorOccurred = $true
            if ($retryCount -gt 3) {
                throw
            }
            Start-Sleep -Seconds 5
        }
    } while ($errorOccurred)
}

function Get-AvsClusterInformation {
    param (
        [String] $AvsResourceGroupName,
        [String] $AvsPrivateCloudName,
        [String] $AvsClusterName
    )

    $AzContext = Get-AzContextWrapper
    $AuthHeader = Get-AzureAuthHeader -AzContext $AzContext

    # Construct REST API URI
    $AzureSubscriptionId = $AzContext.Subscription.Id
    Write-Debug "Using subscriptionId $AzureSubscriptionId."
    $RestUri = "https://management.azure.com/subscriptions/$($AzureSubscriptionId)/resourceGroups/$($AvsResourceGroupName)/providers/Microsoft.AVS/privateClouds/$($AvsPrivateCloudName)/clusters/$($AvsClusterName)?api-version=2022-05-01"
    Write-Debug "RestUri: $RestUri"
    Write-Debug "Authorization: $AuthHeader['Authorization']"

    # Execute REST API call
    try {
        $Response = Invoke-RestMethod -Uri $RestUri -Method Get -Headers $AuthHeader
        if (-not $Response) {
            throw "Failed to get AVS cluster information. Ensure the cluster '$AvsClusterName' exists."
        }
        return $Response
    } catch {
        throw "Error retrieving AVS cluster information: $_"
    }
}

function Get-AvsClusterSku {
    param (
        [String] $AvsResourceGroupName,
        [String] $AvsPrivateCloudName,
        [String] $AvsClusterName
    )

    $result = Get-AvsClusterInformation -AvsResourceGroupName $AvsResourceGroupName -AvsPrivateCloudName $AvsPrivateCloudName -AvsClusterName $AvsClusterName
    return $result.sku.name
}

function Get-AVSClusterProvisionStatus {
    param (
        [String] $AvsResourceGroupName,
        [String] $AvsPrivateCloudName,
        [String] $AvsClusterName
    )

    $result = Get-AvsClusterInformation -AvsResourceGroupName $AvsResourceGroupName -AvsPrivateCloudName $AvsPrivateCloudName -AvsClusterName $AvsClusterName
    return $result.properties.provisioningState
}

$MPIO_SKUS = @(
    "av36",
    "av36p",
    "av52"
)
function Test-MPIOProvisionStatus {
    param (
        [Parameter(Mandatory = $true)]
        [String] $AvsResourceGroupName,
        [Parameter(Mandatory = $true)]
        [String] $AvsPrivateCloudName,
        [Parameter(Mandatory = $true)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster

    )

    $sku = Get-AvsClusterSku -AvsResourceGroupName $AvsResourceGroupName -AvsPrivateCloudName $AvsPrivateCloudName -AvsClusterName $Cluster.Name
    if ($sku -in $MPIO_SKUS) {
        Write-Host "MPIO is supported for cluster $Cluster. Checking vmk interfaces...."
        $VMHosts = Get-VMHost -Location $Cluster
        foreach ($VMHost in $VMHosts) {
            $MpioAdapterOne = Get-VMHostNetworkAdapter -VMHost $VMHost | Where-Object { $_.Name -eq "vmk5" }
            $MpioAdapterTwo = Get-VMHostNetworkAdapter -VMHost $VMHost | Where-Object { $_.Name -eq "vmk6" }
            if ($MpioAdapterOne -and $MpioAdapterTwo) {
                Write-Host "MPIO is ready on $VMHost..."
            }
            else {
                throw "The current provisioning status of MPIO is not ready for host $VMHost. Please check if VMkernel adapter 'vmk5' and 'vmk6' are ready for host $VMhost. If not, please wait for twenty minutes and try again."
            }
        }
    }
    else {
        Write-Host "MPIO is not available for SKU $sku. Skipping MPIO check for cluster $Cluster..."
    }
}

function Get-VMKInterface {
    param (
        [Parameter(Mandatory = $true)]
        [String] $AvsResourceGroupName,
        [Parameter(Mandatory = $true)]
        [String] $AvsPrivateCloudName,
        [Parameter(Mandatory = $true)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster
    )

    $sku = Get-AvsClusterSku -AvsResourceGroupName $AvsResourceGroupName -AvsPrivateCloudName $AvsPrivateCloudName -AvsClusterName $Cluster.Name
    if ($sku -in $MPIO_SKUS) {
        return "vmk5", "vmk6"
    }
    return "vmk1"
}

function Get-RunCommandNamedOutput {
    param (
        [String] $RunCmdExecutionName,
        [String] $AvsResourceGroupName,
        [String] $AvsPrivateCloudName
    )
    $AzContext = Get-AzContextWrapper
    $AzureSubscriptionId = $AzContext.Subscription.Id
    Write-Debug "The subscriptionId $AzureSubscriptionId will be used."
    $AuthHeader = Get-AzureAuthHeader -AzContext $AzContext
    $RestUri = “https://management.azure.com/subscriptions/$($AzureSubscriptionId)/resourceGroups/$($avsResourceGroupName)/providers/Microsoft.AVS/privateClouds/$($avsPrivateCloudName)/scriptExecutions/$($RunCmdExecutionName)?api-version=2021-12-01”
    $Response = Invoke-RestMethod -Uri $restUri -Method Get -Headers $AuthHeader
    return $Response.properties.namedOutputs
}

function Get-Timeout {
    param (
        [Parameter(Mandatory = $true)]
        [string]$Cluster,

        [Parameter(Mandatory = $true)]
        $InputParams
    )

    if ($InputParams.ContainsKey("TimeoutInMinutes")) {
        return $InputParams["TimeoutInMinutes"]
    }
    $HostCount = (Get-VMHost -Location $Cluster).Count
    $Timeout = [Math]::Ceiling(($HostCount * $DEFAULT_PER_HOST_TIMEOUT_IN_SECONDS) / 60)
    if ($Timeout -lt $MIN_TIMEOUT_IN_MINUTES ) {
        $Timeout = $MIN_TIMEOUT_IN_MINUTES
    }
    elseif ($Timeout -gt $MAX_TIMEOUT_IN_MINUTES) {
        $Timeout = $MAX_TIMEOUT_IN_MINUTES
    }

    return $Timeout
}

function Test-ArrayHasNVmeInterface {
    param (
        [Parameter(Mandatory = $true)]
        $FlashArray
    )

    $NVMeIinterfaces = Get-Pfa2NetworkInterface -Array $FlashArray | Where-Object { "nvme-tcp" -in $_.Services }
    $IsNvme = $NVMeIinterfaces.Count -gt 0
    if ($IsNvme) {
        Write-Debug "Array $($FlashArray.Name) has NVMe interface"
    }
    else {
        Write-Debug "Array $($FlashArray.Name) has iSCSI interface"
    }
    return $IsNvme
}

function Get-VmHostNqn {
    param (
        [Parameter(Mandatory = $true)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.VMHost]$Esxi
    )
    $esxicli = Get-EsxCli -VMHost $Esxi -V2
    $NvmeInfo = $esxicli.nvme.info.get.Invoke()
    $nqn = $NvmeInfo.HostNQN
    Write-Debug "NQN for $($Esxi.Name) is $nqn"
    return $nqn
}

function Test-RunCommandPackageAvailability {
    param (
        [Parameter(Mandatory = $true)]
        [PSCustomObject] $AzContext,

        [Parameter(Mandatory = $true)]
        [String] $RunCommandModule,

        [Parameter(Mandatory = $true)]
        [String] $RunCommandPackageVersion,

        [Parameter(Mandatory = $false)]
        [String] $AVSCloudName,

        [Parameter(Mandatory = $false)]
        [String] $AVSResourceGroup
    )

    # Ensure the required parameters are present in the AzContext
    if (-not $AzContext.Subscription.Id -or -not $AzContext.TenantId) {
        throw "Invalid AzContext: Missing Subscription ID or Tenant ID"
    }

    $AuthHeader = Get-AzureAuthHeader -AzContext $AzContext

    $Uri = "https://management.azure.com/subscriptions/$($AzContext.Subscription.Id)/resourceGroups/$($AVSResourceGroup)/providers/Microsoft.AVS/privateClouds/$($AVSCloudName)/scriptPackages/$($RunCommandModule)@$($RunCommandPackageVersion)?api-version=2023-03-01"

    try {
        $Response = Invoke-RestMethod -Uri $Uri -Method GET -Headers $AuthHeader -ErrorAction Stop
        if ($Response.StatusCode -eq 200) {
            Write-Host "Found '$($RunCommandModule)@$($RunCommandPackageVersion)' RunCommand package in Azure."
            return $true
        }
        throw "Could not find '$($RunCommandModule)@$($RunCommandPackageVersion)' RunCommand package in Azure. Please make sure the package is available in Azure."
    } catch {
        throw "Could not find '$($RunCommandModule)@$($RunCommandPackageVersion)' RunCommand package in Azure. Ensure the package is available in Azure."
    }
}

function Get-DefaultAzureSubscriptionId {
    # Azure Resource Manager cmdlets use this settings by default when making Azure Resource Manager requests.
    # To list all of subscription, it would be Get-AzContext -ListAvailable
    # In our case, getting default account is sufficient
    $AzContext = Get-AzContextWrapper
    if ($AzContext.Subscription) {
        return $AzContext.Subscription.Id
    }
    else {
        throw "Could not find default Azure subscription. Please make sure you are logging in to Azure using 'Connect-AzAccount' or 'az login' (for CBS_AVS_SCRIPT_EXECUTION_MODE='AzureCLI'). If you have multiple subscriptions, please use 'Select-AzSubscription' to select the subscription you want to use."
    }
}

function Get-VMKIpAddress {
    param (
        [Parameter(Mandatory = $true)]
        $Cluster,
        [Parameter(Mandatory = $true)]
        [String] $VMKName
    )

    $Cluster | Get-VMHost | Select Name, @{ Name = "IPAddress"; Expression = { ( $_.ExtensionData.Config.Network.Vnic | Where-Object { $_.Device -eq $VMKName }).Spec.Ip.IpAddress } }
}

function Test-NetworkConnectionFromServer {
    param (
        [Parameter(Mandatory = $true)]
        [String] $AVSCloudName,

        [Parameter(Mandatory = $true)]
        [String] $AVSResourceGroup,

        [Parameter(Mandatory = $false)]
        [String] $PureCloudBlockStoreConnection,

        [Parameter(Mandatory = $false)]
        [String] $ClusterName,

        [Parameter(Mandatory = $false)]
        [String] $ServerName,

        [Parameter(Mandatory = $true)]
        [String] $ConnectionType,

        [Parameter(Mandatory = $true)]
        [Int] $TargetPort,

        [Parameter(Mandatory = $true)]
        [String] $ServiceType
    )

    Write-Progress -Activity "Testing $ConnectionType connections" -Status "0% Complete:" -PercentComplete 1

    $fa = Connect-PureCloudBlockStore -PureCloudBlockStoreConnection $PureCloudBlockStoreConnection
    $WorkflowID = New-WorkflowID
    New-PhoneHomeWorkflowLogEntry -RestClient $fa -Event "Begin" -ID $WorkflowID -Name $MyInvocation.MyCommand
    try {
        $success = $true
        $vCenterServer = Connect-AVSvCenter -AVSResourceGroupName $AVSResourceGroup -AVSPrivateCloudName $AVSCloudName
        if ($ClusterName) {
            $Cluster = Get-Cluster -Name $ClusterName
            if (-not $Cluster) {
                throw "Cluster $ClusterName does not exist"
            }
        }
        $ServerList = @()
        if ($ServerName) {
            if ("VC" -eq $ServerName) {
                $ServerList += "VC"
            }
            else {
                $ServerList = ($Cluster | Get-VMHost -Name $ServerName ).Name

            }
        }
        else {
            $ServerList = ($Cluster | Get-VMHost).Name
        }
        if (-not $ServerList) {
            throw "No matching VMHost(s) found"
        }

        # Get the array interfaces
        $AddressList = (Get-Pfa2NetworkInterface -Array $fa | Where-Object { $_.Services -contains $ServiceType -and $_.Enabled }).Eth.Address
        if (-not $AddressList) {
            throw "No $ConnectionType interfaces found on the array"
        }

        $ServerList = @()
        if ("VASA" -eq $ConnectionType) {
            $ServerList += "VC"

        }
        $count = $ServerList.Count * $AddressList.Count
        $CurrentCount = 0
        foreach ($ServerName in $ServerList) {
            Write-Host "Testing $ConnectionType connection on Server ($ServerName) ..."
            foreach ($Address in $AddressList) {
                try {
                    Write-Host  "Testing $ConnectionType connection from Server ($ServerName) to address $Address ..."
                    $CurrentCount++
                    $Percent = $CurrentCount * 100 / $count
                    Write-Progress -Activity "Testing $ConnectionType connections" -Status "$Percent% Complete" -PercentComplete $Percent
                    $params = @{
                        ServerName      = $ServerName
                        TargetIpAddress = $Address
                        TargetPort      = $TargetPort
                    }
                    Invoke-RunScript -RunCommandName  "Test-ConnectionFromServer" -RunCommandModule "Microsoft.AVS.VMFS" -Parameters $params `
                        -AVSCloudName $AVSCloudName -AVSResourceGroup $AVSResourceGroup -TimeoutInMinutes $TimeoutInMinutes
                    # If no exception is thrown, the connection is successful
                    Write-Host "$ConnectionType connection from Server $ServerName to address $Address is successful"
                }
                catch {
                    $success = $false
                    Write-Error "$ConnectionType connection from Server $ServerName to address $Address failed with error: $_"
                }
            }
        }
    }
    catch {
        New-PhoneHomeWorkflowLogEntry -RestClient $fa -Event "Error" -ID $WorkflowID -Name $MyInvocation.MyCommand -ErrorMessage $_
        Write-Progress -Activity "Error" -Status "100% Complete:" -PercentComplete 100
        throw
    }
    Disconnect-VIServer -Server $vCenterServer -Confirm:$false -ErrorAction SilentlyContinue
    New-PhoneHomeWorkflowLogEntry -RestClient $fa -Event "Complete" -ID $WorkflowID -Name $MyInvocation.MyCommand
    Write-Progress -Activity "Operation is done" -Status "100% Complete:" -PercentComplete 100
    if (-not $success) {
        throw "Failed one or more connection tests. Please check the error messages above for more details."
    }
}

function Get-NvmeDeviceIdFromVolumeSrial {
    param (
        [Parameter(Mandatory = $true)]
        [String] $VolumeSerial
    )

    return  "eui.00" + $VolumeSerial.substring(0, 14) + "24a937" + $VolumeSerial.substring(14)
}

function Get-ISCSIDeviceIdFromVolumeSrial {
    param (
        [Parameter(Mandatory = $true)]
        [String] $VolumeSerial
    )

    return "naa.624a9370" + $VolumeSerial
}
function Get-VolumeSerialFromDeviceId {
    param (
        [Parameter(Mandatory = $true)]
        [String] $DeviceId
    )
    $DeviceId = $DeviceId.ToUpper()
    if ($DeviceId -like "eui.00*") {
        # NVMe device id
        return $DeviceId.substring(6, 14) + $DeviceId.substring(26)
    }
    elseif ($DeviceId -like "naa.624a9370*") {
        # iSCSI device id
        return $DeviceId.substring(12)
    }
    else {
        throw "Unsuppored device id $DeviceId"
    }
}