PureStorage.CBS.VVolUtil.ps1

function Set-PCBSVmHostiSCSI{
    <#
    .SYNOPSIS
      Configure FlashArray iSCSI target information on ESXi host
    .DESCRIPTION
      Takes in an ESXi host and configures FlashArray iSCSI target info
    #>


    Param(
        [Parameter(Mandatory=$true)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.VMHost]$Esxi,

        [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
        $Flasharray
    )
    Begin {
      $allESXitargets = @()
    }
    Process {
        if ($esxi.ExtensionData.Runtime.ConnectionState -ne "connected")
        {
            Write-Warning "Host $($esxi.NetworkInfo.HostName) is not in a connected state and cannot be configured."
            return
        }
        $ESXitargets = @()
        $faiSCSItargets = Get-Pfa2NetworkInterface -Array $FlashArray |Where-Object {$_.services -eq "iscsi"} |Where-Object {$_.enabled -eq $true} | Where-Object {$null -ne $_.Eth.address}
        if ($null -eq $faiSCSItargets)
        {
            throw "The target FlashArray does not currently have any iSCSI targets configured."
        }
        $iscsi = $esxi |Get-VMHostStorage
        if ($iscsi.SoftwareIScsiEnabled -ne $true)
        {
            $esxi | Get-vmhoststorage |Set-VMHostStorage -SoftwareIScsiEnabled $True |out-null
        }
        foreach ($faiSCSItarget in $faiSCSItargets)
        {
            $iscsiadapter = $esxi | Get-VMHostHba -Type iScsi | Where-Object {$_.Model -eq "iSCSI Software Adapter"}
            if (!(Get-IScsiHbaTarget -IScsiHba $iscsiadapter -Type Send -ErrorAction stop | Where-Object {$_.Address -cmatch $faiSCSItarget.Eth.address}))
            {
                New-IScsiHbaTarget -IScsiHba $iscsiadapter -Address $faiSCSItarget.Eth.address -ErrorAction stop
            }
            $esxcli = $esxi |Get-esxcli -v2
            $iscsiargs = $esxcli.iscsi.adapter.discovery.sendtarget.param.get.CreateArgs()
            $iscsiargs.adapter = $iscsiadapter.Device
            $iscsiargs.address = $faiSCSItarget.Eth.address
            $delayedAck = $esxcli.iscsi.adapter.discovery.sendtarget.param.get.invoke($iscsiargs) |where-object {$_.name -eq "DelayedAck"}
            $loginTimeout = $esxcli.iscsi.adapter.discovery.sendtarget.param.get.invoke($iscsiargs) |where-object {$_.name -eq "LoginTimeout"}
            if ($delayedAck.Current -eq "true")
            {
                $iscsiargs = $esxcli.iscsi.adapter.discovery.sendtarget.param.set.CreateArgs()
                $iscsiargs.adapter = $iscsiadapter.Device
                $iscsiargs.address = $faiSCSItarget.Eth.address
                $iscsiargs.value = "false"
                $iscsiargs.key = "DelayedAck"
                $esxcli.iscsi.adapter.discovery.sendtarget.param.set.invoke($iscsiargs) |out-null
            }
            if ($loginTimeout.Current -ne "30")
            {
                $iscsiargs = $esxcli.iscsi.adapter.discovery.sendtarget.param.set.CreateArgs()
                $iscsiargs.adapter = $iscsiadapter.Device
                $iscsiargs.address = $faiSCSItarget.Eth.address
                $iscsiargs.value = "30"
                $iscsiargs.key = "LoginTimeout"
                $esxcli.iscsi.adapter.discovery.sendtarget.param.set.invoke($iscsiargs) |out-null
            }
            $ESXitargets += Get-IScsiHbaTarget -IScsiHba $iscsiadapter -Type Send -ErrorAction stop | Where-Object {$_.Address -cmatch $faiSCSItarget.Eth.address}
        }
        $allESXitargets += $ESXitargets

    }
    End {
      return $allESXitargets
    }
}

function New-PCBSHostFromVmHost {
    <#
    .SYNOPSIS
      Create a FlashArray host from an ESXi vmhost object
    .DESCRIPTION
      Takes in a vCenter ESXi host and creates a FlashArray host
    #>


    Param(
            [Parameter(Mandatory=$true)]
            [VMware.VimAutomation.ViCore.Types.V1.Inventory.VMHost]$Esxi,

            [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
            $Flasharray
    )

    $ArrayName = Get-ArrayName -FlashArray $FlashArray
    Write-Verbose "Creating FlashArray $ArrayName host for VMHost $($Esxi.Name)..."
    $newFaHost = $null
    try {
        $newFaHost = Get-PCBSHostFromVmHost -flasharray $flasharray -esxi $esxi -ErrorAction Stop
    }
    catch {}
    if ($null -eq $newFaHost) {
        Set-PCBSVMHostiSCSI -esxi $esxi -flasharray $flasharray  -ErrorAction Stop | Out-Null
        $iscsiadapter = $esxi | Get-VMHostHBA -Type iscsi | Where-Object { $_.Model -eq "iSCSI Software Adapter" }
        if ($null -eq $iscsiadapter) {
            throw "No Software iSCSI adapter found on host $($esxi.NetworkInfo.HostName)."
        }
        else {
            $iqn = $iscsiadapter.ExtensionData.IScsiName
        }
        try {
            try {
                $newFaHost = New-Pfa2Host -Array $FlashArray -Name ($esxi.NetworkInfo.HostName) -Iqns $iqn -ErrorAction Stop
            }
            catch {
                if (($PSItem.ToString() | convertfrom-json).msg -eq "Host already exists.") {
                    $randName = $esxi.NetworkInfo.HostName + (Get-Random -Maximum 99999 -Minimum 10000).ToString()
                    $newFaHost = New-Pfa2Host -Array $FlashArray -Name $($randName) -Iqns $iqn
                }
                else {
                    throw ($PSItem.ToString() | convertfrom-json).msg
                }
            }
            $arrayInfo =  Get-Pfa2Array -Array $FlashArray
            $majorVersion = [int](($arrayInfo.Version.Split('.'))[0])
            if ($majorVersion -ge 5) {
                Update-Pfa2Host -Array $FlashArray -Name $($newFaHost.name) -Personality "esxi" | Out-Null
            }
        }
        catch {
            Write-Error "$_"
            return $null
        }
    }
    else {
        if ($newFaHost.wwn.count -gt 0) {
            throw "The host $($esxi.NetworkInfo.HostName) is already configured on array $Arrayname with FibreChannel. Multiple-protocols at once are not supported by VMware."
        }
    }
    return $newFaHost
}

function Get-PCBSHostFromVmHost {
    <#
    .SYNOPSIS
      Gets a FlashArray host object from a ESXi vmhost object
    .DESCRIPTION
      Takes in a vmhost and returns a matching FA host if found
    #>


    Param(
        [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.VMHost]$Esxi,

        [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
        $Flasharray
    )
    $iscsiadapter = $esxi | Get-VMHostHBA -Type iscsi | Where-Object {$_.Model -eq "iSCSI Software Adapter"}
    $fahosts = Get-Pfa2Host -Array $FlashArray
    $ArrayName = Get-ArrayName -FlashArray $FlashArray
    if ($null -ne $iscsiadapter)
    {
        $iqn = $iscsiadapter.ExtensionData.IScsiName
        foreach ($fahost in $fahosts)
        {
            if ($fahost.iqns.count -ge 1)
            {
                foreach ($fahostiqn in $fahost.iqns)
                {
                    if ($iqn.ToLower() -eq $fahostiqn.ToLower())
                    {
                        $faHostMatch = $fahost
                        break
                    }
                }
            }
        }
    }
    if ($null -ne $faHostMatch)
    {
      return $faHostMatch
    }
    else
    {
        throw "No matching host for $($esxi.Name) could be found on the FlashArray $ArrayName"
    }
}

function Get-PCBSHostGroupfromVcCluster {
    <#
    .SYNOPSIS
      Retrieves a FA host group from an ESXi cluster
    .DESCRIPTION
      Takes in a vCenter Cluster and retrieves corresonding host group
    #>


    Param(
        [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster,

        [Parameter(Mandatory=$True)]
        $Flasharray
    )

    $esxiHosts = $cluster |Get-VMHost
    $faHostGroups = @()
    $faHostGroupNames = @()
    $ArrayName = Get-ArrayName -FlashArray $FlashArray
    foreach ($esxiHost in $esxiHosts)
    {
        try {
            $faHost = $esxiHost | Get-PCBSHostFromVmHost -flasharray $flasharray
            if ($null -ne $faHost.HostGroup.Name)
            {
                if ($faHostGroupNames.contains($faHost.HostGroup.Name))
                {
                    continue
                }
                else {
                    $faHostGroupNames += $faHost.HostGroup.Name
                    $faHostGroup = Get-Pfa2HostGroup -Array $Flasharray -Name $($faHost.HostGroup.Name)  -ErrorAction stop
                    $faHostGroups += $faHostGroup
                }
            }
        }
        catch{
            continue
        }
    }
    if ($null -eq $faHostGroup)
    {
        throw "No host group found for cluster $($Cluster.Name) on $ArrayName."
    }
    if ($faHostGroups.count -gt 1)
    {
        Write-Warning -Message "Cluster $($Cluster.Name) spans more than one host group. The recommendation is to have only one host group per cluster"
    }
    return $faHostGroups
}

function Remove-PCBSUnusedHosts {
    Param (
        [Parameter(Mandatory=$true)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster,

        [Parameter(Mandatory=$True)]
        $Flasharray
    )

    $hostGroups =  Get-PCBSHostGroupfromVcCluster -flasharray $Flasharray -cluster $cluster
    $faHostList = @()

    foreach ($hostGroup in $hostGroups) {
        $faHosts = Get-Pfa2Host -Array $Flasharray | Where-Object{$_.HostGroup.Name -eq $hostGroup.Name}
        $faHostList = $faHostList + $faHosts
    }
    $vmHosts = $Cluster | Get-VMHost
    foreach ($vmHost in $vmHosts) {
        $faHost = Get-PCBSHostFromVmHost -Flasharray $FlashArray -Esxi $vmHost -ErrorAction Ignore
        # Remove used hosts from the list
        $faHostList = $faHostList | Where-Object {$_.Name -ne $faHost.Name}
    }

    foreach ($faHost in $faHostList) {
        Write-Host "Removing unused host '$faHost' ..."
        Update-Pfa2Host -Array $Flasharray -Name $faHost.Name -HostGroupName ''
        Remove-Pfa2Host -Array $Flasharray -Name $faHost.Name
    }

}

function New-PCBSHostGroupfromVcCluster {
    <#
    .SYNOPSIS
      Create a host group from an ESXi cluster
    .DESCRIPTION
      Takes in a vCenter Cluster and creates hosts (if needed) and host group
    #>


    Param(
        [Parameter(Mandatory=$true)]
        [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster,

        [Parameter(Mandatory=$True)]
        $Flasharray
    )


    $updated_hosts = @()
    # We have to use try catch here as adding -ErrorAction SilentlyContinue still throw terminating error
    try {
        $hostGroup =  Get-PCBSHostGroupfromVcCluster -flasharray $Flasharray -cluster $cluster
    }
    catch {}
    if ($hostGroup.count -gt 1)
    {
        throw "The cluster already is configured on the FlashArray and spans more than one host group. This cmdlet does not support a multi-hostgroup configuration."
    }
    if ($null -ne $hostGroup)
    {
        $clustername = $hostGroup.name
    }
    $esxiHosts = $cluster |Get-VMHost
    $HostMap = @{}
    foreach ($esxiHost in $esxiHosts)
    {
        $faHost = $null
        try {
            $faHost = Get-PCBSHostFromVmHost -flasharray $Flasharray -esxi $esxiHost -ErrorAction Ignore
            if ($null -ne $faHost.HostGroup.Name)
            {
                if ($null -ne $hostGroup)
                {
                    if ($hostGroup.name -ne $faHost.HostGroup.Name)
                    {
                        Write-Warning "The host $($faHost.name) already exists and is already in the host group $($faHost.hgroup)."
                    }
                }
            }
        }
        catch {
            Write-Host "$_"
        }
        if ($null -ne $faHost)
        {
            if ($fahost.wwn.count -ge 1)
            {
                throw "The host $($esxiHost.NetworkInfo.HostName) is already configured on the FlashArray for FC. Mixed mode is not supported by VMware."
            }
        }
        if ($null -eq $faHost)
        {
            try {
                $faHost = New-PCBSHostFromVmHost -flasharray $Flasharray -ErrorAction Stop -esxi $esxiHost
                $HostMap[$faHost.Name] = $esxiHost
                $updated_hosts += $esxiHost.Name
            }
            catch {
                Write-Error $Global:Error[0]
                throw "Could not create host. Cannot create host group."
            }
        }
        else {
            $HostMap[$faHost.name] = $esxiHost.Name
        }
    }
    #FlashArray only supports Alphanumeric or the dash - character in host group names. Checking for VMware cluster name compliance and removing invalid characters.
    if ($null -eq $hostGroup)
    {
        if ($cluster.Name -match "^[a-zA-Z0-9\-]+$")
        {
            $clustername = $cluster.Name
        }
        else
        {
            $clustername = $cluster.Name -replace "[^\w\-]", ""
            $clustername = $clustername -replace "[_]", ""
            $clustername = $clustername -replace " ", ""
        }
        $hg = $null
        $hg =  Get-Pfa2HostGroup -Array $Flasharray -Name $clustername -ErrorAction SilentlyContinue
        if ($null -ne $hg)
        {
            if ($hg.hosts.count -ne 0)
            {
                #if host group name is already in use and has only unexpected hosts i will create a new one with a random number at the end
                $nameRandom = Get-random -Minimum 1000 -Maximum 9999
                $clustername = "$($clustername)-$($nameRandom)"
                $hostGroup = New-Pfa2HostGroup -Array $Flasharray -Name $clustername  -ErrorAction stop

            }
            else {
                $hostGroup = $hg
            }
        }
        else {
                #if there is no host group, it will be created
                $hostGroup = New-Pfa2HostGroup -Array $Flasharray -Name $clustername  -ErrorAction stop
        }
    }
    $faHostNames = @()
    foreach ($faHostName in $HostMap.Keys)
    {
        if ($null -eq $faHost.HostGroup.Name)
        {
            $faHostNames += $faHostName
        }
    }
    #any hosts that are not already in the host group will be added
    if ($faHostNames.count -gt 0)
    {
        foreach ($faHostName in $faHostNames)
        {
            Write-Host "Adding $faHostName to $clustername..."
            Update-Pfa2Host -Array $Flasharray -Name $faHostName -HostGroupName $clustername | Out-Null
            if (-not $updated_hosts -contains $HostMap[$faHostName]) {
                $updated_hosts += $HostMap[$faHostName]
            }
        }
    }
    return $updated_hosts
}
function Get-PCBSConnectionOfDatastore {
    <#
    .SYNOPSIS
      Takes in a vVol or VMFS datastore, FlashArray connections and returns the correct connection.
    .DESCRIPTION
      Takes in a vVol or VMFS datastore, FlashArray connections and returns the correct connection or $null if not connected.
    #>


    Param(

      [Parameter(Mandatory=$True,ValueFromPipeline=$True)]
      $Flasharray,

      [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
      [ValidateScript({
        if (($_.Type -ne 'VMFS') -and ($_.Type -ne 'VVOL'))
        {
            throw "The entered datastore is not a VMFS or vVol datastore. It is type $($_.Type). Please only enter a VMFS or vVol datastore"
        }
        else {
          $true
        }
      })]
      [VMware.VimAutomation.ViCore.Types.V1.DatastoreManagement.Datastore]$Datastore
    )

    if ($datastore.Type -eq 'VMFS')
      {
          $lun = $datastore.ExtensionData.Info.Vmfs.Extent.DiskName |select-object -unique
          if ($lun -like 'naa.624a9370*')
          {
            $volserial = ($lun.ToUpper()).substring(12)
            $pureVolumes = Get-Pfa2Volume -Array $Flasharray
            $purevol = $purevolumes | where-object { $_.serial -eq $volserial }
            if ($null -ne $purevol.name)
            {
                return $flasharray
            }
          }
          else
          {
              throw "This VMFS is not hosted on FlashArray storage."
          }
      }
      elseif ($datastore.Type -eq 'VVOL')
      {
          $datastoreArraySerial = $datastore.ExtensionData.Info.VvolDS.StorageArray[0].uuid.Substring(16)

        $arraySerial = (Get-Pfa2Array -Array $Flasharray).id
        if ($arraySerial -eq $datastoreArraySerial)
        {
            return $flasharray
        }

      }
      # The datastore was not found on any of the FlashArray connections.
      return $null
    }

function Get-ArrayID {
    Param(
        [Parameter(Mandatory=$true)]
        $FlashArray
    )
    $ArrayID = "com.purestorage:" + (Get-Pfa2Array -Array $Flasharray).id
    return $arrayID
}

function Get-ArrayName {
    Param(
        [Parameter(Mandatory=$true)]
        $FlashArray
    )
    $ArrayName = (Get-Pfa2Array -Array $Flasharray).Name
    return $ArrayName
}

function Get-ArrayOUI {
    Param (
        [Parameter(Mandatory=$true)]
        [string]$ArrayID
    )
    $ArrayOui = $ArrayID.substring(16,36)
    $ArrayOui = $ArrayOui.replace("-","")
    $ArrayOui = $ArrayOui.Substring(0,16)
    return $ArrayOui
}

function Get-RootStorageContainerID {
    Param(
        [Parameter(Mandatory=$true)]
        [string]$ArrayID
    )
    $md5 = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider
    $utf8 = new-object -TypeName System.Text.UTF8Encoding
    $hash = $md5.ComputeHash($utf8.GetBytes($ArrayID.substring(16,36)))
    $hash2 = $md5.ComputeHash(($hash))
    $hash2[6] = $hash2[6] -band 0x0f
    $hash2[6] = $hash2[6] -bor 0x30
    $hash2[8] = $hash2[8] -band 0x3f
    $hash2[8] = $hash2[8] -bor 0x80
    $newGUID = (new-object -TypeName System.Guid -ArgumentList (,$hash2)).Guid
    $fixedGUID = $newGUID.Substring(18)
    $scId = $newGUID.Substring(6,2) + $newGUID.Substring(4,2) + $newGUID.Substring(2,2) + $newGUID.Substring(0,2) + "-" + $newGUID.Substring(11,2) + $newGUID.Substring(9,2) + "-" + $newGUID.Substring(16,2) + $newGUID.Substring(14,2) + $fixedGUID
    $scId = $scId.Replace("-","")
    $scId = "vvol:" + $scId.Insert(16,"-")
    return $scId
}

function Rescan-HBA {
    Param(
        $ESXIHosts,
        $FlashArray
    )
    foreach ($esxiRescan in $ESXIHosts)
            {
                $hbas = $esxiRescan |get-PCBSHostFromVmHost -flasharray $FlashArray
                if ($hbas.iqns.count -ge 1)
                {
                  $hbaType = "iSCSI"
                }
                elseif ($hbas.wwn.count -ge 1)
                {
                  $hbaType = "FibreChannel"
                }
                $esxiHost = $esxiRescan.ExtensionData
                $storageSystem = Get-View -Id $esxiHost.ConfigManager.StorageSystem
                Write-Host "Rescanning HBAs for $($esxiRescan.Name)..."
                $hbas = ($esxiRescan |Get-VMHostHba |where-object {$_.Type -eq $hbaType}).device
                foreach ($hba in $hbas)
                {
                    $storageSystem.rescanHba($hba)
                }
            }
}

function Find-ProtocolEndpoint {
    Param (
        $Esxi,
        $PEVolumes,
        $multiStorageContainersSupport,
        $UseDefaultStore,
        $arrayOui
    )

    $esxcli = $esxi |Get-EsxCli -v2
    $hostProtocolEndpoint = $esxcli.storage.core.device.list.invoke() |where-object {$_.IsVVOLPE -eq $true}
    foreach ($hostPE in $hostProtocolEndpoint)
    {
        $peID = $hostPE.Device.Substring(12,24)
        $peID = $peID.Substring(0,16)
        if ($peID -eq $arrayOui)
        {
            if ($multiStorageContainersSupport -and -not $UseDefaultStore) {
                if ($PEVolumes.Count -gt 0){
                    foreach ($peVolume in $PEVolumes) {
                        $volumeSerial = $peVolume.Serial.ToLower()
                        if ($hostPe.Device -like "*$volumeSerial*"){
                            return $hostPE
                        }
                    }
                }
            }
        }
        else {
            return $hostPE
        }
    }
    return $null
}

function Update-PCBSVASAProvider {
    Param (
        $FlashArray,
        [PScredential] $FlashArrayCredential,
        [switch] $Refresh
    )

    $vasaProviders = @()
    $MaxRetryCount = 10
    $mgmtIPs =  Get-Pfa2NetworkInterface | Where-Object {$_.services -eq "management" -and $_.Enabled -and $_.Name -like "ct*"}
    $registeredController = @()
    $ArrayName = Get-ArrayName -FlashArray $FlashArray
    foreach ($mgmtIP in $mgmtIPs)
    {
        $vasaRegistered = $false
        $retry_count = 0
        $controller_name = ($mgmtIp.Name.Split("."))[0]

        if ($registeredController -contains $controller_name) {
            continue
        }
        $provider_name = "$ArrayName-$controller_name"
        if ($Refresh) {
            Write-Host "Removing provider $provider_name ..."
            Get-VasaProvider -Name $provider_name |Remove-VasaProvider -Confirm:$false
        }
        do
        {
            $vasaProvider = Get-VasaProvider -Name $provider_name -ErrorAction Ignore
            if ($vasaProvider) {
                Write-Host "VASA Provider already exists for $ArrayName controller $controller_name ..."
                if ($vasaProvider.status -eq "online") {
                    $vasaRegistered = $True
                }
                elseif ($vasaProvider.status -eq "offline") {
                    # VASA provider can be offline if:
                    # - Array is not accessible (in this case, we will get an error when trying to connect)
                    # - Certificate is expired
                    # - Certificate is invalid (for example a new certificate uploaded to the Array)

                    Write-Warning "VASA provider $provider_name status is $($vasaProvider.status)"
                    # No need to check for expiry time as the cert might not be expired but invalid. Remove the certificate and re-register the VASA provider
                    $cert_name = "vasa-$controller_name"
                    Write-Host "Removing certificate $cert_name from FlashArray: $ArrayName ..."
                    Remove-Pfa2Certificate -Name $cert_name
                    Write-Host "Creating self-signed certificate $cert_name on FlashArray: $ArrayName ..."
                    New-Pfa2Certificate -Name $cert_name -CommonName $mgmtIP.Eth.Address -Organization "Pure Storage" -OrganizationalUnit "Pure Storage" | Out-Null
                    Write-Host "Removing VASA provider $provider_name ..."
                    Remove-VasaProvider -Provider $vasaProvider -Confirm:$false
                    $vasaRegistered = $False
                }
                else {
                    throw  "Could not update VASA provider. VASA provider $provider_name status is $($vasaProvider.status)."
                }
            }
            else {
                Write-Host "Creating VASA Provider for $ArrayName controller $controller_name ..."
                try {
                    $vasaProviders += New-VasaProvider -Name ($provider_name) -Credential $FlashArrayCredential -Url ("https://$($mgmtIP.Eth.Address):8084") -force -ErrorAction Stop
                    $vasaRegistered = $True
                    Write-Host "VASA Provider for $ArrayName-$controller_name created successufully."
                }
                catch {
                    if ($retry_count -lt $MaxRetryCount) {
                        Write-Warning "Failed to register VASA provider $provider_name with error $_ Retrying ..."
                        $vasaRegistered = $false
                        Start-Sleep -Seconds 10
                    }
                    else {
                        throw "Failed to register VASA provider $provider_name with error $_"
                    }
                }
            }
            $retry_count = $retry_count + 1
        }
        while ($vasaRegistered -ne $true -and $retry_count -lt $MaxRetryCount)
        if (-not $vasaRegistered) {
            throw "Failed to register VASA provider $provider_name after $retry_count tries."
        } else {
            $registeredController += $controller_name
        }
    }
     # Certificate refresh on all hosts in vCenter
    Update-ESXCertificates
    return $vasaProviders
}
function Mount-PCBSVvolDatastore {
    <#
    .SYNOPSIS
      Mounts a FlashArray VVol Datastore to a host or cluster
    .DESCRIPTION
      Mounts a FlashArray VVol Datastore to a cluster, connects a PE to the cluster if not present.
      The datastore will be created if it does not already exist. The function will return the exting (or newly created) datastore
    #>


    Param(
            [Parameter(Mandatory=$True)]
            $Flasharray,

            [Parameter(Mandatory=$True)]
            [PSCredential]$FlasharrayCredential,

            [Parameter(Mandatory=$false)]
            [string]$DatastoreName,

            [Parameter(Mandatory=$false)]
            [bool]$UseDefaultStore,

            [Parameter(Mandatory=$true,ValueFromPipeline=$True)]
            [VMware.VimAutomation.ViCore.Types.V1.Inventory.Cluster]$Cluster
        )
        $multiStorageContainersSupport = $false
        $arrayVersion = ((Get-Pfa2Array -Array $Flasharray).Version).Split('.')
        $arrayVersionObject = [Version]::new($arrayVersion[0],$arrayVersion[1],$arrayVersion[2])
        $baseVersionObject = [Version]::new("6","4","1")
        $multiStorageContainersSupport = $arrayVersionObject -ge $baseVersionObject

        $arrayID = Get-ArrayID -FlashArray $Flasharray
        $arrayOui = Get-ArrayOUI -ArrayID $arrayID
        $ArrayName = Get-ArrayName -FlashArray $FlashArray



        if ($True -eq $UseDefaultStore) {
            $scId = Get-RootStorageContainerID -ArrayID $arrayID
            $datastoreExists = Get-Datastore |Where-Object {$_.Type -eq "VVol"} |Where-Object {$_.ExtensionData.Info.VVolds.Scid -eq $scId}
            if ($null -eq $datastoreExists)
            {
                if (-not $DatastoreName) {
                    $DatastoreName = $ArrayName + "-vvol-DS"
                }
            }
            else {
                if (-not $DatastoreName) {
                    $DatastoreName = $datastoreExists.Name
                }
            }
        }
        else {
            # Using a multi-storage container Pod datastore
            if (-not $multiStorageContainersSupport) {
                throw "Non default containers can only be used for purity version 6.4.1 or later"
            }
            if (-not $DatastoreName) {
                throw "Datastore name must be provided when using non default containers."
            }
            # See if a pod was already created
            $pod = Get-Pfa2Pod -Array $Flasharray -Name $DatastoreName -ErrorAction Ignore
            if (-not $pod) {
                Write-Host "Creating a pod $DatastoreName ..."
                $pod = New-Pfa2Pod -Name $DatastoreName -ErrorAction Stop

            } else {
                # Get list of protocol endpoints in the pod
                $PEVolumes = Get-Pfa2Volume -Array $Flasharray | Where-Object {($_.Pod.Name -eq $DatastoreName) -and ($_.Subtype -eq "protocol_endpoint")}
            }
            $pod_id = $pod.id.Replace("-","")
            $scid = "vvol:"+$pod_id.substring(0,16)+"-"+$pod_id.substring(16)
        }
        Write-Verbose "Using storage container ID '$scid'..."
        $datastoreExists = Get-Datastore |Where-Object {$_.Type -eq "VVol"} |Where-Object {$_.ExtensionData.Info.VVolds.Scid -eq $scId}
        if ($null -ne $datastoreExists) {
            if ($DatastoreName -ne $datastoreExists.Name) {
                throw "A datastore '$($datastoreExists.Name)' already exists. Will not be able to create a datastore with the name '$DatastoreName."
            }
            Write-Host "A datastore already exists. Using datastore $($datastoreExists.Name)..."
            $datastore = $datastoreExists
        }

        $esxiHosts = $cluster |Get-VMHost
        foreach ($esxi in $esxiHosts)
        {
          $hostPE = Find-ProtocolEndpoint -Esxi $esxi -PEVolumes $PEVolumes -multiStorageContainersSupport $multiStorageContainersSupport -UseDefaultStore $UseDefaultStore -arrayOui $arrayOui
          $foundPE = $null -ne $hostPE

          if ($foundPE -eq $false)
          {
            if ($null -ne $datastore)
            {
                $fa = Get-PCBSConnectionOfDatastore -FlashArray $Flasharray -datastore $datastore -ErrorAction Ignore
                if ($null -eq $fa) {
                    throw "No protocol endpoints found on the host $($esxi.name) for this array. Attempt to provision a PE failed as no valid PowerShell connections found for the array. Please either provision the protocol endpoint or connect the array to an existing PE."
                }
            }

            $hGroup = Get-PCBSHostGroupfromVcCluster -cluster $cluster -flasharray $fa -ErrorAction Stop
            $allPEs = Get-Pfa2Volume -Array $Flasharray | Where-Object {$_.Subtype -eq "protocol_endpoint"}  -ErrorAction Stop
            if (($null -eq $protocolEndpoint) -or ($protocolEndpoint -eq ""))
            {
                $protocolEndpoint = "pure-protocol-endpoint"
                if ($multiStorageContainersSupport -and -not $UseDefaultStore) {
                    $ProtocolEndpoint = "$DatastoreName::$ProtocolEndpoint"
                }
            }
            $pe = $allPEs |Where-Object {$_.name -eq $protocolEndpoint}
            if ($null -eq $pe)
            {
              $pe =  New-Pfa2Volume -Array $Flasharray -Name $protocolEndpoint -Subtype "protocol_endpoint"
            }
            try
            {
                New-Pfa2Connection -Array $FlashArray -HostGroupNames $hGroup.name -VolumeNames $pe.name -ErrorAction Stop
            }
            catch
            {
              if ($_.Exception -notlike "*Connection already exists.*")
              {
                  throw $_.Exception
              }
            }

            # Refresh VASA Providers
            $provider_id = (Get-VasaStorageArray -Id ("com.purestorage:" + (Get-Pfa2Array).id)).provider.id
            Write-Host "Refreshing VASA Provider..."
            $out = Get-VasaProvider -id  $provider_id -Refresh
            Write-Host "Refreshed VASA provider $($out.Name) ($($out.Id)"
          }
          $datastoreSystem = Get-View -Id $esxi.ExtensionData.ConfigManager.DatastoreSystem
          $spec = New-Object VMware.Vim.HostDatastoreSystemVvolDatastoreSpec
          $spec.Name = $datastoreName
          $spec.ScId = $scId
          $foundDatastore = $esxi |get-datastore |Where-Object {$_.Type -eq "VVol"} |Where-Object {$_.ExtensionData.Info.VVolds.Scid -eq $scId}
          if ($null -eq $foundDatastore -or $foundDatastore.Name -ne $DatastoreName)
          {
            $datastore = Get-Datastore -Id ($datastoreSystem.CreateVvolDatastore($spec))
          }
          else {
            $datastore = $foundDatastore
          }
          $foundDatastore = $null
        }
        return $datastore
  }