JSDR.Configuration

6.0.1-preview

jetdr_avs_lib.ps1

trap
{
    Write-Host "$_"
    exit 1
}

function download_jetdr_bundle{
    param(
        $url,
        $download_file_path #"f:\tools\" + $zip_filename
    )
    $download_path = Split-Path $download_file_path
    if(!(Test-Path -path $download_path))
    {
        write-error "`n[ERROR] The path $download_path doesn't exist. Please create the folder and try again`n" -ErrorAction Stop
    }
    if(Test-Path -path $download_file_path)
    {
        Write-Host "File already exists. Skipping download."
    }
    else {
        Write-Host "### $($MyInvocation.MyCommand) ###"
        $start_time = Get-Date
        Invoke-WebRequest $url -OutFile $download_file_path -Resume
        if (!$?) {
            write-error "Downloading jetdr bundle failed, rc=$?" -ErrorAction Stop
        }
        Write-Host "Time taken: $((Get-Date).Subtract($start_time).TotalSeconds) second(s)"
    }
}

function unzip_ova {
    param(
        $zip_filename, #JSDR-GA.2.7.zip
        $destination_folder
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    # delete the extract directory if it exists
    if(Test-Path -path $destination_folder)
    {
        Write-Host "Destination directory $destination_folder already exists, deleting."
        Remove-Item $destination_folder -Recurse
    }
    try {
        Expand-Archive -Path $zip_filename -DestinationPath $destination_folder
    }
    catch {
        write-host "Unzip failed. The downloaded file may be corrupt. Deleting. Please run the script again."
        Remove-Item $zip_filename
        write-error "Unzip jetdr bundle failed, rc=$?" -ErrorAction Stop
    }
    $checksumfile = Get-ChildItem -Path $destination_folder -Filter "*checksum*txt" -Recurse | %{$_.FullName}
    $ovafile = Get-ChildItem -Path $destination_folder -Filter "*jetstream*ova" -Recurse | %{$_.FullName}
    $filehash = Get-FileHash $ovafile -Algorithm SHA256
    $checksum = Select-String -Path $checksumfile -Pattern $filehash.hash
    if ($checksum -ne $null) {
        write-host "Ova checksum validated successfully"
    }
    else{
        write-host "Checksum validation failed. The downloaded file may be corrupt. Deleting. Please run the script again."
        Remove-Item $zip_filename
        write-error "Ova checksum validation failed." -ErrorAction Stop
    }
}

function ms_rest_state {
    param(
        $ms_ip,
        $ms_user,
        $ms_pwd,
        $timeout = 600
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $rlt = 0
    while ($timeout -gt 0)
    {
        $resp_obj = get_ms_state $ms_ip $ms_user $ms_pwd
        if($resp_obj -eq $null){
            Write-Host "Unable to get MSA state, sleep for 10 seconds.."
            start-sleep 10
        }
        else
        {
            Write-Host "MSA REST server is up and running"
            $rlt = 1
            break
        }
        $timeout -= 10
    }
    return $rlt
}

function deploy_ova {
    param(
        $ovf_path,
        $signature_path,
        $ms_network,
        $ms_hostname,
        $ms_pwd,
        $ms_gateway,
        $ms_dns,
        $ms_ip,
        $ms_netmask,
        $ms_vlantag,
        $ms_cluster_name,
        $ms_vm_name,
        $ms_datastore,
        $ms_folder,
        $disk_format="Thin",
        $disable_ssh=$true
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $ovfConfig = Get-OvfConfiguration -Ovf $ovf_path
    $ovfConfig.NetworkMapping.Network_1.Value = $ms_network
    $ovfConfig.Common.hostname.Value = $ms_hostname
    $ovfConfig.Common.password.Value = $ms_pwd
    $ovfConfig.Common.disablessh.Value = $disable_ssh
    $ovfConfig.vami.JetStream_Software_Disaster_Recovery_Management_Server.gateway.Value = $ms_gateway
    $ovfConfig.vami.JetStream_Software_Disaster_Recovery_Management_Server.DNS.Value = $ms_dns
    $ovfConfig.vami.JetStream_Software_Disaster_Recovery_Management_Server.ip0.Value = $ms_ip
    $ovfConfig.vami.JetStream_Software_Disaster_Recovery_Management_Server.netmask0.Value = $ms_netmask
    $ovfConfig.vami.JetStream_Software_Disaster_Recovery_Management_Server.vlantag.Value = $ms_vlantag

    $cluster = Get-Cluster -Name $ms_cluster_name
    $disk_format = "Thin"
    $vm_name = $ms_vm_name
    $datastore = Get-Datastore -Name $ms_datastore
    $vmhost = Get-Cluster $ms_cluster_name | Get-VMHost | Select -first 1
    $ova_status = verifyHash $ovf_path $signature_path
    if ($ova_status) {
        try {
            Write-Host "Validated ova with key used to sign it. Proceeding with installation."
            Import-VApp -Source $ovf_path -OvfConfiguration $ovfConfig -Name $vm_name -vmHost $vmhost -Location $ms_cluster_name -Datastore $datastore -DiskStorageFormat $disk_format -Force -Confirm:$false | Out-Null
        }
        catch {
            write-error "Import MS ova failed, rc=$?" -ErrorAction Stop
        }
    }
    else {
        Write-Error "Could not verify the ova with the key used to sign it. Ova might be corrupt. Exiting." -ErrorAction Stop
    }

    # Moving the VM to secure folder. Import-Vapp doesn't allow -Location to be a folder
    Move-VM -VM $ms_vm_name -InventoryLocation $ms_folder -confirm:$false | Out-Null
    start-vm $ms_vm_name -confirm:$false | Out-Null
    if (!$?) {
        write-error "Starting VM $ms_vm_name failed" -ErrorAction Stop
    }

    $vm_ip=get_vm_ip $ms_vm_name 600
    return $vm_ip
}

function ping_vm
{
    param(
        $vm_ip = $( throw "specify vm ip" ),
        $timeout = 600
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"

    $rlt = 0
    while ($timeout -gt 0)
    {
        $rc = test-connection -computername $vm_ip -quiet
        if (!$rc)
        {
            Write-Host "$vm_ip is not reachable, sleep for 5 seconds.."
            start-sleep 5
        }
        else
        {
            Write-Host "$vm_ip is pingable now."
            $rlt = 1
            break
        }
        $timeout -= 5
    }
    return $rlt
}

function get_vm_ip {
    param(
        $vm_name = $(throw "vmName is required"),
        $timeout = 600
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"

    while ($timeout -gt 0) {
        $vm_ip = (Get-VM $vm_name).Guest.IPAddress[0]
        if (($null -ne $vm_ip) -and ($vm_ip -match "\d+\.\d+\.\d+\.\d+")) {
            $timeout = 0
        } else {
            $timeout -= 30
            Write-Host "couldn't get ip of $vm_name. Sleep for 30 seconds"
            Start-Sleep 30
        }
    }
    if (($null -eq $vm_ip) -or ($vm_ip -notmatch "\d+\.\d+\.\d+\.\d+")) {
        write-error "Failed getting ip of vm $vm_name"
        $vm_ip = $null
    }
    return $vm_ip
}

function create_user {
    param(
        [String]$user, # e.g "testuser"
        [String]$pwd
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $user=New-SsoPersonUser -UserName $user -Password $pwd
    if(!$user) {
        write-error "Failed to create user $user" -ErrorAction Stop
    }
}

function delete_user {
    param(
        [String]$user
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $domain = $user.split("\")[0]
    $username = $user.split("\")[1]
    $ssouser = Get-SsoPersonUser -Name $username -Domain $domain
    Remove-SsoPersonUser -User $ssouser[0]
}

function assign_new_role_to_user {
    param(
        [String]$new_role_name,
        [String]$sso_user_name, # e.g "vsphere.local\testuser"
        [String[]]$new_privileges # e.g "Maintenance", "Query patch", "CIM interaction", "Profile-driven storage update"
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $existing_role = "CloudAdmin"

    # chk if Cloudadmin role exists
    $cloudadmin_role = Get-VIRole | Where-Object {$_.Name -eq $existing_role}

    $cloudadmin_privileges = Get-VIPrivilege -Role $existing_role

    # check if new role already exists
    $new_role = Get-VIRole | Where-Object {$_.Name -eq $new_role_name}
    if ($new_role -ne $null) {
        #remove_role $new_role_name
        Write-Host "Role $new_role_name already exists in vCenter. Skipping creation"
    }
    else{
        # create a new role with cloudadmin privileges
        $my_role = New-VIRole -Privilege $cloudadmin_privileges -Name $new_role_name
        if (!$?) {
            write-error "Failed to create new role ${new_role_name}, rc=$?" -ErrorAction Stop
        }
    }
    $my_role = Get-VIRole | Where-Object {$_.Name -eq $new_role_name}
    # add new privileges to this new role
    $priv = @()
    Foreach ($cust_priv in $new_privileges){
        $priv += Get-VIPrivilege | Where-Object {$_.Name -eq $cust_priv}
    }

    Set-VIRole -Role $my_role -AddPrivilege $priv
    if (!$?) {
        write-error "Failed to add new privileges ${priv} to role ${my_role}, rc=$?" -ErrorAction Stop
    }

    # get root folder and then grant $sso_user_name access to the role on rootfolder level
    Write-Host "assign permissions"
    $rootfolder = Get-Folder -NoRecursion
    $new_permission = New-VIPermission -Role $new_role_name -Principal $sso_user_name -Entity $rootfolder
    if (!$?) {
        write-error "Failed to grant user ${sso_user_name} to role ${new_role_name}, rc=$?" -ErrorAction Stop
    }
    Write-Host "new permissions type: " + $new_permission
    return $new_permission
}

function remove_role {
    param(
        $role_name
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Removing role $role_name"
    # remove a role
    Remove-VIRole $role_name -confirm:$false -Force
    if (!$?) {
        write-error "Failed to remove role ${role_name}, rc=$?" -ErrorAction Stop
    }
    return
}

function remove_permissions {
    param($permissions)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Remove-VIPermission $permissions -confirm:$false
    if (!$?) {
        write-error "Failed to remove permissions ${permissions}, rc=$?" -ErrorAction Stop
    }
}

function generate_password {
    param(
        [ValidateRange(8, 20)]
        [int]
        $length = 14
    )
    $symbols = '!@#$%^&*'.ToCharArray()
    $characterList = 'a'..'z' + 'A'..'Z' + '0'..'9' + $symbols
    do {
        $password = -join (0..$length | % { $characterList | Get-Random })
        [int]$hasLowerChar = $password -cmatch '[a-z]'
        [int]$hasUpperChar = $password -cmatch '[A-Z]'
        [int]$hasDigit = $password -match '[0-9]'
        [int]$hasSymbol = $password.IndexOfAny($symbols) -ne -1

    }
    until (($hasLowerChar + $hasUpperChar + $hasDigit + $hasSymbol) -ge 4)

    return $password
}

function get_request {
    param(
        $endpoint,
        $header,
        $outfile="get_request_output.csv"
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    try {
        $resp_obj = Invoke-RestMethod -Method 'Get' -Uri $endpoint -Headers $header -skipCertificateCheck
    } catch {
        # Dig into the exception to get the Response details.
        # Note that value__ is not a typo.
        $resp_obj = $_.Exception.Response.StatusCode.value__
    }
    return $resp_obj
}

function get_dr_header {
    param($fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $header = @{
        'Accept' = "application/json"
        'content-type' = 'application/json'
        'fios-session' = $fios_session_id
    }
    return $header
}

function post_request {
    param(
        $endpoint,
        $header,
        $body,
        $outfile="post_request_output.csv"
    )

    try {
        $resp_obj = Invoke-RestMethod -Method 'Post' -Uri $endpoint -Headers $header -Body $body -skipCertificateCheck
    } catch {
        $_.Exception | format-list -force
    }
    return $resp_obj
}

function put_request {
    param(
        $endpoint,
        $header,
        $body
    )

    try {
        $resp_obj = Invoke-RestMethod -Method 'PUT' -Uri $endpoint -Headers $header -Body $body -skipCertificateCheck
    } catch {
        $_.Exception | format-list -force
    }
    return $resp_obj
}

function delete_request {
    param(
        $endpoint,
        $header
    )

    try {
        $resp_obj = Invoke-RestMethod -Method 'DELETE' -Uri $endpoint -Headers $header -skipCertificateCheck
    } catch {
        $_.Exception | format-list -force
    }
}

function get_encoded_auth {
    param($user, $pwd)
    $plain_str = "${user}:${pwd}"
    $bytes = [System.Text.Encoding]::UTF8.GetBytes($plain_str)
    $encoded_text =[Convert]::ToBase64String($bytes)
    $encoded_text
}

function get_vcenter_session {
    param($ms_ip, $vcenter_fqdn, $base64_cred)

    # https://<MS hostname>/jss/domains
    $endpoint = 'https://' + $ms_ip + '/jss/vmplatform/' + $vcenter_fqdn + "/loginByAddress"
    $header = @{
        'Accept' = "application/json"
        'Authorization' = 'Basic ' + $base64_cred
    }

    $resp_obj = get_request $endpoint $header
    #$resp_obj | Out-String | Write-Host
    if ($resp_obj -match '=\"(.*?)\"') {
        $fios_session = $matches[1]
    } else {
        write-error "Failed to get vcenter session id. Please check if MSA is registered to vCenter and credentials are correct." -ErrorAction Stop
    }
    return $fios_session
}


function refresh_vc_session {
    param($ms_ip, $vc_hostname, $vc_user, $vc_pwd)
    $vc_base64 = get_encoded_auth $vc_user $vc_pwd
    $fios_session_id = get_vcenter_session $ms_ip $vc_hostname $vc_base64
    return $fios_session_id
}


function login_ms_portal {
    param($ms_ip, $ms_user, $ms_pwd)
    $timeout = 600
    $ms_base64 = get_encoded_auth $ms_user $ms_pwd
    $request_url = 'https://' + $ms_ip + '/jss/ms/login'
    $header = @{
        "Authorization" = "Basic " + $ms_base64
    }
    while ($timeout -gt 0) {
        try {
            $resp_obj = get_request $request_url $header
            $session_id = $resp_obj.loginResponse.sessionId
        } catch {
            $_.Exception |format-list -force
        }
        if ($session_id -ne $null) {
            return $session_id
        } else {
            Write-host "Waiting get MSA session id. Sleep for 1 minute and retry"
            $timeout -= 60
            start-sleep 60
        }
    }
    if ($session_id -eq $null) {
        Write-Error "Failed to get MSA session id after 10 minutes. Please check your MSA" -ErrorAction Stop
    }
}

function register_ms_with_vc {
    param($ms_ip, $ms_user, $ms_pwd, $vc_hostname, $vc_user, $vc_pwd, $ms_mode, $register_with_ip)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Registering vCenter $vc_hostname to MS $ms_ip with user $vc_user"
    $endpoint = 'https://' + $ms_ip + '/jss/vmplatform'
    $vc_base64 = get_encoded_auth $vc_user $vc_pwd
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }
    $body = @{
        "address" = $vc_hostname
        "credential" = $vc_base64
        "msMode" = $ms_mode
        "registerWithMsIp" = $register_with_ip
    }
    $json = $body | Convertto-JSON
    $resp_obj = post_request $endpoint $header $json
    if ($resp_obj.id -eq $null) {
        write-error "Post request to Register MS with VC failed" -ErrorAction Stop
    }
}

function unregister_vcenter_from_ms {
    param($ms_ip, $vc_hostname, $vc_user, $vc_pwd, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Unregistering vCenter $vc_hostname from MS $ms_ip"
    $endpoint = 'https://' + $ms_ip + '/jss/vmplatform/' + $vc_hostname + "/unregisterByAddress"
    $vc_base64 = get_encoded_auth $vc_user $vc_pwd
    $body = @{
        "credential" = $vc_base64
    }
    $json = $body | Convertto-JSON
    $header = get_dr_header($fios_session_id)
    $resp_obj = post_request $endpoint $header $json
}

function force_unregister_ms {
    $extMgr = Get-View ExtensionManager
    $jsdrplugin = $extMgr.ExtensionList | select key
    if ($jsdrplugin.Key -contains "com.jetstream.primary.jetdrplugin") {
        Write-Host "Forcefully unregistering MSA plugin from vCenter"
        $extMgr.UnregisterExtension("com.jetstream.primary.jetdrplugin")
    }
}

function get_drvas {
    param($ms_ip, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $endpoint = 'https://' + $ms_ip + '/jss/drvas'
    $header = get_dr_header($fios_session_id)
    $resp_obj = get_request $endpoint $header
    if ($resp_obj -eq $null) {
        Write-Host "No DRVA is configured."
    }
    else {
        Write-Error "DRVA(s) are still configured. Please unconfigure and try again" -ErrorAction Stop
    }
}

function get_site_dc_state {
    param(
        $ms_ip, $fios_session_id
    )

    $endpoint = 'https://' + $ms_ip + "/jss/datacenters"
    $header = get_dr_header($fios_session_id)
    $primary_site_registered_vc = get_request $endpoint $header
    Write-Host "request_url: $endpoint"
    return $primary_site_registered_vc
}

function get_cluster_state {
    param(
        $ms_ip, $fios_session_id, $cluster_name
    )

    $primary_site_dcs = get_site_dc_state $ms_ip $fios_session_id
    foreach ($dc in $primary_site_dcs) {
        foreach ($cluster in $dc.clusters) {
            if ($cluster.clusterName.ToLower() -eq $cluster_name.ToLower()) {
                $cluster_id = $cluster.id
            }
        }
    }

    $endpoint = 'https://' + $ms_ip + "/jss/clusters/" + $cluster_id
    $header = get_dr_header($fios_session_id)
    $cluster_state = get_request $endpoint $header
    Write-Host "request_url: $endpoint"
    return $cluster_state

}

function get_dr_task_by_id {
    param($ms_ip, $fios_session_id, $task_id)
    $endpoint = 'https://' + $ms_ip + "/jss/tasks/" + $task_id
    $header = get_dr_header($fios_session_id)
    $response = get_request $endpoint $header
    return $response
}

function wait_for_dr_task_finish
{
    param($ms_ip, $fios_session_id, $task_id)
    Write-Host "Task id: $task_id"
    $dr_task = get_dr_task_by_id $ms_ip $fios_session_id $task_id
    $task_state = $dr_task.state
    $task_progress = $dr_task.progress
    $task_description = $dr_task.description
    while ($task_progress -ne "100")
    {
        Write-Host "State: $task_state"
        Write-Host "Progress: $task_progress"
        Write-Host "Sleep for 30 seconds and wait for task $task_description to finish"
        start-sleep 30
        $dr_task = get_dr_task_by_id $ms_ip $fios_session_id $task_id
        $task_state = $dr_task.state
        $task_progress = $dr_task.progress
    }
    $err_code = $dr_task.errorCode
    if ($err_code -eq 0) {
        Write-Host "Task $task_description finished successfully: $task_state"
        return 0
    } else {
        throw "Task $task_description finished w/ error: $err_code $dr_task.errorMessage"
    }
}

function configure_cluster {
    param($ms_ip, $cluster_name, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $retry_count=0
    $cluster_params = @()
    $endpoint = 'https://' + $ms_ip + "/jss/clusters/configure"
    $primary_site_dcs = get_site_dc_state $ms_ip $fios_session_id
    foreach ($dc in $primary_site_dcs) {
        foreach ($cluster in $dc.clusters) {
            if ($cluster.clusterName.ToLower() -eq $cluster_name.ToLower()) {
                $params = @{
                    "clusterId" = $cluster.id
                    "packageType" = "DR"
                }
                $cluster_params += $params
            }
        }
    }
    $header = get_dr_header($fios_session_id)
    $body = @{
        "clustersConfigureParams" = $cluster_params
    }
    $json = $body | Convertto-JSON
    do{
        $task = post_request $endpoint $header $json
        if ($task -eq $null) {
            Write-Error "Post request to configure cluster failed" -ErrorAction Continue
            $retry_count++
            #Sleeping for 15 secs before retrying
            start-sleep 15
            continue
        }
        else {
            try {
                # wait for cluster configuration finish before we can remove the elevated privileges
                $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
            }
            catch {
                Write-Error "Configure cluster failed. Retrying" -ErrorAction Continue
                $retry_count++
                # Sleeping for 15 secs before retrying
                start-sleep 15
            }
        }
    }while(($retry_count -lt 3) -and ($status -ne 0))
    if (($retry_count -eq 3) -and ($status -ne 0)){
        Write-Error "ERROR: Configure Cluster failed. Please check MS task logs, resolve the issues and run again." -ErrorAction Stop
    }
    return $status
}

function unconfigure_cluster {
    param($ms_ip, $cluster_name, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $retry_count=0
    $cluster_params = @()
    $endpoint = 'https://' + $ms_ip + "/jss/clusters/unconfigure"
    $primary_site_dcs = get_site_dc_state $ms_ip $fios_session_id
    $cluster_state=get_cluster_state $ms_ip $fios_session_id $cluster_name
    if ($cluster_state.software.name -eq 'jetdr') {
        Write-Host "Cluster $cluster_name has JetStream iofilter installed. Checking if there are 4 hosts in the custer in order to proceed with uninstall"
        $HostCount=Get-Cluster -Name $cluster_name -ErrorAction SilentlyContinue | Get-VMHost
        if ($HostCount.count -lt 4) {
            Write-Error "Cluster to be unconfigured has less than 4 hosts. Please make sure to have at least 4 hosts in protected cluster" -ErrorAction Stop
        }
        else {
            Write-Host "SUCCESS: Protected cluster satisfies the number of host requirement"
        }
    foreach ($dc in $primary_site_dcs) {
        foreach ($cluster in $dc.clusters) {
            if ($cluster.clusterName.ToLower() -eq $cluster_name.ToLower()) {
                if ($cluster.domainCnt -ne 0) {
                    Write-Error "ERROR: Cannot unconfigure cluster as it is used by a Protected Domain" -ErrorAction Stop
                }
                $params = @{
                    "clusterId" = $cluster.id
                }
                $cluster_params += $params
            }
        }
    }
    $header = get_dr_header($fios_session_id)
    $body = @{
        "clustersUnconfigureParams" = $cluster_params
    }
    $json = $body | Convertto-JSON
    do{
        $task = post_request $endpoint $header $json
        if ($task -eq $null) {
            Write-Error "Post request to unconfigure cluster failed" -ErrorAction Continue
            $retry_count++
            #Sleeping for 15 secs before retrying
            start-sleep 15
            continue
        }
        else {
            try {
                # wait for cluster unconfigure to finish before we can remove the elevated privileges
                $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
            }
            catch {
                Write-Error "Unconfigure cluster task failed. Retrying" -ErrorAction Continue
                $retry_count++
                # Sleeping for 15 secs before retrying
                start-sleep 15
            }
        }
    }while(($retry_count -lt 3) -and ($status -ne 0))
    if (($retry_count -eq 3) -and ($status -ne 0)){
        Write-Error "ERROR: Unconfigure Cluster failed. Please check MS task logs, resolve the issues and run again." -ErrorAction Stop
    }
    return $status
    }
    else {
        Write-Host "Cluster doesn't have jetdr iofilter installed. Skipping unconfigure cluster"
    }
}

function get_ms_state {
    param($ms_ip, $ms_user, $ms_pwd)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }

    $endpoint = 'https://' + $ms_ip + "/jss/ms"
    $response = get_request $endpoint $header
    if (!$response) {
        Write-Host "MSA state api didn't return anything."
    }
    else {
        if($response.configured -eq $False) {
            Write-Host "MSA is not registered to any vCenter server"
        }
        else {
            Write-Host "MSA is registered"
        }
    }
    return $response
}

function get_plugin_upgrade_status {
    param (
        $ms_ip,
        $ms_user,
        $ms_pwd
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $plugin_upgrade_status = $false
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }

    $endpoint = 'https://' + $ms_ip + "/jss/vmplatform"
    $response = get_request $endpoint $header
    if (!$response) {
        Write-Host "MSA state api didn't return anything."
    }
    else {
        if ($response.availableExtension) {
            write-host "Extension available for MSA to upgrade"
            $plugin_upgrade_status = $true
        }
        else {
            write-host "Extension not available for MSA to upgrade"
        }
    }
    return $plugin_upgrade_status
}

function get_system_state {
    param($ms_ip, $ms_user, $ms_pwd, $hostname, $dns)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $ms_up = ping_vm $ms_ip 5
    if (!$ms_up) {
        Write-Host "MSA ip $ms_ip not pingable."
        return
    }
    else {
        Write-Host "MSA is pingable."
        get_ms_state $ms_ip $ms_user $ms_pwd
    }
    $rc = ping_vm $dns 5
    if (!$rc) {
        Write-Host "ERROR: Dns server $dns not pingable."
        return
    }
    else {
        Write-Host "Dns server $dns is pingable"
    }
    $check_hostname=test-connection $hostname
    $var=$check_hostname[0].Address
    $var | Out-String | Write-Host
    Write-Host "DNS IP address for your Management Server appliance $hostname is $var"
    if ($check_hostname[0].Address.IPAddressToString -eq $ms_ip) {
        Write-Host "DNS IP address $var resolves to hostname $hostname"
    }
    else{
        Write-Host "ERROR: DNS IP address $var doesn't resolve to hostname $hostname"
    }
}

function get_system_state_install {
    param($msvm_name, $msa_cluster, $protected_cluster, $datastore, $network)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $error=$False
    $rc=Get-Cluster -Name $protected_cluster -ErrorAction SilentlyContinue
    if (!$rc) {
        Write-Error "Specified cluster '$protected_cluster' does not exist in the Datacenter. Please provide correct details."
        $error=$True
    }
    else {
        Write-Host "SUCCESS: Cluster '$protected_cluster' provided for protection exists in the Datacenter"
        $HostCount=Get-Cluster -Name $protected_cluster -ErrorAction SilentlyContinue | Get-VMHost
        if ($HostCount.count -lt 3) {
            Write-Error "Cluster to be protected has less than 3 hosts. Please make sure to have at least 3 hosts in protected cluster"
            $error=$True
        }
        else {
            Write-Host "SUCCESS: Protected cluster satisfies the number of host requirement"
        }
    }
    $rc =  getVasaProvider $protected_cluster
    if ($rc -eq 1) {
        $error=$True
    }
    $rc=Get-Cluster -Name $msa_cluster -ErrorAction SilentlyContinue
    if (!$rc) {
        Write-Error "Specified cluster '$msa_cluster' for deploying MSA does not exist in the Datacenter. Please provide correct details."
        $error=$True
    }
    else {
        Write-Host "SUCCESS: Cluster '$msa_cluster' provided for deploying MSA exists in the Datacenter"
    }

    $rc=Get-VM $msvm_name -ErrorAction SilentlyContinue
    if ($rc) {
        Write-Error "VM with name '$msvm_name' already exists in datacenter. Please remove the VM before proceeding"
        $error=$True
    }
    else{
        Write-Host "SUCCESS: MSA vm doesn't exist in datacenter."
    }

    # Check if provided datastore exists in the datacenter
    $ds_obj=Get-DataStore $datastore -ErrorAction SilentlyContinue
    if ($ds_obj -eq $null) {
        Write-Error "Specified datastore $datastore not found in the datacenter. Please provide a correct datastore name."
        $error = $True
    }
    else {
        $ds_space_gb=$ds_obj.FreeSpaceGB
        if ($ds_space_gb -lt 60) {
            Write-Error "Datastore provided for MS deployment doesn't have enough free space for the deployment. Minimum required 60GB"
            $error = $True
        }
        else {
            Write-Host "SUCCESS: Datastore has enough free space for JetDR ova deployment."
        }
    }

    # Check if provided network exists in the datacenter
    $nw_obj = Get-VirtualNetwork $network -ErrorAction SilentlyContinue
    if ($nw_obj -eq $null) {
        Write-Error "Specified network name $network not found in the datacenter. Please provide a correct network name"
        $error = $True
    }
    else {
        Write-Host "SUCCESS: Network provided for MSA is available."
    }

    $extMgr = Get-View ExtensionManager
    $jsdrplugin=$extMgr.ExtensionList | select Key
    if ($jsdrplugin.Key -contains "com.jetstream.primary.jetdrplugin") {
        Write-Error "VCenter already has JetDR plugin. Please unregister vCenter or cleanup any stale plugins before continuing."
        $error=$True
    }
    else {
        Write-Host "SUCCESS: VCenter doesn't have JetDR plugin."
    }

    if ($error -eq $true) {
        Write-Error "There were errors in the preflight check. Please resolve the errors and try again" -ErrorAction Stop
    }
    else {
        Write-Host "SUCCESS: Preflight checks succeeded."
    }
}

function get_system_state_configure {
    param($protected_cluster)
    $rc=Get-Cluster -Name $protected_cluster -ErrorAction SilentlyContinue
    if (!$rc) {
        Write-Error "Specified cluster '$protected_cluster' does not exist in the Datacenter. Please provide correct details."
        $error=$True
    }
    else {
        Write-Host "SUCCESS: Cluster '$protected_cluster' provided for protection exists in the Datacenter"
        $HostCount=Get-Cluster -Name $protected_cluster -ErrorAction SilentlyContinue | Get-VMHost
        if ($HostCount.count -lt 3) {
            Write-Error "Cluster to be protected has less than 3 hosts. Please make sure to have at least 3 hosts in protected cluster"
            $error=$True
        }
        else {
            Write-Host "SUCCESS: Protected cluster satisfies the number of host requirement"
        }
    }
    $rc =  getVasaProvider $protected_cluster
    if ($rc -eq 1) {
        $error=$True
    }

    $extMgr = Get-View ExtensionManager
    $jsdrplugin=$extMgr.ExtensionList | select Key
    if ($jsdrplugin.Key -contains "com.jetstream.primary.jetdrplugin") {
        Write-Host "SUCCESS: VCenter is registered to JetDR MSA."
    }
    else {
        Write-Host "ERROR: VCenter is not registered to any JetDR MSA. Please make sure VCenter is registered before trying configure"
        $error=$True
    }

    if ($error -eq $true) {
        Write-Error "There were errors in the preflight check. Please resolve the errors and try again" -ErrorAction Stop
    }
    else {
        Write-Host "SUCCESS: Preflight checks succeeded."
    }
}

function get_system_state_uninstall {
    param($ms_ip, $protected_cluster, $ms_user, $ms_pwd)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $error=$False
    $rc=Get-Cluster -Name $protected_cluster -ErrorAction SilentlyContinue
    if (!$rc) {
        Write-Error "Specified cluster '$protected_cluster' does not exist in the Datacenter. Please provide correct details."
        $error=$True
    }
    else {
        Write-Host "SUCCESS: Cluster '$protected_cluster' provided exists in the Datacenter"
    }

    $ms_up=ms_rest_state $ms_ip $ms_user $ms_pwd 60
    if ($ms_up) {
        Write-Host "JetStream MSA is up and running"
        $response=get_ms_state $ms_ip $ms_user $ms_pwd
        if($response.configured -eq $True) {
            Write-Host "SUCCESS: VCenter is registered to MSA $ms_ip."
        }
        else {
            Write-Error "VCenter is not registered to any MSA. Cannot proceed."
            $error=$true
        }
    }
    else {
        Write-Error "JetStream MSA is not accessible. Please make sure it is running and try again"
        $error=$true
    }
    if ($error -eq $true) {
        Write-Error "There were errors in the preflight check. Please resolve the errors and try again" -ErrorAction Stop
    }
    else {
        Write-Host "SUCCESS: Preflight check succeeded. Proceeding."
    }
}

function restart_cim {
    param($cluster)
    $vcenter_8 = is_vcenter_8
    if (!$vcenter_8) {
        Write-Host "### $($MyInvocation.MyCommand) ###"
        Write-Host "Restarting CIM"

        $hosts=Get-Cluster $cluster | Get-VMHost
        foreach ($ESXhost in $hosts){
            Get-VMHost -name $ESXhost | Get-VMHostService | where {$_.key -eq 'sfcbd-watchdog'} | Stop-VMHostService -Confirm:$false
            Get-VMHost -name $ESXhost | Get-VMHostService | where {$_.key -eq 'sfcbd-watchdog'} | Start-VMHostService -Confirm:$false
        }
    }
    else {
        Write-Host "CIM restart is not required for vSphere-8"
    }
}

function restart_jetdr {
    param($cluster)
    Write-Host "### $($MyInvocation.MyCommand) ###"

    $hosts=Get-Cluster $cluster | Get-VMHost
    foreach ($ESXhost in $hosts) {
        try {
            $host_with_jetdr = Get-VMHost -name $ESXhost | Get-VMHostService | where {$_.key -eq 'jetdr'}
            if ($host_with_jetdr -eq $null) {
                Write-Error "jetdr daemon not found on host $ESXhost" -ErrorAction Continue
            }
            else {
                try {
                    Write-Host "Restarting jetdr daemon on host $ESXhost"
                    $host_with_jetdr | Restart-VMHostService -Confirm:$false
                }
                catch {
                    Write-Error "Could not restart host services. $ESXhost might not be reponsive" -ErrorAction Continue
                }
            }
        }
        catch {
            Write-Error "Could not get host services. $ESXhost might not be reponsive" -ErrorAction Continue
        }
    }
    Write-Host "Script execution has completed. Please check status of the task for success/failure"
}

function test_jetdr_conn {
    param($ms_ip)
    Write-Host "### $($MyInvocation.MyCommand) ###"

    $agent_ip = ip -4 -o addr | grep -v "127.0.0.1" | awk '{print $4}'
    Write-Host "Run Command Agent ip: $agent_ip"
    $endpoint = 'https://' + $ms_ip + '/ms/pages/ms/ms.html'
    Write-Host "Checking connectivity to MSA $ms_ip"
    try {
        $resp_obj = Invoke-RestMethod -Method 'Get' -Uri $endpoint -skipCertificateCheck
        if ($resp_obj.Contains("jetdrplugin")) {
            Write-Host "Able to connect to MSA $ms_ip"
        }
        else {
            throw "Unexpected reply on ip $ms_ip"
        }
    } catch {
        $_.Exception.message
        Write-Error "Invalid or no response from MSA $ms_ip"
    }

    Write-Host "Checking if Run Cmd infra can connect to MSA on port 443"
    $conn_to_msa = Test-Connection $ms_ip -TcpPort 443
    if ($conn_to_msa) {
        Write-Host "Port 443 to MSA $ms_ip is open"
    } else {
        Write-Error "Port 443 to MSA $ms_ip is closed. Please open the port and try again"
    }
    Write-Host "Script execution has completed. Please check status of the task for success/failure"
}

function checkPSVersion {
    # This function checks whether Powershell version in the system is greatear than 7 or not
    $PowerShellVersion = $PSVersionTable.PSVersion.Major
    if($PowerShellVersion -lt 7) {
        write-error "Powershell version is less than 7. Please install powershell version 7.x and run again." -ErrorAction Stop
    }
    else {
        Write-Host "SUCCESS: Check for powershell version passed"
    }
}

function checkRole {
    # Checks for the presence of a role in the vCenter
    param(
          [String]$role
    )
    $role_exists = Get-VIRole | Where-Object {$_.Name -eq $role}
    if ($role_exists) {
        Write-Host "SUCCESS: Check for $role role in vCenter passed"
    }
    else {
        write-error "$role role does not exist in the VCenter. Please make sure it exists before we can proceed with the configuration" -ErrorAction Stop
    }
}

function checkModule {
    param(
          [String]$module
    )

    $module_exists=Get-Module -Name $module
    if($module_exists) {
        Write-Host "SUCCESS: Check for module $module passed"
    }
    else {
        write-error "Module $module not available. Please make sure the module is imported and try again." -ErrorAction Stop
    }
}

function checkVcaddress {
    if (-Not $VC_ADDRESS) {
        Write-Error "VC_ADDRESS is not set as a powershell variable. Please set it and run the script again" -ErrorAction Stop
    }
    else {
        Write-Host "SUCCESS: VCenter Address is set."
    }
}

function getVcFqdn {
    $view = Get-View OptionManager-VpxSettings
    $fqdn = $view.setting | Where-Object { $_.Key  -eq  'VirtualCenter.fqdn'}
    if (($null -ne $fqdn) -or ($null -ne $fqdn.value)) {
        $vc_fqdn = $fqdn.value
    }
    else {
        $vc_fqdn = [System.Net.Dns]::GetHostByName("vc").HostName
    }
    if ($null -ne $vc_fqdn) {
        return $vc_fqdn
    }
    else {
        write-error "Could not determine vCenter fqdn. Cannot proceed." -ErrorAction Stop
    }
}

function get_registered_vcenter {
    param(
        $ms_ip,
        $ms_user,
        $ms_pwd
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $endpoint = 'https://' + $ms_ip + '/jss/vmplatform/'
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }
    $response = get_request $endpoint $header
    if (!$response.address) {
        $vcenter = $null
        Write-Host "Couldn't get registered vCenter details"
    }
    else {
        $vcenter = $response.address
        Write-Host "$vcenter is registered to MSA $ms_ip"
    }
    return $vcenter
}

function getVasaProvider {
    param(
          [String]$cluster
    )
    $vasacount=0
    $error=0
    $vasaprovider = Get-VasaProvider -Server $VC_ADDRESS
    $hosts=Get-Cluster -Name $cluster -ErrorAction SilentlyContinue | Get-VMHost
    $hostcount = $hosts.count

    # Checking if storage provider is available for each host
    ForEach ($vasa in $vasaprovider) {
        ForEach ($hostname in $hosts.name) {
            if ($vasa.name.split() -Contains $hostname -And $vasa.namespace -eq "iofilters") {
                $vasacount++
                if ($vasa.status -eq "online") {
                    write-host "SUCCESS: Storage Provider for $hostname is Online"
                }
                else {
                    write-error "Iofilter Storage Provider for $hostname is not Online. Please synchronize storage providers and make sure they are online before trying again"
                    $error=1
                }
            }
        }
    }
    if ($vasacount -ne $hostcount){
        write-error "Iofilter Storage providers for all hosts in the cluster to be protected is not available. Please synchronize and make sure they are present and online before trying again."
        $error=1
    }
    return $error
}

function updateMSA {
    param(
        [String]$ms_ip,
        [String]$user,
        [String]$pwd,
        [String]$datacenter,
        [String]$datastore,
        [String]$msa_vm,
        [String]$iso_path,
        [String]$signature_path
    )

    $artifact_status = verifyHash $iso_path $signature_path
    if ($artifact_status) {
        Write-Host "Validated upgrade iso with key used to sign it. Mounting iso to MSA."
        mount_iso_to_msa $msa_vm $datacenter $datastore $iso_path
    }
    else {
        Write-Error "Could not verify the iso with the key used to sign it. Upgrade iso might be corrupt. Skipping uprade." -ErrorAction Stop
    }
    write-host "Wait for 1 minute"
    start-sleep 60
    try {
        $check_upgrade = Invoke-VMScript -VM $msa_vm -ScriptText 'js_upgrade.py check -v' -GuestUser $user -GuestPassword $pwd
        if ( $check_upgrade.ExitCode -ne 0 ) {
            write-error $check_upgrade.ScriptOutput
            write-error "Check upgrade failed. Exiting" -ErrorAction Stop
        }
        else {
            write-host "Upgrading MSA from upgrade iso"
            $perform_upgrade = Invoke-VMScript -VM $msa_vm -ScriptText 'js_upgrade.py upgrade -v' -GuestUser $user -GuestPassword $pwd
            if ( $perform_upgrade.ExitCode -ne 0 ) {
                write-error $perform_upgrade.ScriptOutput
                write-error "MSA upgrade failed. Exiting" -ErrorAction Stop
            }
            else {
                write-host "MSA upgraded succesfully. Upgrading plugin."
                start-sleep 10
                upgrade_vcenter_plugin $ms_ip $user $pwd
            }
        }
    }
    catch {
        Write-Error "Upgrading MSA failed. Please contact JetStream support for assistance." -ErrorAction Stop
    }
}

function mount_iso_to_msa {
    param(
        [String]$msa_vm,
        [String]$datacenter,
        [String]$datastore,
        [String]$isoPath
    )
    $upgrade_iso = Split-Path -Leaf $isoPath
    $ds = Get-Datastore $datastore
    New-PSDrive -Location $ds -Name DS -PSProvider VimDatastore -Root "\" | Out-Null
    $upgrade_iso_folder = Get-Item -Path DS:\js_upgrade_iso -ErrorAction SilentlyContinue
    if ($upgrade_iso_folder -eq $null) {
        New-Item -Path DS:\js_upgrade_iso
    }
    $ds_browser = Get-View $ds.ExtensionData.Browser
    $ds_spec = new-object VMware.Vim.HostDatastoreBrowserSearchSpec
    $folder_file_query = New-Object Vmware.Vim.FolderFileQuery
    $ds_spec.query = $folder_file_query
    $file_query_flags = New-Object VMware.Vim.FileQueryFlags
    $file_query_flags.fileOwner = $false
    $file_query_flags.fileSize = $false
    $file_query_flags.fileType = $true
    $file_query_flags.modification = $false
    $ds_spec.details = $file_query_flags
    $ds_spec.sortFoldersFirst = $true
    $results = $ds_browser.SearchDatastore("[$($ds.Name)]", $ds_spec)

    $files = $results.file
    foreach ($file in $files) {
        if(($file.getType().Name -eq "FolderFileInfo") -And (($file.Path -eq "js_upgrade_iso") -Or ($file.FriendlyName -eq "js_upgrade_iso"))) {
            $folder_path = $results.FolderPath.Trim("[]") + "\" + $file.Path
        }
    }
    try {
        Copy-DatastoreItem -Item $isoPath vmstore:\$datacenter\$folder_path

        $update_iso=Get-ChildItem -Path vmstore:\$datacenter\$folder_path\$upgrade_iso
        $update_path = $update_iso.DataStoreFullPath
        Get-VM -Name $msa_vm | Get-CDDrive | Set-CDDrive -Connected $true -IsoPath $update_path -Confirm:$false
    }
    catch {
        Write-Error "Failed to mount upgrade iso to Management Server Appliance." -ErrorAction Stop
    }
}

function upgrade_vcenter_plugin {
    param($ms_ip, $ms_user, $ms_pwd)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $vc_hostname = get_registered_vcenter $ms_ip $ms_user $ms_pwd
    Write-Host "Upgrading vCenter plugin on $vc_hostname from MS $ms_ip"
    $endpoint = 'https://' + $ms_ip + '/jss/vmplatform/' + $vc_hostname + "/upgradeByAddress"
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }
    $response = post_request $endpoint $header
    if($response -eq $null) {
        Write-Error "Failed to upgrade MSA UI plugin. Please contact JetStream support for assistance" -ErrorAction Stop
    }
}

function upgrade_drvas {
    param($ms_ip, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Checking for DRVA(s) with available upgrade"
    $endpoint = 'https://' + $ms_ip + '/jss/drvas'
    $header = get_dr_header($fios_session_id)
    $drvas = get_request $endpoint $header
    foreach ($drva in $drvas) {
        $drva_name = $drva.name
        if ($drva.availableSoftware -eq $null) {
            Write-Host "No upgrade available for DRVA $drva_name. Skipping"
            continue
        }
        $upgrade_version = $drva.availableSoftware.version
        $drva_id = $drva.id
        Write-Host "$drva_name has an upgrade available"
        $endpoint = 'https://' + $ms_ip + '/jss/drvas/' + $drva_id + '/upgrade'
        $body = @{
            "version" = $upgrade_version
        }
        $json = $body | ConvertTo-Json
        $task = post_request $endpoint $header $json
        if ($task -eq $null) {
            # We don't want to continue upgrade if it fails at any point.
            # Customer should check the cause of failure, fix it and run again.
            Write-Error "Post request to upgrade DRVA $drva_name failed" -ErrorAction Stop
        }
        else {
            try {
                # wait for DRVA upgrade to finish
                $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
            }
            catch {
                Write-Error "Upgrade DRVA $drva_name failed. Please check JetStream UI task logs and re-run the cmdlet with Resume as True" -ErrorAction Stop
            }
        }
    }
}

function upgrade_recovery_vas {
    param($ms_ip, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Checking for recovery virtual appliances with available upgrade"
    $endpoint = 'https://' + $ms_ip + '/jss/domains'
    $header = get_dr_header($fios_session_id)
    $domains = get_request $endpoint $header
    $cfo_domain = $false
    foreach ($domain in $domains) {
        if ($domain.mode -eq "CONTINUOUS_FAILOVER_RUNNING") {
            $cfo_domain = $true
            upgrade_recva $ms_ip $fios_session_id $domain "rocva"
            Start-Sleep 30 #The apis take some time to update, hence the sleep
            upgrade_recva $ms_ip $fios_session_id $domain "rvms"
        }
    }
    if (!$cfo_domain) {
        Write-Host "No domain with a running CFO found."
    }
}

function upgrade_recva {
    param($ms_ip, $fios_session_id, $domain, $recva)
    $domain_id = $domain.id
    $domain_name = $domain.name
    $header = get_dr_header($fios_session_id)
    $endpoint = 'https://' + $ms_ip + '/jss/domains/azureblob/' + $domain_id + '/continuousfailoverstatus'
    $retry_count = 0
    do {
        $cfo_status = get_request $endpoint $header

        if ($null -eq $cfo_status.id) {
            Write-Warning "Getting cfo status failed, retrying" -WarningAction Continue
            Start-Sleep 30
            $retry_count++
        }
    } while ($retry_count -lt 3 -and ($null -eq $cfo_status.id))
    if (($retry_count -eq 3) -and ($null -eq $cfo_status.id)) {
            Write-Error "ERROR: Upgrade $recva on $domain_name failed. Please check JetStream UI task logs and re-run the cmdlet with Resume as True" -ErrorAction Stop
    }
    if ($recva -eq "rocva") {
        $recvaState = $cfo_status.rocvaState
    }
    elseif ($recva -eq "rvms") {
        $recvaState = $cfo_status.rvmsState
    }
    if ($recvaState.availableSoftware -ne $null) {
        $retry_count = 0
        $upgrade_version = $recvaState.availableSoftware.version
        Write-Host "Upgrading $recva for domain $domain_name to $upgrade_version"
        $task_id = $cfo_status.rocvaState.rehydrationTaskId
        $endpoint = 'https://' + $ms_ip + '/jss/domains/' + $domain_id + '/rehydrationtask/' + $task_id + '/' + $recva + '/upgrade'
        $body = @{
            "version" = $upgrade_version
        }
        $json = $body | ConvertTo-Json
        do {
            $task = post_request $endpoint $header $json
            if ($task.id -eq $null) {
                # We don't want to continue upgrade if it fails at any point.
                # Customer should check the cause of failure, fix it and run again.
                Write-Error "Post request to upgrade $recva for $domain_name failed, retrying." -ErrorAction Continue
                start-sleep 15
                $retry_count++
                continue
            }
            else {
                try {
                    # wait for $recva upgrade to finish
                    $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
                }
                catch {
                    Write-Error "Upgrade $recva for $domain_name failed. Please check JetStream UI task logs and re-run the cmdlet with Resume as True" -ErrorAction Stop
                }
            }
        }while(($retry_count -lt 3) -and ($status -ne 0))
        if (($retry_count -eq 3) -and ($status -ne 0)){
            Write-Error "ERROR: Upgrade $recva on $domain_name failed. Please check JetStream UI task logs and re-run the cmdlet with Resume as True" -ErrorAction Stop
        }
        else {
            Write-Host "$recva on domain $domain_name upgraded successfully"
        }
    }
    else {
        Write-Host "No upgrade available for $recva in $domain_name. Skipping"
    }
}

function update_clusters {
    param($ms_ip, $fios_session_id, $user, $pwd, $msa_vm, $major_upgrade)
    Write-Host "### $($MyInvocation.MyCommand) ###"

    Write-Host "Checking for Cluster(s) with available upgrade"
    $endpoint = 'https://' + $ms_ip + '/jss/clusters?partial=true'
    $header = get_dr_header $fios_session_id
    $clusters = get_request $endpoint $header
    $upgrade_available = $false
    $cluster_upgrade = $false

    foreach ($cluster in $clusters) {
        $cluster_name = $cluster.clusterName
        $cluster_state=get_cluster_state $ms_ip $fios_session_id $cluster_name
        $version = $cluster_state.availableSoftware.version
        $software_name = $cluster_state.software.name
        if (($null -eq $version) -or ($software_name -ne 'jetdr')) {
            write-host "No upgrade available for cluster $cluster_name."
        }
        else {
            $upgrade_available = $true
            Write-Host "Upgrading configured clusters on vc_hostname from MS $ms_ip"
            $endpoint = 'https://' + $ms_ip + '/jss/clusters/upgrade'

            $retry_count=0
            $cluster_params = @()

            $hosts=Get-Cluster -Name $cluster_name -ErrorAction SilentlyContinue | Get-VMHost
            $drsEnabled = (Get-Cluster $cluster_name).DrsEnabled
            $drsAutomationLevel = (Get-Cluster $cluster_name).DrsAutomationLevel
            if (($hosts.count -lt 4) -or (($drsEnabled -ne "True") -and ($drsAutomationLevel -ne "FullyAutomated"))) {
                Write-Error "Cluster to be upgraded has less than 4 hosts or DRS is not enabled or not Fully Automated. Please check number of hosts and DRS automation level" -ErrorAction Stop
            }
            $params = @{
                "clusterId" = $cluster.id
                "version" = $cluster_state.availableSoftware.version
            }
            $cluster_params += $params
            $body = @{
                "upgradeClustersParams" = $cluster_params
            }
            $json = $body | Convertto-JSON
            do{
                $task=post_request $endpoint $header $json
                if ($task -eq $null) {
                    Write-Error "Post request to update cluster failed"
                    $retry_count++
                    #Sleeping for 15 secs before retrying
                    start-sleep 15
                    continue
                }
                else {
                    try {
                        # wait for cluster upgrade to finish before we can remove the elevated privileges
                        $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
                    }
                    catch {
                        Write-Error "Cluster upgrade task failed. Retrying." -ErrorAction Continue
                        $retry_count++
                        # Sleeping for 15 secs before retrying
                        start-sleep 15
                    }
                }
            }while(($retry_count -lt 3) -and ($status -ne 0))
            if (($retry_count -eq 3) -and ($status -ne 0)){
                $cluster_upgrade = $false
                Write-Error "ERROR: Upgrade Cluster failed. Please check MS task logs, resolve the issues and run again with Resume as True" -ErrorAction Stop
            }
            else {
                Write-Host "Cluster upgrade on $cluster_name completed successfully. Restarting cim service on all hosts of $cluster_name"
                restart_cim $cluster_name
                $cluster_upgrade = $true
            }
        }
        if (($cluster_upgrade -or !$upgrade_available) -and !$major_upgrade) {
            Write-Host "Performing cleanup."
            $clean_upgrade = Invoke-VMScript -VM $msa_vm -ScriptText 'js_upgrade.py clean -v' -GuestUser $user -GuestPassword $pwd
            if ( $clean_upgrade.ExitCode -ne 0 ) {
                write-error $clean_upgrade.ScriptOutput
                write-error "MSA upgrade clean-up failed. Exiting" -ErrorAction Stop
            }
        }
    }
}

function resolveHostIssue {
    param($ms_ip, $cluster_name, $host_name, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"

    $cluster_state = get_cluster_state $ms_ip $fios_session_id $cluster_name
    $host_params = @()
    $node_in_cluster = 0

    foreach ($node in $cluster_state.nodes) {
        if ($node.hostname -eq $host_name) {
            $params= @{
                "hostId" = $node.id
            }
            $host_params += $params
            $node_in_cluster = 1
            break
        }
    }
    if ($node_in_cluster -eq 0) {
        write-error "Given hostname $host_name is not a part of the cluster $cluster_name" -ErrorAction Stop
    }

    Write-Host "Resolving host config on $host_name"
    $endpoint = 'https://' + $ms_ip + '/jss/hosts/resolveConfig'

    $retry_count=0

    $header = get_dr_header $fios_session_id
    $body = @{
        "resolveHostsConfigParams" = $host_params
    }
    $json = $body | Convertto-JSON
    do{
        $task=post_request $endpoint $header $json
        if ($task -eq $null) {
            Write-Error "Post request to resolve host configuration failed" -ErrorAction Continue
            #Sleeping for 15 secs before retrying
            start-sleep 15
            $retry_count++
            continue
        }
        else {
            try {
                $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
            }
            catch {
                Write-Error "Resolve host issue task failed. Retrying" -ErrorAction Continue
                $retry_count++
                # Sleeping for 15 secs before retrying
                start-sleep 15
            }
        }
    }while(($retry_count -lt 3) -and ($status -ne 0))
    if (($retry_count -eq 3) -and ($status -ne 0)){
        Write-Error "ERROR: Resolve host config task failed. Please check MS task logs, resolve the issues and run again." -ErrorAction Stop
    }
    return $status
}

function resolveClusterIssue {
    param($ms_ip, $cluster_name, $fios_session_id)
    Write-Host "### $($MyInvocation.MyCommand) ###"

    $HostCount=Get-Cluster -Name $cluster_name -ErrorAction SilentlyContinue | Get-VMHost
    if ($HostCount.count -lt 4) {
        Write-Warning "Cluster has less than 4 hosts, Resolve cluster operation may fail if there is a requirement for hosts to be equal to or more than 4 hosts" -WarningAction Continue
    }
    else {
        Write-Host "SUCCESS: Protected cluster satisfies the number of host requirement"
    }

    $retry_count=0
    $cluster_state = get_cluster_state $ms_ip $fios_session_id $cluster_name
    $cluster_params = @()

    Write-Host "Resolving cluster config on $cluster_name"
    $endpoint = 'https://' + $ms_ip + '/jss/clusters/resolveConfig'
    $primary_site_dcs = get_site_dc_state $ms_ip $fios_session_id

    foreach ($dc in $primary_site_dcs) {
        foreach ($cluster in $dc.clusters) {
            if ($cluster.clusterName.ToLower() -eq $cluster_name.ToLower()) {
                $params = @{
                    "clusterId" = $cluster.id
                }
                $cluster_params += $params
                break
            }
         }
    }
    $header = get_dr_header $fios_session_id
    $body = @{
        "resolveClustersConfigParams" = $cluster_params
    }
    $json = $body | ConvertTo-JSON
    do{
        $task=post_request $endpoint $header $json
        if ($task -eq $null) {
            Write-Error "Post request to resolve cluster configuration failed" -ErrorAction Continue
            $retry_count++
            #Sleeping for 15 secs before retrying
            start-sleep 15
            continue
        }
        else {
            try {
                $status = wait_for_dr_task_finish $ms_ip $fios_session_id $task.id
            }
            catch {
                Write-Error "Resolve cluster issue failed. Retrying" -ErrorAction Continue
                $retry_count++
                # Sleeping for 15 secs before retrying
                start-sleep 15
            }
        }
     }while(($retry_count -lt 3) -and ($status -ne 0))
     if (($retry_count -eq 3) -and ($status -ne 0)){
        Write-Error "ERROR: Resolve cluster config task failed. Please check MS task logs, resolve the issues and run again." -ErrorAction Stop
     }
     return $status
 }

function verifyHash {
    param($artifact_path, $signature_path)

    # public key created for JetDR at 08/05/2022
    $pub_file = @"
-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1ufN+FC2ADo5nyW5zwBi
jS9Wian5GCS7DhvmeuiTxm9pY+umHxJDXyDAUm3otTrlB1FI89CrBwEJh2L3QO0Z
2Z0YmvbYLDKSAiHOGeHe0RPQirvHGe4secKt+IB9S/vYVToAcGCsm/tUhRJGmdjy
FwciYDZrvj6aAJUGnwnx0a7zxpwMLOJZFOyN6tnoP1hTHaZ4BQOSzSa/XkKqMF2s
4n5G2sF0ydiQcYNiZ/1SQfmTr1wI7aQQOQ0YqgUEdgPyT6FAYE1zd3QSVgDBbwR/
lXRMR8EpbWBz3ddWeSBh/EYSZovuQlcr9aoGQnk6LQSCcY2YVilF60i2DxVU0IJP
fSPyIidd7fWXO0TjP79EI+7013OXofQWIV0x1yydlzUs3zKY6p4JtT41yAjlxzBD
35Q6s4uIHUeQgIoxGo7AsJuX9mAPWXzc3g+O98qSSxtnNf1mTjkN2NEMSjMmlXWP
aatR+aV2OftQRE0CFUZcFcv8Sg7fBIAtkid4glYWvotKhe24GPGy3nwoUkp4vsec
WDtmUUT71p03WtqRyEsJmZYmzrT9V/CwIh+R3GnQ8XGY3zmBRaFNXqyCejwhwjuc
CNhfRB+se52P90PHqIgekkebDB2o8m87D8gH40OSzAeUqrXavqOLoZgU03ZeWVth
sj0dwJnsJBPNMfeQQrsqxCcCAwEAAQ==
-----END PUBLIC KEY-----
"@
    $rsa = [System.Security.Cryptography.RSA]::Create()
    $len=0
    $pub_key = $pub_file -Split '\n' | Select-String -Pattern "---" -NotMatch | Join-String
    $pub_data = [System.Convert]::FromBase64String($pub_key)
    $rsa.ImportSubjectPublicKeyInfo($pub_data, [ref] $len)
    $sig_b64 = Get-Content -Path $signature_path
    $sig_str = $sig_b64 -Split '`n' | Join-String
    $sig_data = [System.Convert]::FromBase64String($sig_str)
    $hash=Get-FileHash -Algorithm SHA512 -Path $artifact_path
    $byte_hash=[byte[]]($hash.Hash -replace '..','0x$&,' -split ',' -ne '')
    $artifact_status = $rsa.VerifyData($byte_hash, $sig_data, "SHA512", [System.Security.Cryptography.RSASignaturePadding]::Pkcs1)
    return $artifact_status
}

function majorUpdateMSA {
    param (
        [string]$msa_name_4x,
        [string]$user,
        [string]$pwd,
        [string]$new_msa_name
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    $msa_upgrade = $false
    $src_vm = Get-VM -Name $msa_name_4x
    $src_disk = $src_vm | Get-HardDisk
    if ($null -eq $src_disk) {
        Write-Error -Message "Could not find disk in $src_vm" -ErrorAction Stop
    }

    #Powering off MSA 4.x vm before proceding
    $vm_is_poweredon = get_vm_power_state $msa_name_4x
    if ($vm_is_poweredon) {
        $result = msa_operation $msa_name_4x ([VmOperations]::ShutDown) 900
        if (!$result) {
            Write-Error "MSA $msa_name_4x did not power off even after 15 minutes. Exiting."  -ErrorAction Stop
        }
    }

    #Attach the disk to msa5
    $target_vm = Get-VM -Name $new_msa_name
    Write-Host "Attaching disk from $msa_name_4x to $new_msa_name"
    New-HardDisk -vm $target_vm -DiskPath $src_disk.Filename | Out-Null
    Start-Sleep 10 #Sleeping 10sec before starting the next VM operation
    $target_disks = Get-VM -Name $new_msa_name | Get-HardDisk

    if ($target_disks.Filename -contains $src_disk.Filename) {
        Write-Host "Disk from $msa_name_4x is successfully attached to $new_msa_name"
        try{
            $perform_upgrade = Invoke-VMScript -VM $new_msa_name -ScriptText 'python3 /opt/fio/vme2/bin/msa-migrate.py' -GuestUser $user -GuestPassword $pwd
        }
        catch {
            $Exception = $_.Exception.Message
            Write-Error "Execution of upgrade script failed with exception $Exception"
        }

        if ( $perform_upgrade.ExitCode -ne 0 ) {
            Write-Error $perform_upgrade.ScriptOutput
            Write-Error "Execution of upgrade script failed. Performing cleanup" -ErrorAction Continue
        }
        else {
            #Write-Host $perform_upgrade.ScriptOutput
            Write-Host "Execution of upgrade script is successfull"
            $msa_upgrade = $true
        }

        Start-Sleep 10
        Write-Host "Detaching the disk from $new_msa_name"
        $target_disk = get-vm -Name $target_vm | Get-HardDisk | Where-Object {$_.Filename -eq $src_disk.Filename}
        try {
            Remove-HardDisk $target_disk -Confirm:$false | Out-Null
            Start-Sleep 10
        }
        catch {
            $Exception = $_.Exception.Message
            Write-Error "Detaching disk failed with exception $Exception"
            Write-Warning "Error while detaching the disk from $new_msa_name post upgrade" -WarningAction SilentlyContinue
        }

        #Upgrade vcenter plugin
        if ($msa_upgrade) {
            $ms_ip = get_vm_ip $new_msa_name 120
            $plugin_upgrade_status = get_plugin_upgrade_status $ms_ip $user $pwd
            if ($plugin_upgrade_status) {
                upgrade_vcenter_plugin $ms_ip $user $pwd
            }
        }
    }
    else {
        Write-Error "Target VM $new_msa_name does not contain $msa_name_4x disk, upgrade is not possible. Contact Jetstream Support for assistance" -ErrorAction Continue
    }
    if (!$msa_upgrade) {
        $vm_is_poweredon =  get_vm_power_state $new_msa_name
        if ($vm_is_poweredon) {
            #MSA 5.x vm is powered off
            $result = msa_operation $new_msa_name ([VmOperations]::ForcePowerOff) 300
            if (!$result) {
                Write-Error "MSA $new_msa_name failed to poweroff, cannot proceed to power-on vm $msa_vm_4x. Contact Jetstream Support for assistance" -ErrorAction Stop
            }
            Write-Host "MSA $new_msa_name powered off successfully"

            #Delete MSA 5.x VM permanently
            msa_operation $new_msa_name ([VmOperations]::DeletePermanently) | Out-Null
        }

        #Power on MSA 4.x vm
        $vm_is_poweredon = get_vm_power_state $msa_name_4x
        if (!$vm_is_poweredon) {
            $result = msa_operation $msa_name_4x ([VmOperations]::PowerOn) 300
            if (!$result) {
                write-error "MSA $msa_name_4x has not powered on after 5 minutes" -ErrorAction Stop
            }
            Write-Host "MSA $msa_name_4x powered on successfully"
            $vm_ip=get_vm_ip $msa_name_4x 600
            if ($null -ne $vm_ip) {
                $rc=ms_rest_state $vm_ip "root" $Password 600
                if (!$rc) {
                    Write-Error "MSA $vm_ip is not ready after 10 mins" -ErrorAction Stop
                }
                write-host "Sleep for 2 minutes after Management server becomes pingable"
                start-sleep 120
            }
            else {
                Write-Error "Could not get ip of MSA VM after 10 mins." -ErrorAction Stop
            }
        }
        Write-Error "MSA and Jetstream Plugin upgrade failed, Contact Jetstream Support for assistance" -ErrorAction Stop
    }
    else {
        Write-Host "MSA and Jetstream Plugin upgraded sucessfully"
    }
}

function compare_upgrade_builds {
    param (
        $js_url
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    try {
        $major_upgrade = $false
        $extMgr = Get-View ExtensionManager
        $plugins = $extMgr.ExtensionList | Select-Object Key
        $jsdrplugin_key = "com.jetstream.primary.jetdrplugin"
        if ($plugins.Key -contains $jsdrplugin_key) {
            #To get index of the $jsdrplugin to find version
            $index = [array]::IndexOf($extMgr.ExtensionList,$jsdrplugin)
            $jsdrplugin_version = $extMgr.ExtensionList[$index].Version
            $major_version_number_curr_msa = $jsdrplugin_version[0..2] | Join-String
        }
        else {
            Write-Error "The vCenter is not registered to JetStream. Upgrade is not possible. Exiting." -ErrorAction Stop
        }

        $file = Split-Path -Path $js_url -Leaf
        $major_version_number_upgraded_msa = $file.Split('-')[-2][0..2] | Join-String

        #Returns major upgrade as true when current msa version is less than 4.3 and msa upgrade version is greater than 4.3.
        if (($major_version_number_curr_msa -lt '4.3') -and ($major_version_number_upgraded_msa -ge '4.3')) {
            $major_upgrade = $true
        }
        return $major_upgrade
    }
    catch {
        Write-Error "Failed to determine if it is a major upgrade or not. Please contact JetStream support for assistance." -ErrorAction Stop
    }
}

enum VmOperations {
    <# List of supporting vm operations #>
    PowerOn
    ShutDown
    ForcePowerOff
    Restart
    DeletePermanently
}

function msa_operation {
    param (
    [string]$vm_name,
    [VmOperations]$vm_operation,
    [Int32]$timeout = 120
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    #Precheck to see if the vm exists
    try {
        $vm_obj = Get-VM -Name $vm_name -ErrorAction Stop
    }
    catch {
        $exception = $_
        write-host "$exception"
    }
    if ($null -eq $vm_obj) {
        return $false
    }

    $result = $true
    $timeout_in_min = $timeout/60
    switch ($vm_operation)
    {
        ([VmOperations]::PowerOn) {
            if ($vm_obj.PowerState -match 'PoweredOn') {
                Write-Host "$vm_name is already in PoweredOn state"
            }
            else {
                Start-VM -vm $vm_obj -Confirm:$false | Out-Null
                do {
                    Write-Host "Checking VM power state"
                    $vm_obj = Get-VM -Name $vm_name
                    Start-Sleep 30
                    $timeout -= 30
                } while (($timeout -gt 0) -and ($vm_obj.PowerState -match 'PoweredOff'))
                if (($timeout -le 0) -and (($vm_obj.PowerState -match 'PoweredOff'))) {
                    Write-Host "VM $vm_name did not power on even after '$timeout_in_min' minutes."
                    $result = $false
                }
            }
            Break
        }
        ([VmOperations]::ShutDown) {
            if ($vm_obj.PowerState -match 'PoweredOff') {
                Write-Host "$vm_name is already in PoweredOff state"
            }
            elseif($vm_obj.ExtensionData.Guest.ToolsRunningStatus -ne 'guestToolsRunning') {
                Write-Error "Cannot gracefully shutdown $vm_name as vmware tools is not running. Use force to forcefully poweroff the VM"
                $result = $false
            }
            else {
                Stop-VMGuest -vm $vm_obj -Confirm:$false | Out-Null
                do {
                    Write-Host "Checking VM power state"
                    $vm_obj = Get-VM -Name $vm_name
                    Start-Sleep 30
                    $timeout -= 30
                } while (($timeout -gt 0) -and ($vm_obj.PowerState -match 'PoweredOn'))
                if (($timeout -le 0) -and (($vm_obj.PowerState -match 'PoweredOn'))) {
                    Write-Host "VM $vm_name did not power off even after '$timeout_in_min' minutes."
                    $result = $false
                }
            }
            Break
        }
        ([VmOperations]::ForcePowerOff) {
            if ($vm_obj.PowerState -match 'PoweredOff') {
                Write-Host "$vm_name is already in PoweredOff state"
            }
            else {
                Stop-VM -vm $vm_obj -Confirm:$false | Out-Null
                do {
                    Write-Host "Checking VM power state"
                    $vm_obj = Get-VM -Name $vm_name
                    Start-Sleep 30
                    $timeout -= 30
                } while (($timeout -gt 0) -and ($vm_obj.PowerState -match 'PoweredOn'))
                if (($timeout -le 0) -and ($vm_obj.PowerState -match 'PoweredOn')) {
                    Write-Host "VM $vm_name did not power off even after '$timeout_in_min' minutes."
                    $result = $false
                }
            }
            Break
        }
        ([VmOperations]::Restart) {
            if ($vm_obj.PowerState -match 'PoweredOff') {
                Write-Error "$vm_name is in PoweredOff state. Cannot restart."
                $result = $false
            }
            else {
                Restart-VM -VM $vm_obj -Confirm:$false | Out-Null
                do {
                    Write-Host "Checking VM power state"
                    $vm_obj = Get-VM -Name $vm_name
                    Start-Sleep 30
                    $timeout -= 30
                } while (($timeout -gt 0) -and ($vm_obj.PowerState -match 'PoweredOff'))
                if (($timeout -le 0) -and ($vm_obj.PowerState -match 'PoweredOff')) {
                    Write-Host "VM $vm_name is not powered on even after '$timeout_in_min' minutes."
                    $result = $false
                }
            }
            Break
        }
        ([VmOperations]::DeletePermanently) {
            Get-VM -Name $vm_name | Remove-VM -DeletePermanently -Confirm:$false | Out-Null
            Start-Sleep 5
            $vm_exists = Get-VM -Name $msa_vm -ErrorAction SilentlyContinue
            if ($null -eq $vm_exists) {
                Write-Host "$msa_vm deleted successfully"
            }
            else {
                Write-Error "Failed to delete $msa_vm"
                $result = $false
            }
            Break
        }
        Default {
            Write-Host "The vm operation $vm_operation is currently not supported"
            $result = $false
        }
    }
    return $result
}

function get_vm_power_state {
    param (
        [string] $vm_name
    )
    Write-Host "### $($MyInvocation.MyCommand) ###"
    #Check the existence of vm
    try {
        $vm_obj = Get-VM -Name $vm_name -ErrorAction Stop
    }
    catch {
        $exception = $_
        Write-Host "$exception" -ErrorAction Stop
    }

    if ($null -ne $vm_obj) {
        $result = $null
        $powerstate  = (Get-VM -Name $vm_name).PowerState

        if ($powerstate -match "PoweredOff") {
            Write-Host "VM $vm_name is powered off"
            $result = 0
        }
        if ($powerstate -match "PoweredOn") {
            Write-Host "VM $vm_name is powered on"
            $result = 1
        }
        return $result
    }
    else {
        Write-Error "VM $vm_name not present in the vcenter" -ErrorAction Stop
    }
}

function create_rest_user {
    param($privilege_group, $new_role_name, $sso_user_name, $sso_user_pwd)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Creating jetstream iofrest user"
    $vi_privilege = Get-VIPrivilege -PrivilegeGroup $privilege_group
    $virole = Get-VIRole -Name $new_role_name -ErrorAction SilentlyContinue
    if(!$virole) {
        Write-Host "Creating Role $new_role_name"
        $new_role = New-VIRole -Privilege $vi_privilege -Name $new_role_name
        if(!$new_role) {
            Write-Error "Failed to create role $new_role_name" -ErrorAction Stop
        }
    }
    else {
        Write-Host "Role $new_role_name already exists on the vCenter. Continuing"
    }

    $vc_role_id = (Get-VIRole -Name $new_role_name).ExtensionData.RoleId

    $domain = $sso_user_name.split("@")[1]
    $user = $sso_user_name.split("@")[0]
    $ssouser = Get-SsoPersonUser -Name $user -Domain $domain
    if(!$ssouser) {
        write-host "Creating user $new_user"
        $new_user = New-SsoPersonUser -UserName $user -Password $sso_user_pwd
        if(!$new_user) {
            Write-Error "Failed to create user $new_user" -ErrorAction Stop
        }
        else {
            Write-Host "Created user $new_user successfully on the vCenter."
        }
    }
    else {
        Write-Host "User $user already exists in the vCenter."
        $vc_role_id = $null
    }
    return $vc_role_id
}

function update_rest_user_on_msa {
    param($ms_ip, $ms_user, $ms_pwd, $privilege_group, $new_role_name, $new_user_name)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Updating credentials of jetstream iofrest user on MSA"
    $sso_user_pwd = generate_password
    $endpoint = 'https://' + $ms_ip + '/jss/jetiofrest/credentials'
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }

    $sso_user_obj = get_rest_user $ms_ip $ms_user $ms_pwd

    if ($sso_user_obj -eq $null) {
        $sso_user_name = $new_user_name
    }
    else {
        $sso_user_name = $sso_user_obj
    }

    $body = @{
        "userName" = $sso_user_name
        "password" = $sso_user_pwd
    }

    $json = $body | Convertto-JSON
    if ($sso_user_obj -eq $null) {
        Write-Host "User details not present on MSA. Creating the user."
        $vc_role_id = create_rest_user $privilege_group $new_role_name $sso_user_name $sso_user_pwd
        if ($vc_role_id -eq $null) {
            # There is no way to get password of the user which is already present.
            # Reset it with a new password and update that to the MSA
            reset_rest_user_password $sso_user_name $sso_user_pwd
        }
        else {
            # TODO: Assign Global permission to the user when api is available
            Write-Warning "Please assign global permission to the user $sso_user_name"
        }
        $resp_obj = post_request $endpoint $header $json
    }
    else {
        Write-Host "User details found on MSA. Refreshing the user."
        # The rest user can be refreshed using PUT, but once the password
        # is reset and before the new password is updated in the MSA, the
        # MSA tries to login using the old password and after three failed
        # attempts, the user gets locked in the vCenter. So, we are deleting
        # the user from MSA, refreshing the password and then updating MSA
        # via POST call again.
        disable_rest_access $ms_ip $ms_user $ms_pwd $sso_user_name
        reset_rest_user_password $sso_user_name $sso_user_pwd
        $resp_obj = post_request $endpoint $header $json
    }
    Start-Sleep 2
    $sso_user_obj = get_rest_user $ms_ip $ms_user $ms_pwd
    if ($sso_user_obj -eq $null) {
        Write-Error "Did not find iofrest user on MSA" -ErrorAction Stop
    }
    else {
        Write-Host "iofrest user successfully updated on MSA"
    }
}

function reset_rest_user_password {
    param($sso_user_name, $sso_user_pwd)
    $sso_user = $sso_user_name.split("@")[0]
    $sso_domain = $sso_user_name.split("@")[1]

    $user_on_vc = Get-SsoPersonUser -Domain $sso_domain -Name $sso_user
    if ($user_on_vc -eq $null) {
        Write-Error "No user $sso_user_name exists in the vCenter" -ErrorAction Stop
    }

    Get-SsoPersonUser -Domain $sso_domain -Name $sso_user | Set-SsoPersonUser -NewPassword $sso_user_pwd | Out-Null

    # PasswordExpirationRemainingDays gets reset to 90 upon successful reset
    $user_on_vc = Get-SsoPersonUser -Domain $sso_domain -Name $sso_user
    if ($user_on_vc.PasswordExpirationRemainingDays -eq 90) {
        Write-Host "Password for $sso_user_name sucessfully refreshed."
    }
    else {
        Write-Error "Failed to refresh the password of $sso_user_name on vCenter" -ErrorAction Stop
    }
}

function get_rest_user {
    param($ms_ip, $ms_user, $ms_pwd)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Fetching credentials of jetstream iofrest user"
    $sso_user = $null
    $endpoint = 'https://' + $ms_ip + '/jss/jetiofrest/credentials'
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }
    $resp_obj = get_request $endpoint $header
    if ($resp_obj -eq "404") {
        Write-Host "Couldn't get iofrest user details from MSA $ms_ip"
    }
    else {
        $sso_user = $resp_obj
    }
    return $sso_user
}

function disable_rest_access {
    param($ms_ip, $ms_user, $ms_pwd, $sso_user)
    Write-Host "### $($MyInvocation.MyCommand) ###"
    Write-Host "Deleting jetream iofrest user $sso_user from MSA"
    $endpoint = 'https://' + $ms_ip + '/jss/jetiofrest/credentials' + '?userName=' + $sso_user
    $header = @{
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }
    delete_request $endpoint $header
    Start-Sleep 2

    $endpoint = 'https://' + $ms_ip + '/jss/jetiofrest/credentials'
    $header = @{
        "Accept" = "application/json"
        'content-type' = 'application/json'
        'ms-session' = login_ms_portal $ms_ip $ms_user $ms_pwd
    }
    $resp_obj = get_request $endpoint $header
    if ($resp_obj -eq "404") {
        Write-Host "Successfully deleted user from MSA"
    }
    else {
        Write-Error "Could not delete user from MSA" -ErrorAction Stop
    }
}

function is_vcenter_8 {
    $vcenter_obj = $global:DefaultVIServer
    $major_version = $vcenter_obj.Version.Split(".")[0]
    if ($major_version -ge 8) {
        return $true
    }
    return $false
}

# SIG # Begin signature block
# MIIUoAYJKoZIhvcNAQcCoIIUkTCCFI0CAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCQuMNMxM+XV1QT
# mCAdwYeU/htefvkHAAuVo/Y5LHRNAKCCEVswggVvMIIEV6ADAgECAhBI/JO0YFWU
# jTanyYqJ1pQWMA0GCSqGSIb3DQEBDAUAMHsxCzAJBgNVBAYTAkdCMRswGQYDVQQI
# DBJHcmVhdGVyIE1hbmNoZXN0ZXIxEDAOBgNVBAcMB1NhbGZvcmQxGjAYBgNVBAoM
# EUNvbW9kbyBDQSBMaW1pdGVkMSEwHwYDVQQDDBhBQUEgQ2VydGlmaWNhdGUgU2Vy
# dmljZXMwHhcNMjEwNTI1MDAwMDAwWhcNMjgxMjMxMjM1OTU5WjBWMQswCQYDVQQG
# EwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMS0wKwYDVQQDEyRTZWN0aWdv
# IFB1YmxpYyBDb2RlIFNpZ25pbmcgUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEBAQUA
# A4ICDwAwggIKAoICAQCN55QSIgQkdC7/FiMCkoq2rjaFrEfUI5ErPtx94jGgUW+s
# hJHjUoq14pbe0IdjJImK/+8Skzt9u7aKvb0Ffyeba2XTpQxpsbxJOZrxbW6q5KCD
# J9qaDStQ6Utbs7hkNqR+Sj2pcaths3OzPAsM79szV+W+NDfjlxtd/R8SPYIDdub7
# P2bSlDFp+m2zNKzBenjcklDyZMeqLQSrw2rq4C+np9xu1+j/2iGrQL+57g2extme
# me/G3h+pDHazJyCh1rr9gOcB0u/rgimVcI3/uxXP/tEPNqIuTzKQdEZrRzUTdwUz
# T2MuuC3hv2WnBGsY2HH6zAjybYmZELGt2z4s5KoYsMYHAXVn3m3pY2MeNn9pib6q
# RT5uWl+PoVvLnTCGMOgDs0DGDQ84zWeoU4j6uDBl+m/H5x2xg3RpPqzEaDux5mcz
# mrYI4IAFSEDu9oJkRqj1c7AGlfJsZZ+/VVscnFcax3hGfHCqlBuCF6yH6bbJDoEc
# QNYWFyn8XJwYK+pF9e+91WdPKF4F7pBMeufG9ND8+s0+MkYTIDaKBOq3qgdGnA2T
# OglmmVhcKaO5DKYwODzQRjY1fJy67sPV+Qp2+n4FG0DKkjXp1XrRtX8ArqmQqsV/
# AZwQsRb8zG4Y3G9i/qZQp7h7uJ0VP/4gDHXIIloTlRmQAOka1cKG8eOO7F/05QID
# AQABo4IBEjCCAQ4wHwYDVR0jBBgwFoAUoBEKIz6W8Qfs4q8p74Klf9AwpLQwHQYD
# VR0OBBYEFDLrkpr/NZZILyhAQnAgNpFcF4XmMA4GA1UdDwEB/wQEAwIBhjAPBgNV
# HRMBAf8EBTADAQH/MBMGA1UdJQQMMAoGCCsGAQUFBwMDMBsGA1UdIAQUMBIwBgYE
# VR0gADAIBgZngQwBBAEwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21v
# ZG9jYS5jb20vQUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEE
# KDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZI
# hvcNAQEMBQADggEBABK/oe+LdJqYRLhpRrWrJAoMpIpnuDqBv0WKfVIHqI0fTiGF
# OaNrXi0ghr8QuK55O1PNtPvYRL4G2VxjZ9RAFodEhnIq1jIV9RKDwvnhXRFAZ/ZC
# J3LFI+ICOBpMIOLbAffNRk8monxmwFE2tokCVMf8WPtsAO7+mKYulaEMUykfb9gZ
# pk+e96wJ6l2CxouvgKe9gUhShDHaMuwV5KZMPWw5c9QLhTkg4IUaaOGnSDip0TYl
# d8GNGRbFiExmfS9jzpjoad+sPKhdnckcW67Y8y90z7h+9teDnRGWYpquRRPaf9xH
# +9/DUp/mBlXpnYzyOmJRvOwkDynUWICE5EV7WtgwggXGMIIELqADAgECAhEAx5f+
# KBbVHZKKYsCngzhy/DANBgkqhkiG9w0BAQwFADBUMQswCQYDVQQGEwJHQjEYMBYG
# A1UEChMPU2VjdGlnbyBMaW1pdGVkMSswKQYDVQQDEyJTZWN0aWdvIFB1YmxpYyBD
# b2RlIFNpZ25pbmcgQ0EgUjM2MB4XDTIzMDUwMzAwMDAwMFoXDTI1MDUwMjIzNTk1
# OVowXDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExGzAZBgNVBAoM
# EkpldFN0cmVhbSBTb2Z0d2FyZTEbMBkGA1UEAwwSSmV0U3RyZWFtIFNvZnR3YXJl
# MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA303037LJy/ygxXhA/dpm
# taFbP40OUibB16KHBG8MsH1RA3KyDNm8IATCm5b8F4YT4umB7osCUewzuHmMebba
# iG4R7+0eJ4ks+RpNbx5XXQ85KnaYRHqyqxeokJzD24Hn154M6Fjmbfj1cG/QgqRX
# +tGngozNVouAhVP7XjNFPeNpNo16aAQElyAkdbJ6phDhENKT8sINmzVxPT53GZZm
# mZx3Kk5VSHamELY4uD3fivSKjxUXAQEUOsEs4nm1Bet7J2Zl+VG6jbSLwWbzbCf3
# Y9ue3MZR95LCASqaKGppMj5PDeYtUm8Ww4Kjc3tvPtJfDaYE8Xwr8T6C6AwGqacE
# a9Vry7XKuL2nucqWvc5bZwHFrpoHphvLXeGqNucyKvT47/w7ASonz3D3u7HvZnHH
# xFL2XljDntOvIGDUvf7q4+P6rmJrrF+8VDLwx6us56bxUi30Hl8hVdPYHwDCEZB8
# T6arb5dq1XhAgS+m3v8GbP5KYwh08bWSXH+HDtv35aK9AgMBAAGjggGJMIIBhTAf
# BgNVHSMEGDAWgBQPKssghyi47G9IritUpimqF6TNDDAdBgNVHQ4EFgQUDjCOWzEl
# bVPAwKfIjtElrJOezhUwDgYDVR0PAQH/BAQDAgeAMAwGA1UdEwEB/wQCMAAwEwYD
# VR0lBAwwCgYIKwYBBQUHAwMwSgYDVR0gBEMwQTA1BgwrBgEEAbIxAQIBAwIwJTAj
# BggrBgEFBQcCARYXaHR0cHM6Ly9zZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQQBMEkG
# A1UdHwRCMEAwPqA8oDqGOGh0dHA6Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb1B1
# YmxpY0NvZGVTaWduaW5nQ0FSMzYuY3JsMHkGCCsGAQUFBwEBBG0wazBEBggrBgEF
# BQcwAoY4aHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvUHVibGljQ29kZVNp
# Z25pbmdDQVIzNi5jcnQwIwYIKwYBBQUHMAGGF2h0dHA6Ly9vY3NwLnNlY3RpZ28u
# Y29tMA0GCSqGSIb3DQEBDAUAA4IBgQCEQuY+MjBvY7xvU4BU0+PAASEGVEJgVO6e
# eu0IBb1d0HJCOgOhhg3T6vLAIGTfB5BRhpHOnI73/7t+4GaqAORJkAqQNhryfxtm
# N9JqUDUx/KvCq0+5oHYaVCWvjap3U1/vc5TbdIVIRRh+EZ92fEafQLlIDVrHun1Z
# TKEdeoRkCDnpexs0X3Lz2IwTPGXqz0OIlGwsyUI/nBuwa//APQPq+q9MS6e1D3RI
# 2IpCd9VuSNiFCcispPaT/E0dwqDe8vYZX7sOdcJTp5whLKC8bFQL2Fpt1pHRzmSz
# hLAJbrRcnmqcnJr7hR+hWsYycnlkvUUxIHPg27bsZwjNJbG0xNSepo8VN8/MdeFE
# ghkRE1QYSryE8JoQjrbk4oXsRvviZAFSGCp9yqeqvmpDrDNCaLwJk0JZ2L5Cpz79
# NvHJ+15kcIFKipn/ExiMZ5Hbj8IehE2vOI0L2olQ5LsPvB8/PA4E/yMq4bgbGg23
# Tn9u11KQlnaeju+dZW3DBs2WVEZmpUkwggYaMIIEAqADAgECAhBiHW0MUgGeO5B5
# FSCJIRwKMA0GCSqGSIb3DQEBDAUAMFYxCzAJBgNVBAYTAkdCMRgwFgYDVQQKEw9T
# ZWN0aWdvIExpbWl0ZWQxLTArBgNVBAMTJFNlY3RpZ28gUHVibGljIENvZGUgU2ln
# bmluZyBSb290IFI0NjAeFw0yMTAzMjIwMDAwMDBaFw0zNjAzMjEyMzU5NTlaMFQx
# CzAJBgNVBAYTAkdCMRgwFgYDVQQKEw9TZWN0aWdvIExpbWl0ZWQxKzApBgNVBAMT
# IlNlY3RpZ28gUHVibGljIENvZGUgU2lnbmluZyBDQSBSMzYwggGiMA0GCSqGSIb3
# DQEBAQUAA4IBjwAwggGKAoIBgQCbK51T+jU/jmAGQ2rAz/V/9shTUxjIztNsfvxY
# B5UXeWUzCxEeAEZGbEN4QMgCsJLZUKhWThj/yPqy0iSZhXkZ6Pg2A2NVDgFigOMY
# zB2OKhdqfWGVoYW3haT29PSTahYkwmMv0b/83nbeECbiMXhSOtbam+/36F09fy1t
# sB8je/RV0mIk8XL/tfCK6cPuYHE215wzrK0h1SWHTxPbPuYkRdkP05ZwmRmTnAO5
# /arnY83jeNzhP06ShdnRqtZlV59+8yv+KIhE5ILMqgOZYAENHNX9SJDm+qxp4Vqp
# B3MV/h53yl41aHU5pledi9lCBbH9JeIkNFICiVHNkRmq4TpxtwfvjsUedyz8rNyf
# QJy/aOs5b4s+ac7IH60B+Ja7TVM+EKv1WuTGwcLmoU3FpOFMbmPj8pz44MPZ1f9+
# YEQIQty/NQd/2yGgW+ufflcZ/ZE9o1M7a5Jnqf2i2/uMSWymR8r2oQBMdlyh2n5H
# irY4jKnFH/9gRvd+QOfdRrJZb1sCAwEAAaOCAWQwggFgMB8GA1UdIwQYMBaAFDLr
# kpr/NZZILyhAQnAgNpFcF4XmMB0GA1UdDgQWBBQPKssghyi47G9IritUpimqF6TN
# DDAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADATBgNVHSUEDDAK
# BggrBgEFBQcDAzAbBgNVHSAEFDASMAYGBFUdIAAwCAYGZ4EMAQQBMEsGA1UdHwRE
# MEIwQKA+oDyGOmh0dHA6Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb1B1YmxpY0Nv
# ZGVTaWduaW5nUm9vdFI0Ni5jcmwwewYIKwYBBQUHAQEEbzBtMEYGCCsGAQUFBzAC
# hjpodHRwOi8vY3J0LnNlY3RpZ28uY29tL1NlY3RpZ29QdWJsaWNDb2RlU2lnbmlu
# Z1Jvb3RSNDYucDdjMCMGCCsGAQUFBzABhhdodHRwOi8vb2NzcC5zZWN0aWdvLmNv
# bTANBgkqhkiG9w0BAQwFAAOCAgEABv+C4XdjNm57oRUgmxP/BP6YdURhw1aVcdGR
# P4Wh60BAscjW4HL9hcpkOTz5jUug2oeunbYAowbFC2AKK+cMcXIBD0ZdOaWTsyNy
# BBsMLHqafvIhrCymlaS98+QpoBCyKppP0OcxYEdU0hpsaqBBIZOtBajjcw5+w/Ke
# FvPYfLF/ldYpmlG+vd0xqlqd099iChnyIMvY5HexjO2AmtsbpVn0OhNcWbWDRF/3
# sBp6fWXhz7DcML4iTAWS+MVXeNLj1lJziVKEoroGs9Mlizg0bUMbOalOhOfCipnx
# 8CaLZeVme5yELg09Jlo8BMe80jO37PU8ejfkP9/uPak7VLwELKxAMcJszkyeiaer
# lphwoKx1uHRzNyE6bxuSKcutisqmKL5OTunAvtONEoteSiabkPVSZ2z76mKnzAfZ
# xCl/3dq3dUNw4rg3sTCggkHSRqTqlLMS7gjrhTqBmzu1L90Y1KWN/Y5JKdGvspbO
# rTfOXyXvmPL6E52z1NZJ6ctuMFBQZH3pwWvqURR8AgQdULUvrxjUYbHHj95Ejza6
# 3zdrEcxWLDX6xWls/GDnVNueKjWUH3fTv1Y8Wdho698YADR7TNx8X8z2Bev6SivB
# BOHY+uqiirZtg0y9ShQoPzmCcn63Syatatvx157YK9hlcPmVoa1oDE5/L9Uo2bC5
# a4CH2RwxggKbMIIClwIBATBpMFQxCzAJBgNVBAYTAkdCMRgwFgYDVQQKEw9TZWN0
# aWdvIExpbWl0ZWQxKzApBgNVBAMTIlNlY3RpZ28gUHVibGljIENvZGUgU2lnbmlu
# ZyBDQSBSMzYCEQDHl/4oFtUdkopiwKeDOHL8MA0GCWCGSAFlAwQCAQUAoIGEMBgG
# CisGAQQBgjcCAQwxCjAIoAKAAKECgAAwGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcC
# AQQwHAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIE
# IEQ6wj12ss8Lw1ivK6simIj+FpObKJOzxSCTy0Td6BiTMA0GCSqGSIb3DQEBAQUA
# BIIBgKLWm/78K88UfzIiiOmVBKql8IQMfgoSg2dPG7a4pT3uHyjOfGY658/j1c0K
# b79INCCFBFp0gbLob8xEdHpTAtWbcEX6o9VaV7ZYvwpn9v71F2PDbIvVdPFNosOf
# nPtszWXH4/9ZE5QsD6synE6nQ6Q4udwrEQfK1XU25j70McwfKLWCDpLLWyHw/kjE
# tQGarLibQSWAnfHnGKPwC3uLw21j5rmaHBxI9nR0QY+CjfRCBtoHUbvULdAiSHSM
# md3KXHwVoebse86S15QG1wVQy9o8ut/+e+g1AljRgLI2yyUs3y5nS6kdPA2gDsJT
# GlPFCE1NUe57OO7c8OT3P77CdoyTvT9L9ops7s1KMi2Y7inA8/3UlHChj3NY/Ajb
# b25vFKFVspsEh3dgrfKIpK/jDlXab71kKbkR9BaQpJ4QAHYriCBquYiUm4KMOYo/
# 66hWfg1QmwpwQT+EtplBsiGt+k/gO7piLjTOoFrQuBpprKsC6h+cvVRHnc1f7DNw
# tc9QZg==
# SIG # End signature block