ZertoAVSModule.psm1

$ZERTO_FOLDER_ON_HOST = "/var/zerto"
$LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS = ('{0}/zertoDriverLogs/' -f $psScriptRoot)
$LOCAL_TEMP_FOLDER_FOR_DOWNLOADED_FILES = ('{0}/filesFromDatastore/' -f $psScriptRoot)
$ZERTO_USER_NAME = "ZertoDR"
$ZERTO_ROLE = "ZertoRole"
$DOMAIN = "vsphere.local"
$ZAPPLIANCE_USER = "zadmin"
$SDDC_RESOURCE_ID_PATTERN = "/subscriptions/(?<AvsSubscriptionId>[^/]+)/resourceGroups/(?<AvsResourceGroup>[^/]+)/providers/Microsoft\.AVS/privateClouds/(?<AvsCloudName>[^/]+)"

function Test-ZertoUserExists {
    <#
        .DESCRIPTION
        Get a zertoUsername and a domain, and return whether or not the user exists in the domain.
 
        .EXAMPLE
 
        Test-ZertoUserExists
    #>

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        if (Get-SsoPersonUser -Name $ZERTO_USER_NAME -Domain $DOMAIN -ErrorAction SilentlyContinue) {
            Write-Host "$ZERTO_USER_NAME already exists in $VC_ADDRESS, domain: $DOMAIN."
            return $true;
        }

        Write-Host "$ZERTO_USER_NAME does not exist in $VC_ADDRESS, domain: $DOMAIN."
        return $false;
    }
}

function Test-ZertoRoleExists {
    <#
        .DESCRIPTION
        Return true if ZertoRole exists, otherwise return false.
 
        .EXAMPLE
 
        Test-ZertoRoleExists
    #>

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        If (Get-VIRole -Name $ZERTO_ROLE -ErrorAction SilentlyContinue) {
            Write-Host "$ZERTO_ROLE already exists in $VC_ADDRESS"
            return $true
        }

        Write-Host "$ZERTO_ROLE does not exist in $VC_ADDRESS"
        return $false;
    }
}

function Test-ZertoDriverLoaded {
    <#
        .DESCRIPTION
        Return true if Zerto driver is loaded, otherwise return false.
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
        .EXAMPLE
 
         Test-ZertoDriverLoaded -HostName <HostName>
    #>

    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName

    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $vmkload = '/tmp/vmklod_output'
        $ZertoDriver = '/tmp/zertoDriver'

        $Command =    ('vmkload_mod -l > {0}' -f $vmkload)
        $Res = Invoke-SSHCommands -HostName $HostName -Commands $Command
        $Command = ('grep "zdriver" {0} > {1}' -f $vmkload, $ZertoDriver)
        $Res = Invoke-SSHCommands -HostName $HostName -Commands $Command -ExitStatusAction "Skip"

        $ExitStatus = $Res["0_exitStatus"]
        if ($ExitStatus -eq '0') {
            Write-Host "Zerto driver is loaded"
            return $true;
        }

        if ($ExitStatus -eq '1') {
            Write-Host "Zerto driver is not loaded"
            return $false;
        }
    }
}

function Get-DriverLogsFromHost {
    <#
        .DESCRIPTION
        Copy Zerto driver logs from host to datastore
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
            .PARAMETER DatastoreName
            Datastore Name
 
        .EXAMPLE
 
        Get-DriverLogsFromHost -HostName <HostName> -DatastoreName <DatastoreName>
    #>

    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."
    Copy-DriverLogFilesFromHostToPSEngine -HostName $HostName
    Copy-DriverLogFilesFromPSEngineToDatastore -DatastoreName $DatastoreName -HostName $HostName
}

function Copy-DriverLogFilesFromHostToPSEngine {
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $sftpSessionId = ($SFTP_Sessions[$HostName]).Value.SessionId

    if (!(Test-Path -path $LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS)) {
        Write-Host "Creating $LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS folder on powerShell engine"
        New-Item $LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS -Type Directory
    }

    $files = '/etc/vmware/zloadmod.txt',
    '/etc/vmware/zunloadmod.txt'

    foreach ($file in $files) {
        if (Test-SFTPPath -SessionId $sftpSessionId -Path $file) {
            Write-Host "$file from $HostName will be downloaded to powerShell engine ($LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS)"
            Get-SFTPItem -SessionId $sftpSessionId -Destination $LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS -Path $file -Force
            Write-Host "$file was copied from $HostName to powerShell engine"
        }
        else {
            Write-Host "File $file does not exist on $HostName"
        }
    }
}

function Copy-DriverLogFilesFromPSEngineToDatastore {
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $psDriverName = "ds"
    $zertoDriverLogsDSPath = ('{0}:\zertoDriverLogsFromHost\{1}\' -f $psDriverName, $HostName)
    $zertoDriverLogsPSPath = ('{0}*' -f $LOCAL_TEMP_FOLDER_FOR_ZERTO_DRIVER_LOGS)
    $datastore = Get-Datastore $DatastoreName

    New-PSDrive -Location $datastore -Name $psDriverName -PSProvider VimDatastore -Root "\" -ErrorAction Stop
    Copy-DatastoreItem -Item $zertoDriverLogsPSPath -Destination $zertoDriverLogsDSPath -Force -ErrorAction Stop

    $files = (Get-ChildItem -Path $zertoDriverLogsDSPath -Name) -join ";"
    Write-Host "$MyInvocation.MyCommand - ZertoDriverFiles: {$files} were copied from powerShell engine ($zertoDriverLogsPSPath) to datastore ($zertoDriverLogsDSPath)"

    Remove-PSDrive -Name $psDriverName
}

function Get-ZertoFilesListFromHost {
    <#
        .DESCRIPTION
        Return a list of all Zerto files on host under /var/zerto
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
        .EXAMPLE
 
        Get-ZertoFilesListFromHost -HostName <HostName>
    #>

    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."
        $Command = ('ls -l {0}' -f $ZERTO_FOLDER_ON_HOST)
        return Invoke-SSHCommands -HostName $HostName -Commands $Command
    }
}

function Test-Connection {

    [AVSAttribute(5, UpdatesSDDC = $false, AutomationOnly = $true)]
    param()

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        return "Test-Connection"
    }

}

function New-ZertoUser {
    <#
        .DESCRIPTION
        Create a ZertoDR user and a ZertoDR role which includes required privileges.
        The script creates a permission by assigning the ZertoDR role to the ZertoDR user.
 
        .EXAMPLE
 
        New-ZertoUser
    #>


    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        try {
            if (Test-ZertoUserExists) {
                throw "$ZERTO_USER_NAME user already exists in $VC_ADDRESS. Run Uninstall-Zerto to delete it."
            }
            elseif (Test-ZertoRoleExists) {
                throw "ZERTO_ROLE role already exists in $VC_ADDRESS. Run Uninstall-Zerto to delete it."
            }
            else {
                #Create Zerto user
                $PersistentSecrets.ZertoPassword = New-RandomPassword
                New-SsoPersonUser -UserName $ZERTO_USER_NAME -Password $PersistentSecrets.ZertoPassword -Description "Zerto DR user" -EmailAddress "ZertoDR@zerto.com" -FirstName "Zerto" -LastName "DR" -ErrorAction Stop | Out-Null

                # Add user to CloudAdmins group
                $group = "CloudAdmins"
                $SsoGroup = Get-SsoGroup -Name $group -Domain $DOMAIN
                Get-SsoPersonUser -Name $ZERTO_USER_NAME -Domain $DOMAIN -ErrorAction Stop | Add-UserToSsoGroup -TargetGroup $SsoGroup -ErrorAction Stop | Out-Null

                Write-Host "ZertoUser ($ZERTO_USER_NAME) creation was completed successfully"
            }

            New-ZertoRole
        }
        catch {
            Write-Error "Failed to create Zerto User. Problem: $_" -ErrorAction Stop
        }
    }
}

function New-ZertoRole {
    <#
        .DESCRIPTION
        Updates a ZertoDR user with the latest ZertoDR role which includes required privileges.
        The script creates a permission by assigning the ZertoDR role to the ZertoDR user.
 
        .EXAMPLE
 
        New-ZertoRole
    #>

    [AVSAttribute(30, UpdatesSDDC = $false)]
    param()

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $zertoPrincipal = $DOMAIN + "\" + $ZERTO_USER_NAME
        $zertoPrivileges = @(
            "Alarm.Create",
            "Alarm.Delete",
            "Authorization.ModifyPermissions",
            "Cryptographer.Access",
            "Datastore.AllocateSpace",
            "Datastore.Browse",
            "Datastore.Config",
            "Datastore.DeleteFile",
            "Datastore.FileManagement",
            "Datastore.UpdateVirtualMachineFiles",
            "StoragePod.Config",
            "Extension.Register",
            "Extension.Unregister",
            "Folder.Create",
            "Global.CancelTask",
            "Global.Diagnostics",
            "Global.DisableMethods",
            "Global.EnableMethods",
            "Global.LogEvent",
            "Host.Config.AdvancedConfig",
            "Host.Config.AutoStart",
            "Host.Config.Settings",
            "Host.Config.NetService",
            "Host.Config.Patch",
            "Host.Inventory.EditCluster",
            "Network.Assign",
            "Resource.AssignVAppToPool",
            "Resource.AssignVMToPool",
            "Resource.ColdMigrate",
            "Resource.HotMigrate",
            "Sessions.ValidateSession",
            "Task.Create",
            "Task.Update",
            "VApp.ApplicationConfig",
            "VApp.AssignResourcePool",
            "VApp.AssignVM",
            "VApp.Create",
            "VApp.Delete",
            "VApp.Import",
            "VApp.PowerOff",
            "VApp.PowerOn",
            "VirtualMachine.Config.AddExistingDisk",
            "VirtualMachine.Config.AddNewDisk",
            "VirtualMachine.Config.AddRemoveDevice",
            "VirtualMachine.Config.AdvancedConfig",
            "VirtualMachine.Config.CPUCount",
            "VirtualMachine.Config.DiskExtend",
            "VirtualMachine.Config.EditDevice",
            "VirtualMachine.Config.ManagedBy",
            "VirtualMachine.Config.Memory",
            "VirtualMachine.Config.RawDevice",
            "VirtualMachine.Config.RemoveDisk",
            "VirtualMachine.Config.Resource",
            "VirtualMachine.Config.Settings",
            "VirtualMachine.Config.SwapPlacement",
            "VirtualMachine.Config.UpgradeVirtualHardware",
            "VirtualMachine.Interact.PowerOff",
            "VirtualMachine.Interact.PowerOn",
            "VirtualMachine.Inventory.CreateFromExisting",
            "VirtualMachine.Inventory.Create",
            "VirtualMachine.Inventory.Register",
            "VirtualMachine.Inventory.Delete",
            "VirtualMachine.Inventory.Unregister",
            "VirtualMachine.State.RemoveSnapshot"
        )

        if ((Test-ZertoUserExists $ZERTO_USER_NAME) -eq $false) {
            Write-Host "$ZERTO_USER_NAME does not exist in $VC_ADDRESS..."
            throw "$ZERTO_USER_NAME does not exist in $VC_ADDRESS..."
        }

        #Create a new role
        New-VIRole -name $ZERTO_ROLE -Privilege (Get-VIPrivilege -Server $VC_ADDRESS -id $zertoPrivileges) -Server $VC_ADDRESS -ErrorAction Stop | Out-Null
        Write-Host "Role $ZERTO_ROLE created on $VC_ADDRESS"

        # Get the Root Folder
        $rootFolder = Get-Folder -NoRecursion
        # Create permission on vCenter object by assigning role to user
        New-VIPermission -Entity $rootFolder[0] -Principal $zertoPrincipal -Role $ZERTO_ROLE -Propagate:$true -ErrorAction Stop | Out-Null
        Write-Host "ZertoRole ($ZERTO_ROLE) update for ZertoUser ($ZERTO_USER_NAME) was completed successfully"
    }
}

function Invoke-ZertoVcPasswordRotation {
    <#
        .DESCRIPTION
        Rotate password for system Zerto VC user
 
            .PARAMETER ZertoAdminPassword
            Enter the current Zerto Appliance admin Password. This must be a valid password for an existing Zerto admin user account (not a new desired password).
 
            .PARAMETER IgnoreZvmLoginErrors
            Ignore Zerto login errors. This may lead to breaking the Zerto application. Use only when the Zerto password is unknown.
 
        .EXAMPLE
 
        Invoke-ZertoVcPasswordRotation -ZertoAdminPassword password1!
    #>

    [AVSAttribute(30, UpdatesSDDC = $false)]
    param(
        [Parameter(Mandatory = $true, HelpMessage = "Enter the current Zerto Appliance admin Password. This must be a valid password for an existing Zerto admin user account (not a new desired password).")]
        [ValidateNotNullOrEmpty()][SecureString]
        $ZertoAdminPassword,

        [Parameter(Mandatory = $false, HelpMessage = "Ignore Zerto login errors. This may lead to breaking the Zerto application. Use only in emergency case when the Zerto password is unknown.")]
        [bool]
        $IgnoreZertoLoginErrors = $false
    )
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."
        if ((Test-VmExists -VmName $ZVM_VM_NAME) -eq $false) {
            Write-Error "Zerto is not installed, please, install ZVM first." -ErrorAction Stop
        }
        $zertoUser = Get-SsoPersonUser -Name $ZERTO_USER_NAME -Domain $DOMAIN
        if (!$zertoUser) {
            Write-Error "Failed to reset VC password for $ZERTO_USER_NAME, user does not exist." -ErrorAction Stop
        }

        $PersistentSecrets.ZertoAdminPassword = ConvertFrom-SecureString -SecureString $ZertoAdminPassword -AsPlainText
        if (!$IgnoreZertoLoginErrors) {
            Test-ZertoPassword
        }

        try {
            $newPassword = New-RandomPassword
            Set-SsoPersonUser -User $zertoUser -NewPassword $newPassword -ErrorAction Stop | Out-Null
            $PersistentSecrets.ZertoPassword = $newPassword
        }
        catch {
            Write-Error "Failed to reset VC password for $ZERTO_USER_NAME. Please try again. Problem: $_" -ErrorAction Stop
        }
        Update-VcPasswordInZvm
    }
}

function Set-SSHTimeout {
    <#
        .DESCRIPTION
        Determines how long SSH session remains open
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
            .PARAMETER SSHTimeout
            SSH timeout value
 
        .EXAMPLE
 
        SetSSHTimeout -HostName <HostName> -SSHTimeout <SSHTimeout>
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "SSH timeout value")]
        [string]$SSHTimeout
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $vmHost = Get-VMHost -Name $HostName

        Get-AdvancedSetting -Entity $vmHost -Name "UserVars.ESXiShellInteractiveTimeOut" -ErrorAction SilentlyContinue | Set-AdvancedSetting -Value $SSHTimeout -Confirm:$false -ErrorAction SilentlyContinue
        Write-Host "Set configuration setting ""UserVars.ESXiShellInteractiveTimeOut"" on $HostName to $SSHTimeout"
    }
}

function Validate-AndUploadFilesFromPSEngineToHost {
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    New-ZertoFolderOnHost -HostName $HostName

    foreach ($file in Get-ChildItem $LOCAL_TEMP_FOLDER_FOR_DOWNLOADED_FILES* -Include *.sh, *.o) {
        $signature = ("{0}_signature" -f $file)

        if ((Validate-FileBySignature -FilePath $file -SignatureFilePath $signature)) {
            Set-SFTPItem -SessionId ($SFTP_Sessions[$HostName]).Value.SessionId -Destination $ZERTO_FOLDER_ON_HOST -Path $file -Force
        }
        else {
            throw "Error! host $HostName failed to verify $file with $signature, openSSL output: $isVerified"
        }
    }
}

function Get-FilesFromDatastoreToPSEngine {
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid")]
        [string]$BiosUuid
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    $psDriverName = "ds"
    $FullRemoteFileLocation = ('{0}:\zagentid\{1}\*' -f $psDriverName, $BiosUuid)
    $datastore = Get-Datastore $DatastoreName

    Write-Host "Attempting to download files from $darastore ($FullRemoteFileLocation) to PS engine ($LOCAL_TEMP_FOLDER_FOR_DOWNLOADED_FILES)."

    New-PSDrive -Location $datastore -Name $psDriverName -PSProvider VimDatastore -Root "\" -ErrorAction Stop
    Copy-DatastoreItem -Item $FullRemoteFileLocation -Destination $LOCAL_TEMP_FOLDER_FOR_DOWNLOADED_FILES -Force -ErrorAction Stop

    $files = (Get-ChildItem -Path $LOCAL_TEMP_FOLDER_FOR_DOWNLOADED_FILES -Name) -join ";"
    Write-Host "ZertoFiles: {$files} were copied from datastore $DatastoreName ($FullRemoteFileLocation) to PS engine ($LOCAL_TEMP_FOLDER_FOR_DOWNLOADED_FILES)"

    Remove-PSDrive -Name $psDriverName
}

function Copy-FilesFromDatastoreToHost {
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid")]
        [string]$BiosUuid
    )

    Write-Host "Starting $($MyInvocation.MyCommand)..."

    Get-FilesFromDatastoreToPSEngine -DatastoreName $DatastoreName -BiosUuid $BiosUuid
    Validate-AndUploadFilesFromPSEngineToHost -HostName $HostName
}

function Invoke-SSHCommands {
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Commands to execute")]
        [String[]]$Commands,
        [Parameter(Mandatory = $false,
            HelpMessage = "Action on exitStatus 1")]
        [string]$ExitStatusAction = "Stop"
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $NamedOutputs = @{}
        Set-Variable -Name NamedOutputs -Value $NamedOutputs -Scope Global

        $i = 0
        foreach ($Command in $Commands) {
            $SSH = Invoke-SSHCommand -SSHSession $SSH_Sessions[$HostName].Value -Command $Command

            if (!$SSH) {
                throw "Error! failed to Invoke-SSHCommand ""$Command"" on host $HostName"
            }

            $ExitStatus = $SSH.ExitStatus
            $SshError = $SSH.Error
            $Output = ($SSH.Output -join ";")

            if ($ExitStatus -ne 0 -Or $SshError) {
                if (($ExitStatus -eq 1) -And (!$SshError) -And ($ExitStatusAction -eq "Skip")) {
                    Write-Host "ExitStatus of ""$Command"" is 1, while ExitStatusAction = Skip. Skipping..."
                }
                else {
                    throw "Error! failed to run ""$Command"" on host $HostName, ExitStatus: $ExitStatus, Output: $Output, Error: $SshError, ExitStatusAction: $ExitStatusAction"
                }
            }

            Write-Host "Command was completed successfully ""$Command"" on host $HostName, ExitStatus: $ExitStatus, Output: $Output, Error: $SshError"

            $NamedOutputs["$($i)_cmd"] = $Command
            $NamedOutputs["$($i)_exitStatus"] = $ExitStatus
            $NamedOutputs["$($i)_output"] = $Output
            $NamedOutputs["$($i)_error"] = $SshError

            $i++;
        }

        return $NamedOutputs
    }
}

function Get-HostTempFolderInfo {
    <#
        .DESCRIPTION
 
        Display information about the available disk space (For Internal Use)
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
        .EXAMPLE
 
        Get-HostTempFolderInfo -HostName xxx.xxx.xxx.xxx
    #>

    [CmdletBinding()]
    [AVSAttribute(5, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $Command = "vdf"
        return Invoke-SSHCommands -HostName $HostName -Commands $Command
    }
}

function Test-HostConnectivity {
    <#
        .DESCRIPTION
 
        Check if the host is up and running (For Internal Use)
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
        .EXAMPLE
 
        Test-HostConnectivity -HostName xxx.xxx.xxx.xxx
    #>

    [CmdletBinding()]
    [AVSAttribute(5, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $Command = "echo testing123"
        return Invoke-SSHCommands -HostName $HostName -Commands $Command
    }
}

function Get-HostEsxiVersion {
    <#
        .DESCRIPTION
 
        Retrieve the ESXi version (For Internal Use)
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
        .EXAMPLE
 
        Get-HostEsxiVersion -HostName xxx.xxx.xxx.xxx
    #>

    [CmdletBinding()]
    [AVSAttribute(5, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        $Command = "vmware -l"
        return Invoke-SSHCommands -HostName $HostName -Commands $Command
    }
}

function Update-StartupFile {
    <#
        .DESCRIPTION
 
        Responsible for loading the driver when the host is booting.
        /etc/rc.local.d/local.sh file is executed after all the normal system services are started
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
            .PARAMETER DatastoreName
            Datastore Name
 
            .PARAMETER BiosUuid
            "Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid"
 
        .EXAMPLE
 
        Update-StartupFile -HostName xxx.xxx.xxx.xxx -DatastoreName xxx -BiosUuid xxx
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid")]
        [string]$BiosUuid,
        [Parameter(Mandatory = $true,
            HelpMessage = "Zerto driver memory size in MB")]
        [string]$DriverMemoryInMB,
        [Parameter(Mandatory = $true,
            HelpMessage = "Use explicit argument for zloadmod script (True / False)")]
        [string]$UseExplicitDriverArgs
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        if (((Validate-DatastoreName -DatastoreName $DatastoreName) -ne $true) -or ((Validate-BiosUUID -DatastoreName $DatastoreName -BiosUuid $BiosUuid) -ne $true) -or ((Validate-DigitsOnly -InputString $DriverMemoryInMB) -ne $true)) {
            throw "validation failed"
        }

        $zloadmod = ('{0}/zloadmod.sh' -f $ZERTO_FOLDER_ON_HOST)

        Copy-FilesFromDatastoreToHost -HostName $HostName -DatastoreName $DatastoreName -BiosUuid $BiosUuid

        $startupFile = ('{0}/startup_file.sh' -f $ZERTO_FOLDER_ON_HOST)

        if ($UseExplicitDriverArgs -eq $true) {
            $driverArgs = "load -ds $DatastoreName -uid $BiosUuid -mem $DriverMemoryInMB -avs"
        }
        else {
            $driverArgs = "load $DatastoreName $BiosUuid 0 `"`" 1"
        }

        $Commands = ('grep -v "ZeRTO\|exit 0" /etc/rc.local.d/local.sh > {0}' -f $startupFile),
        ('echo \#ZeRTO\ >> {0}' -f $startupFile),
        ('echo sh {0} {1} \> /etc/vmware/zloadmod.txt \2\>\&\1 \#ZeRTO\ >> {2}' -f $zloadmod, $driverArgs, $startupFile),
        ('echo \#ZeRTO\ >> {0}' -f $startupFile),
        ('echo "exit 0" >> {0}' -f $startupFile),
        ('cp -f {0} /etc/rc.local.d/local.sh' -f $startupFile),
        ('chmod a+x {0}' -f $zloadmod)

        return Invoke-SSHCommands -HostName $HostName -Commands $Commands
    }
}

function Install-Driver {
    <#
        .DESCRIPTION
 
        Install the driver
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
            .PARAMETER DatastoreName
            Datastore Name
 
            .PARAMETER BiosUuid
            Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid
 
            .PARAMETER EsxiVersion
            Esxi version
 
        .EXAMPLE
        Install-Driver -HostName xxx.xxx.xxx.xxx -DatastoreName <DatastoreName> -BiosUuid <UUID> -EsxiVersion xx
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid")]
        [string]$BiosUuid,
        [Parameter(Mandatory = $true,
            HelpMessage = "Esxi version")]
        [string]$EsxiVersion,
        [Parameter(Mandatory = $true,
            HelpMessage = "Driver memory in MB for Zerto driver")]
        [string]$DriverMemoryInMB,
        [Parameter(Mandatory = $true,
            HelpMessage = "Use explicit argument for zloadmod script (True / False)")]
        [string]$UseExplicitDriverArgs
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        if (((Validate-DatastoreName -DatastoreName $DatastoreName) -ne $true) -or ((Validate-BiosUUID -DatastoreName $DatastoreName -BiosUuid $BiosUuid) -ne $true) -or ((Validate-DigitsOnly -InputString $EsxiVersion) -ne $true) -or ((Validate-DigitsOnly -InputString $DriverMemoryInMB) -ne $true)) {
            throw "validation failed"
        }

        if (Test-ZertoDriverLoaded $HostName) {
            Write-Host "Warning! Zerto driver is already loaded on $HostName"
        }

        $zloadmod = ('{0}/zloadmod.sh' -f $ZERTO_FOLDER_ON_HOST)

        Copy-FilesFromDatastoreToHost -HostName $HostName -DatastoreName $DatastoreName -BiosUuid $BiosUuid

        if ($UseExplicitDriverArgs -eq $true) {
            $driverArgs = "init -ds $DatastoreName -uid $BiosUuid -ver $EsxiVersion -mem $DriverMemoryInMB -avs";
        }
        else {
            $driverArgs = "init $DatastoreName $BiosUuid 0 $EsxiVersion 1";
        }

        $Commands = ('chmod a+x {0}' -f $zloadmod),
                    ('{0} {1} > /etc/vmware/zloadmod.txt' -f $zloadmod, $driverArgs)

        return Invoke-SSHCommands -HostName $HostName -Commands $Commands
    }
}

function Uninstall-Driver {
    <#
        .DESCRIPTION
 
        Uninstall the driver
 
            .PARAMETER HostName
            Host Name to connect with SSH
 
            .PARAMETER DatastoreName
            Datastore Name
 
            .PARAMETER BiosUuid
            Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid
 
 
        .EXAMPLE
        Uninstall-Driver -HostName xxx.xxx.xxx.xxx -DatastoreName <DatastoreName> -BiosUuid <UUID>
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false, AutomationOnly = $true)]
    param(
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName,
        [Parameter(Mandatory = $true,
            HelpMessage = "Host Bios Uuid || mob-> Property Path: host.hardware.systemInfo.uuid")]
        [string]$BiosUuid
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        if (Test-ZertoDriverLoaded $HostName) {
            $zunloadmod = ('{0}/zunloadmod.sh' -f $ZERTO_FOLDER_ON_HOST)

            Copy-FilesFromDatastoreToHost -HostName $HostName -DatastoreName $DatastoreName -BiosUuid $BiosUuid

            $Commands = ('chmod a+x {0}' -f $zunloadmod),
                        ('{0} cleanup > /etc/vmware/zunloadmod.txt' -f $zunloadmod)

            return Invoke-SSHCommands -HostName $HostName -Commands $Commands
        }

        else {
            throw "Error! Failed to run Uninstall-Driver, Zerto driver is not loaded on $HostName."
        }
    }
}

function Install-Zerto {
    <#
        .DESCRIPTION
 
        Install Zerto appliance
 
            .PARAMETER MyZertoToken
            My Zerto token for downloading the OVA and signature file
 
            .PARAMETER HostName
            vSphere host name on which to deploy the ZVM Appliance
 
            .PARAMETER DatastoreName
            Datastore name on which to deploy the ZVM Appliance
 
            .PARAMETER AzureTenantId
            Azure tenant ID: unique global identifier, which can be found in the Azure portal
 
            .PARAMETER AzureClientID
            Azure Client ID
 
            .PARAMETER AvsClientSecret
            AVS Client Secret
 
            .PARAMETER NetworkName
            Network name for the ZVM Appliance
 
            .PARAMETER ApplianceIp
            ZVM Appliance IP address
 
            .PARAMETER SubnetMask
            Subnet mask
 
            .PARAMETER DefaultGateway
            Default gateway
 
            .PARAMETER DNS
            DNS name server for the appliance
 
            .PARAMETER ZertoAdminPassword
            Password to be used to authenticate to the ZVM Appliance
 
        .EXAMPLE
        Install-Zerto -MyZertoToken <MyZertoToken> -HostName xxx.xxx.xxx.xxx -ApplianceIp xxx.xxx.xxx.xxx -DatastoreName <DatastoreName> -AzureTenantId <AzureTenantId>
         -AzureClientID <AzureClientID> -AvsClientSecret ********* -NetworkName <NetworkName> -SubnetMask xxx.xxx.xxx.xxx -DefaultGateway xxx.xxx.xxx.xxx
         -DNS xxx.xxx.xxx.xxx -ZertoAdminPassword password1!
    #>

    [CmdletBinding()]
    [AVSAttribute(60, UpdatesSDDC = $false)]
    param(
        [Parameter(Mandatory = $true, HelpMessage = "Token for MyZerto")]
        [ValidateNotNullOrEmpty()][string]
        $MyZertoToken,

        [Parameter(Mandatory = $false, HelpMessage = "The name of the vSphere host on which to deploy the ZVM Appliance")]
        [string]
        $HostName,

        [Parameter(Mandatory = $true, HelpMessage = "The name of the datastore on which to deploy the ZVM Appliance")]
        [ValidateNotNullOrEmpty()][string]
        $DatastoreName,

        [Parameter(Mandatory = $true, HelpMessage = "Your Microsoft Entra tenant ID")]
        [ValidateNotNullOrEmpty()][string]
        $AzureTenantId,

        [Parameter(Mandatory = $true, HelpMessage = "Your Application (client) ID, found in Azure ""App registrations""")]
        [ValidateNotNullOrEmpty()][string]
        $AzureClientID,

        [Parameter(Mandatory = $true, HelpMessage = "Your client secret value, found under your application's ""Certificates & secrets""")]
        [ValidateNotNullOrEmpty()][SecureString]
        $AvsClientSecret,

        [Parameter(Mandatory = $true, HelpMessage = "The name of the network used for the ZVM Appliance")]
        [ValidateNotNullOrEmpty()][string]
        $NetworkName,

        [Parameter(Mandatory = $true, HelpMessage = "ZVM Appliance IP address")]
        [ValidateNotNullOrEmpty()][string]
        $ApplianceIp,

        [Parameter(Mandatory = $true, HelpMessage = "Subnet mask")]
        [ValidateNotNullOrEmpty()][string]
        $SubnetMask,

        [Parameter(Mandatory = $true, HelpMessage = "Default gateway")]
        [ValidateNotNullOrEmpty()][string]
        $DefaultGateway,

        [Parameter(Mandatory = $true, HelpMessage = "DNS server address")]
        [ValidateNotNullOrEmpty()][string]
        $DNS,

        [Parameter(Mandatory = $true, HelpMessage = "Password to authenticate to the ZVM GUI. Password should contain at least one uppercase letter, one non-alphanumeric character, one digit, and be at least 8 characters long.")]
        [ValidateNotNullOrEmpty()][SecureString]
        $ZertoAdminPassword
    )

    process {

        Write-Host "Starting $($MyInvocation.MyCommand)..."

        try {
            if ($SddcResourceId -match $SDDC_RESOURCE_ID_PATTERN) {
                $AvsSubscriptionId = $matches['AvsSubscriptionId']
                $AvsResourceGroup = $matches['AvsResourceGroup']
                $AvsCloudName = $matches['AvsCloudName']
            }
            else {
                throw "Could not find the AvsSubscriptionId, AvsResourceGroup, and AvsCloudName combination in SddcResourceId ($SddcResourceId). Please contact Microsoft support."
            }

            $DeployVmStarted = $false

            Validate-AvsParams -TenantId $AzureTenantId -ClientId $AzureClientID -ClientSecret $AvsClientSecret -SubscriptionId $AvsSubscriptionId -ResourceGroupName $AvsResourceGroup -AvsCloudName $AvsCloudName

            $HostName = Get-ValidatedHostName -HostName $HostName

            Validate-VcEnvParams -DatastoreName "$DatastoreName" -ZVMLIp "$ApplianceIp" -SubnetMask "$SubnetMask" -DefaultGateway "$DefaultGateway" -DNS "$DNS" -NetworkName "$NetworkName"
            Validate-ZertoPassword -Password $ZertoAdminPassword

            New-ZertoUser

            $OvaFilePath = Get-ZertoOVAFile -MyZertoToken $MyZertoToken

            $PersistentSecrets.ZappliancePassword = New-RandomPassword
            $PersistentSecrets.ZertoAdminPassword = ConvertFrom-SecureString -SecureString $ZertoAdminPassword -AsPlainText
            $PersistentSecrets.AvsClientSecret = ConvertFrom-SecureString -SecureString $AvsClientSecret -AsPlainText

            $DeployVmStarted = $true

            Deploy-Vm -OvaPath $OvaFilePath -VMHostName $HostName -DatastoreName $DatastoreName -ZVMLIp $ApplianceIp -NetworkName $NetworkName -SubnetMask $SubnetMask -DefaultGateway $DefaultGateway -DNS $DNS -AzureTenantId $AzureTenantId -AzureClientID $AzureClientID -AvsSubscriptionId $AvsSubscriptionId -AvsResourceGroup $AvsResourceGroup -AvsCloudName $AvsCloudName

            Write-Host "The ZVM Appliance (ZVMA) was successfully deployed."
        }
        catch {
            $DeploymentFailureMessage = "The ZVM Appliance (ZVMA) failed to be deployed. Resolve the reported issues and run the installation again.`n Reasons for failure include: $_"

            try {
                if ($DeployVmStarted -eq $true) {
                    Remove-ZVMAppliance
                }
            }
            catch {
                Write-Host $_
                Write-Error $_
            }

            Write-Host $DeploymentFailureMessage
            Write-Error $DeploymentFailureMessage -ErrorAction Stop
        }

        try {
            Start-ZVM
            Set-ZertoConfiguration
        }
        catch {
            Write-Error "Failed to configure Zerto. Error: $_" -ErrorAction Stop
        }

        # The following success message will not be written if script execution was aborted with ErrorAction Stop
        Write-Host "To navigate to Zerto UI, in a browser go to https://$ApplianceIp"
    }
}

function Uninstall-Zerto {
    <#
        .DESCRIPTION
 
        Uninstall Zerto appliance
 
        .PARAMETER Confirmation
        Confirmation for uninstalling Zerto appliance that check if the input is "uninstall"
 
 
        .EXAMPLE
        Uninstall-Zerto -Confirmation "uninstall"
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false)]
    param(
        [Parameter(Mandatory = $true, HelpMessage = "Confirmation for unistall, type 'uninstall' to confirm.")]
        [ValidateNotNullOrEmpty()][string]
        $Confirmation
    )

    process {

        Write-Host "Starting $($MyInvocation.MyCommand)..."

        if ($Confirmation -ne "uninstall") {
            Write-Error "Type 'uninstall' to confirm Zerto uninstall." -ErrorAction Stop
        }

        try {
            Remove-ZertoUser
            Remove-ZertoRole
            Remove-ZVMAppliance

            Write-Host "ZVM Appliance uninstall was completed successfully"
        }
        catch {
            $Message = "Failed to uninstall ZVM Appliance. $_"
            Write-Host $Message
            Write-Error $Message -ErrorAction Stop
        }
    }
}

function Remove-ZVMAppliance {
    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        try {
            if ((Test-VmExists -VmName $ZVM_VM_NAME) -eq $true) {
                $VM = Get-VM $ZVM_VM_NAME

                Stop-ZVM #TODO:GK can VM be force deleted if it is not stopped?

                Write-Host "Deleting $ZVM_VM_NAME VM from disk"
                Remove-VM -VM $VM -DeletePermanently -confirm:$false

                Write-Host "ZVM Appliance removed successfully"
            }
            else {
                Write-Host "$ZVM_VM_NAME does not exist"
            }
        }
        catch {
            Write-Error "Failed to remove ZVM Appliance. $_" -ErrorAction Stop
        }
    }
}

function Restart-ZertoAppliance {
    <#
        .DESCRIPTION
 
        Restart Zerto Appliance
 
        .PARAMETER Confirmation
        Confirmation for restarting Zerto appliance that check if the input is "yes"
 
 
        .EXAMPLE
        Restart-Zerto -Confirmation "restart"
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false)]
    param(
        [Parameter(Mandatory = $true, HelpMessage = "Confirmation for restart, type 'restart' to confirm.")]
        [ValidateNotNullOrEmpty()][string]
        $Confirmation
    )

    process {
        Write-Host "Starting $($MyInvocation.MyCommand)..."

        if ($Confirmation -ne "restart") {
            Write-Error "Type 'restart' to confirm appliance restart." -ErrorAction Stop
        }

        try {
            if ((Test-VmExists -VmName $ZVM_VM_NAME) -eq $true) {
                Stop-ZVM
                Start-ZVM

                Write-Host "Zerto restart was completed successfully .."
            }
            else {
                Write-Error "$ZVM_VM_NAME does not exist, failed to restart Zerto"
            }
        }
        catch {
            $Message = "Failed to restart Zerto. $_"
            Write-Host $Message
            Write-Error $Message -ErrorAction Stop
        }
    }
}

function Remove-ZertoUser {
    <#
        .DESCRIPTION
        Remove Zerto User
 
        .EXAMPLE
        Remove-ZertoUser
    #>

    process {
        try {
            $user = Get-SsoPersonUser -Name $ZERTO_USER_NAME -Domain $DOMAIN

            if ($null -ne $user) {
                Remove-SsoPersonUser -User $user
                Write-Host "User '$ZERTO_USER_NAME' has been successfully removed."
            }
            else {
                Write-Host "User '$ZERTO_USER_NAME' not found."
            }
        }
        catch {
            Write-Host "An error occurred while removing the user: $_"
        }
    }
}

function Remove-ZertoRole {
    <#
        .DESCRIPTION
        Remove Zerto Role
 
        .EXAMPLE
        Remove-ZertoRole
    #>

    process {

        try {
            $role = Get-VIRole -Name $ZERTO_ROLE

            if ($null -ne $role) {
                Remove-VIRole -Role $role -Force:$true -Confirm:$false
                Write-Host "Role '$ZERTO_ROLE' has been successfully removed."
            }
            else {
                Write-Host "Role '$ZERTO_ROLE' not found."
            }
        }
        catch {
            Write-Host "An error occurred while removing the role: $_"
        }
    }
}

function Debug-Zerto {
    <#
        .DESCRIPTION
 
        Diagnosing and troubleshooting Zerto components
 
        .EXAMPLE
        Debug-Zerto
        Debug-Zerto -HostName <HostName> -DatastoreName <DatastoreName>
    #>

    [CmdletBinding()]
    [AVSAttribute(30, UpdatesSDDC = $false)]
    param(
        [Parameter(Mandatory = $false,
            HelpMessage = "Host Name to connect with SSH")]
        [string]$HostName,

        [Parameter(Mandatory = $false,
            HelpMessage = "Datastore Name")]
        [string]$DatastoreName

    )

    process {

        Write-Host "Starting $($MyInvocation.MyCommand)..."

        try {
            Test-Connection
            Test-ZertoRoleExists
            Test-ZertoUserExists

            if ($HostName) {
                Write-Host "Host Provided by user: $HostName"
                Test-ZertoDriverLoaded -HostName $HostName
                Get-ZertoFilesListFromHost -HostName $HostName

                if ($Datastore) {
                    Write-Host "Datastore Provided by user: $DatastoreName"
                    Get-DriverLogsFromHost -HostName $HostName -DatastoreName $DatastoreName
                }
            }
        }
        catch {
            $Message = "Failed to run Debug-Zerto, $_"
            Write-Host $Message
            Write-Error $Message -ErrorAction Stop
        }
    }
}