AksHci.psm1
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # AksHci Day 0/2 Operations # ######################################################################################### #requires -runasadministrator using module .\Common.psm1 #region Module Constants $moduleName = "AksHci" $moduleVersion = "1.2.16" #endregion #region requires #Requires -Modules @{ModuleName="Az.Resources"; RequiredVersion="4.4.0"; GUID="48bb344d-4c24-441e-8ea0-589947784700"} #Requires -Modules @{ModuleName="Az.Accounts"; RequiredVersion="2.6.0"; GUID="17a2feff-488b-47f9-8729-e2cec094624c"} #Requires -Modules @{ModuleName="AzureAD"; RequiredVersion="2.0.2.137"; GUID="d60c0004-962d-4dfb-8d28-5707572ffd00"} #endregion #region Download catalog constants $catalogName = "aks-hci-stable-catalogs-ext" $ringName = "stable" # Default AksHci product name from catalog $defaultProductName = "aks-hci-releases" # Major version is checked for compatibilty between PS and product $supportedProductVersion = "1.0.0" #endregion #region Aliases Set-Alias -Name Initialize-AksHciNode -Value Initialize-MocNode Set-Alias -Name Update-AksHciClusterCertificates -Value Repair-AksHciClusterCerts Set-Alias -Name Update-AksHciCertificates -Value Repair-AksHciCerts Set-Alias -Name Restart-AksHci -Value Reinstall-AksHci #endregion #region # Install Event Log New-ModuleEventLog -moduleName $moduleName #endregion #region Private Function function Initialize-AksHciConfiguration { <# .DESCRIPTION Initialize AksHci Configuration Wipes off any existing cached configuration #> if ($global:config.ContainsKey($moduleName)) { $global:config.Remove($moduleName) } $global:config += @{ $moduleName = @{ "installationPackageDir" = "" "installState" = [InstallState]::NotInstalled "manifestCache" = "" "moduleVersion" = $moduleVersion "skipUpdates" = $false "stagingShare" = "" "useStagingShare" = $false "version" = "" "workingDir" = "" "catalog" = "" "ring" = "" "proxyServerCertFile" = "" "proxyServerHTTP" = "" "proxyServerHTTPS" = "" "proxyServerNoProxy" = "" "proxyServerPassword" = "" "proxyServerUsername" = "" "deploymentId" = "" "caCertRotationThreshold" = 90 "offlineDownload" = $false "offsiteTransferCompleted"= $false "concurrentDownloads" = $global:smallBinConcurrentDownloads "enableOptionalDiagnosticData" = $false "mocInstalledByAksHci" = $false "useHTTPSForDownloads" = $false }; } } #endregion #region global config Initialize-AksHciConfiguration #endregion Import-LocalizedData -BindingVariable "GenericLocMessage" -FileName commonLocalizationMessages Import-LocalizedData -BindingVariable "AksHciLocMessage" -FileName AksHciLocalizationMessages #region Exported Functions function New-AksHciNetworkSetting { <# .SYNOPSIS Create an object for a new virtual network. .DESCRIPTION Create a virtual network to set the DHCP or static IP address for the control plane, load balancer, agent endpoints, and a static IP range for nodes in all Kubernetes clusters. This cmdlet will return a VirtualNetwork object, which can be used later in the configuration steps. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER MacPoolName The name of the mac pool .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .EXAMPLE New-AksHciNetworkSetting -name External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE New-AksHciNetworkSetting -name "Defualt Switch" -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNetworkName -Name $_})] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [String] $MacPoolName = $global:cloudMacPool, [Parameter(Mandatory=$false)] [ValidateRange(0, 4094)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIPPrefix -ipprefix $_})] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $gateway, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidDNSServers -dnsservers $_})] [String[]] $dnsservers, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $vippoolstart, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $vippoolend, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $k8snodeippoolend ) $startCmdletTime = Get-Date $networkdetailsCmdletParams = @{name= $name; vswitchName= $vswitchName; MacPoolName= $MacPoolName } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $networkdetailsCmdletParams throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $networkdetailsCmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys return New-VirtualNetwork -name $name -vswitchName $vswitchName -MacPoolName $MacPoolName -vlanID $vlanID -ipaddressprefix $ipaddressprefix ` -gateway $gateway -dnsservers $dnsservers -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend } function New-AksHciSSHConfiguration { <# .SYNOPSIS Create an object for a new ssh configuration. .DESCRIPTION Create a SSH configuration for AksHci virtual machines to define SSH access. .PARAMETER name The name of the sshConfiguration .PARAMETER sshPublicKey Path to an SSH public key file. Using this public key, you will be able to log in to any of the VMs created by the Azure Kubernetes Service on Azure Stack HCI deployment. If you have your own SSH public key, you will pass its location here. If no key is provided, we will look for one under %systemdrive%\akshci\.ssh\akshci_rsa.pub. If the file does not exist, an SSH key pair in the above location will be generated and used. .PARAMETER sshPrivateKey The path to sshPrivateKey file .PARAMETER restrictSSHCommands Restict SSH access to certain commands .PARAMETER ipAddresses Restict SSH access to certain ipaddresses .PARAMETER cidr Restict SSH access to a CIDR .OUTPUTS SSHConfiguration object .EXAMPLE New-AksHciSSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub .EXAMPLE New-AksHciSSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub -cidr 172.16.0.0/24 .EXAMPLE New-AksHciSSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub -ipAddresses 4.4.4.4,8.8.8.8 .EXAMPLE New-AksHciSSHConfiguration -name sshConfig -cidr 172.16.0.0/24 .EXAMPLE New-AksHciSSHConfiguration -name sshConfig -ipAddresses 4.4.4.4,8.8.8.8 .EXAMPLE New-AksHciSSHConfiguration -name sshConfig -ipAddresses 4.4.4.4,8.8.8.8 -restrictSSHCommands #> [CmdletBinding(DefaultParameterSetName = 'noip')] param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [String] $name, [Parameter(Mandatory=$false)] [String] $sshPublicKey, [Parameter(Mandatory=$false)] [String] $sshPrivateKey, [Parameter(Mandatory=$false)] [Switch] $restrictSSHCommands = $false, [Parameter(Mandatory=$true, ParameterSetName='ipaddresses')] [String[]] $ipAddresses, [Parameter(Mandatory=$true, ParameterSetName='cidr')] [String] $cidr ) $startCmdletTime = Get-Date $sshdetailsCmdletParams = @{name= $name; restrictSSHCommands= $restrictSSHCommands; } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $sshdetailsCmdletParams throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $sshdetailsCmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys if ($PSCmdlet.ParameterSetName -ieq "ipaddresses") { return New-SSHConfiguration -name $name -sshPublicKey $sshPublicKey -sshPrivateKey $sshPrivateKey -ipAddresses $ipaddresses -restrictSSHCommands:$restrictSSHCommands.IsPresent } if ($PSCmdlet.ParameterSetName -ieq "cidr") { return New-SSHConfiguration -name $name -sshPublicKey $sshPublicKey -sshPrivateKey $sshPrivateKey -cidr $cidr -restrictSSHCommands:$restrictSSHCommands.IsPresent } return New-SSHConfiguration -name $name -sshPublicKey $sshPublicKey -sshPrivateKey $sshPrivateKey -restrictSSHCommands:$restrictSSHCommands.IsPresent } function Test-ModuleCompatibility { <# .DESCRIPTION Tests if the requested product version is compatible with the version(s) understood by this Powershell module. .PARAMETER Version The AKS HCI product version to be tested for compatibility. #> param ( [String] $Version ) $result = Compare-Versions -Version $script:supportedProductVersion -ComparisonVersion $Version if ($result -eq 0) { return $true } $errorMsg = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_incompatible_version, $Version, $moduleName, $moduleVersion)) if ($result -lt 0) { $errorMsg += $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_older_version, $moduleName)) } else { $errorMsg += $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_newer_version, $moduleName)) } throw $errorMsg } function Set-AksHciDownloadConfig { <# .DESCRIPTION Configures download settings for the AksHci module. .PARAMETER activity Activity name to use when updating progress. .PARAMETER useHttps Use HTTPS for downloads performed by the module. #> param ( [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$true)] [Bool] $useHttps ) $startCmdletTime = Get-Date $configCmdletParams = @{useHttps= $useHttps} trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) throw $_ } try { Import-AksHciConfig -activity $activity } catch {} Set-AksHciConfigValue -name "useHTTPSForDownloads" -value $useHttps Set-KvaDownloadConfig -useHttps:$useHttps Set-MocDownloadConfig -useHttps:$useHttps Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Set-AksHciOffsiteConfig { param ( [parameter()] # [ValidateScript({Test-ValidDirectoryPath -dirPath $_})] [String] $workingDir = $global:defaultWorkingDir, [parameter()] [String] $catalog = $script:catalogName, [parameter()] [String] $ring = $script:ringName, [Parameter(Mandatory=$true)] [String] $stagingShare = $global:defaultStagingShare, [parameter()] [String] $version ) $startCmdletTime = Get-Date $configCmdletParams = @{catalog= $catalog; ring= $ring; version= $version} trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) throw $_ } if ([string]::IsNullOrWhiteSpace($stagingShare)) { throw [CustomException]::new($($GenericLocMessage.generic_staging_share_unspecified), ([ErrorTypes]::IsUserErrorFlag)) } try { Import-AksHciConfig -activity $activity } catch {} $currentState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($currentState) { switch ($currentState) { ([InstallState]::NotInstalled) { # Fresh install break } Default { Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_current_state, $currentState)) throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_no_new_config_in_current_state, $moduleName, $currentState)) } } } Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Set-AksHciConfigValue -name "workingDir" -value $workingDir Set-AksHciConfigValue -name "manifestCache" -value ([io.Path]::Combine($workingDir, $("$catalog.json"))) New-Item -ItemType Directory -Force -Path $workingDir | out-null Set-AksHciConfigValue -name "stagingShare" -value $stagingShare Set-AksHciConfigValue -name "offlineDownload" -value $true Set-AksHciConfigValue -name "offsiteTransferCompleted" -value $false if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if (-not $version) { # If no version is specified, use the latest $version = Get-AksHciLatestVersion Set-AksHciConfigValue -name "version" -value $version } } else { Get-AksHciLatestVersion | out-null # This clears the cache Get-ProductRelease -Version $version -module $moduleName | Out-Null Set-AksHciConfigValue -name "version" -value $version } Set-KvaOffsiteConfig -catalog $catalog -ring $ring -workingDir $workingDir -version $version -stagingShare $stagingShare Set-MocOffsiteConfig -catalog $catalog -ring $ring -workingDir $workingDir -version $version -stagingShare $stagingShare Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Set-AksHciConfig { <# .SYNOPSIS Set or update the configurations settings for the Azure Kubernetes Service host. .DESCRIPTION Set the configuration settings for the Azure Kubernetes Service host. If you're deploying on a 2-4 node Azure Stack HCI cluster or a Windows Server 2019 Datacenter failover cluster, you must specify the imageDir and cloudConfigLocation parameters. For a single node Windows Server 2019 Datacenter, all parameters are optional and set to their default values. However, for optimal performance, we recommend using a 2-4 node Azure Stack HCI cluster deployment. .PARAMETER workingDir This is a working directory for the module to use for storing small files. Defaults to %systemdrive%\akshci for single node deployments. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as c:\ClusterStorage\Volume2\ImageStore or an SMB share such as \\FileShare\ImageStore. .PARAMETER imageDir The path to the directory where Azure Kubernetes Service on Azure Stack HCI will store its VHD images. Defaults to %systemdrive%\AksHciImageStore for single node deployments. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as C:\ClusterStorage\Volume2\ImageStore or a SMB share such as \\fileshare\ImageStore. .PARAMETER isolateImageDir Whether or not, isolate the imageDir from VHD auto placement. .PARAMETER version The version of Azure Kubernetes Service on Azure Stack HCI that you want to deploy. The default is the latest version. We do not recommend changing the default. .PARAMETER cloudConfigLocation The location where the cloud agent will store its configuration. Defaults to %systemdrive%\wssdcloudagent for single node deployments. The location can be the same as the path of -imageDir. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as C:\ClusterStorage\Volume2\ImageStore or an SMB share such as \\fileshare\ImageStore. The location needs to be on a highly available share so that the storage will always be accessible. .PARAMETER nodeConfigLocation The location where the node agents will store their configuration. Every node has a node agent, so its configuration is local to it. This location must be a local path. Defaults to %systemdrive%\programdata\wssdagent for all deployments. .PARAMETER cloudLocation This parameter provides a custom Microsoft Operated Cloud location name. The default name is "MocLocation". We do not recommend changing the default. .PARAMETER createAutoConfigContainers This parameter enables or disables generating auto-config-container- folders for Moc. The default value is $true. .PARAMETER isolateAutoConfiguredContainerLike Isolate the auto configured containers if its path like specified pattern. The default is not to isolate any auto-configured container. .PARAMETER vnet A VirtualNetwork object created using the New-AksHciNetworkSetting cmdlet. .PARAMETER ssh A SSH Configuration object created using the New-AksHciSSHConfiguration cmdlet. .PARAMETER controlplaneVmSize The size of the VM to create for the control plane. To get a list of available VM sizes, use Get-AksHciVmSize. .PARAMETER kvaName Kubernetes Virtual Appliance name. We do not recommend changing the default. .PARAMETER kvaPodCIDR Configures the Kubernetes POD CIDR. We do not recommend changing the default. .PARAMETER nodeAgentPort The TCP/IP port number that node agents should listen on. Defaults to 45000. We do not recommend changing the default. .PARAMETER nodeAgentAuthorizerPort The TCP/IP port number that node agents should use for their authorization port. Defaults to 45001. We do not recommend changing the default. .PARAMETER cloudAgentPort The TCP/IP port number that cloud agent should listen on. Defaults to 55000. We do not recommend changing the default. .PARAMETER cloudAgentAuthorizerPort The TCP/IP port number that cloud agent should use for its authorization port. Defaults to 65000. We do not recommend changing the default. .PARAMETER clusterRoleName This specifies the name to use when creating cloud agent as a generic service within the cluster. This defaults to a unique name with a prefix of ca- and a guid suffix (for example: "ca-9e6eb299-bc0b-4f00-9fd7-942843820c26"). We do not recommend changing the default. .PARAMETER cloudServiceIP This can be used to provide a static IP address to be assigned to the MOC CloudAgent service. This value should be provided using the standard IPv4 format. (Example: 192.168.1.2). The cloudServiceIP address should also be carved out of one of the ClusterAndClient networks in the underlying Failover cluster. You can run Get-ClusterNetwork command in an elevated powershell mode to find the network role. You may want to specify this to ensure that anything important on the network is always accessible because the IP address will not change. Please note that you can also provide a cloudServiceCidr IP/netowrk address prefix. (Example: 192.168.1.2/16). We do not recommend using cloudServiceCidr. Default is none. .PARAMETER proxySettings A ProxySettings object created using the New-AksHciProxySetting cmdlet. .PARAMETER sshPublicKey Path to an SSH public key file. Using this public key, you will be able to log in to any of the VMs created by the Azure Kubernetes Service on Azure Stack HCI deployment. If you have your own SSH public key, you will pass its location here. If no key is provided, we will look for one under %systemdrive%\akshci\.ssh\akshci_rsa.pub. If the file does not exist, an SSH key pair in the above location will be generated and used. .PARAMETER skipHostLimitChecks Requests the script to skip any checks it does to confirm memory and disk space is available before allowing the deployment to proceed. We do not recommend using this setting. .PARAMETER skipRemotingChecks Requests the script to skip any checks it does to confirm remoting capabilities to both local and remote nodes. We do not recommend using this setting. .PARAMETER forceDnsReplication DNS replication can take up to an hour on some systems. This will cause the deployment to be slow. To bypass this issue, try to use this flag. The -forceDnsReplication flag is not a guaranteed fix. If the logic behind the flag fails, the error will be hidden, and the command will carry on as if the flag was not provided. .PARAMETER macPoolStart This is used to specify the start of the MAC address of the MAC pool that you wish to use for the Azure Kubernetes Service host VM. The syntax for the MAC address requires that the least significant bit of the first byte should always be 0, and the first byte should always be an even number (that is, 00, 02, 04, 06...). A typical MAC address can look like: 02:1E:2B:78:00:00. Use MAC pools for long-lived deployments so that MAC addresses assigned are consistent. This is useful if you have a requirement that the VMs have specific MAC addresses. Default is none. .PARAMETER macPoolEnd This is used to specify the end of the MAC address of the MAC pool that you wish to use for the Azure Kubernetes Service host VM. The syntax for the MAC address requires that the least significant bit of the first byte should always be 0, and the first byte should always be an even number (that is, 00, 02, 04, 06...). The first byte of the address passed as the -macPoolEnd should be the same as the first byte of the address passed as the -macPoolStart. Use MAC pools for long-lived deployments so that MAC addresses assigned are consistent. This is useful if you have a requirement that the VMs have specific MAC addresses. Default is none. .PARAMETER useStagingShare Reserved for internal use. We do not recommend using this parameter. .PARAMETER containerRegistry Reserved for internal use. We do not recommend using this parameter. .PARAMETER catalog Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Reserved for internal use. We do not recommend using this parameter. .PARAMETER deploymentId Reserved for internal use. We do not recommend using this parameter. .PARAMETER skipUpdates Reserved for internal use. We do not recommend using this parameter. .PARAMETER stagingShare Reserved for internal use. We do not recommend using this parameter. .PARAMETER kvaSkipWaitForBootstrap Reserved for internal use. We do not recommend using this parameter. .PARAMETER deploymentType Reserved for internal use. We do not recommend using this parameter. .PARAMETER activity Reserved for internal use. We do not recommend using this parameter. .PARAMETER enablePreview Enable AKS HCI Early Access Preview feature on Azure. .PARAMETER skipCleanOnFailure Skip auto cleanup on installation failure .PARAMETER caCertRotationThreshold Threshold days on certificate expiry on when cloudagent CA certificte should be rotated .PARAMETER concurrentDownloads How many parts to segment content downloads into (causes concurrent connections to the hosting server) for the big binaries. .PARAMETER vipPool The global vipPool to use. .PARAMETER offlineDownload For setting to offline download scenario. .PARAMETER offsiteTransferCompleted For offline download offsite scenario to use the offline downloaded artifacts. .PARAMETER enableOptionalDiagnosticData Send information about how you use features, plus additional information about service health, activity, and enhanced error reporting. Diagnostic data is used to help keep the service secure and up to date, troubleshoot problems, and make product improvements. Required diagnostic data will always be included when you choose to send Optional diagnostic data. Regardless of your choice, the service will be equally secure and operate normally. .PARAMETER skipValidationCheck Skips running validation checks if the flag is passed .PARAMETER mode Different modes for choosing different Linux kubernetes versions to download. .PARAMETER useHTTPSForDownloads If enabled, https/tls download URLS will be used (if supported by the download provider) #> [CmdletBinding(DefaultParametersetName='None')] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter()] # [ValidateScript({Test-ValidDirectoryPath -dirPath $_})] [String] $workingDir = $global:defaultWorkingDir, [parameter()] # [ValidateScript({Test-ValidDirectoryPath -dirPath $_})] [String] $imageDir, [parameter()] [switch] $isolateImageDir, [parameter()] # [ValidateScript({Test-ValidVersionNumber -VersionNumber $_})] [String] $version, [parameter()] [String] $stagingShare = $global:defaultStagingShare, [parameter()] # [ValidateScript({Test-ValidDirectoryPath -dirPath $_})] [String] $cloudConfigLocation = $global:defaultCloudConfigLocation, [parameter()] # [ValidateScript({Test-ValidDirectoryPath -dirPath $_})] [String] $nodeConfigLocation = $global:defaultNodeConfigLocation, [parameter()] [String] $cloudLocation = $global:defaultCloudLocation, [parameter()] [ValidateSet($true, $false)] [bool] $createAutoConfigContainers = $global:defaultCreateAutoConfigContainers, [parameter()] [string]$isolateAutoConfiguredContainerLike = '', [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet, [parameter()] [SSHConfiguration] $ssh, [parameter()] [VmSize] $controlplaneVmSize = $global:defaultMgmtControlPlaneVmSize, [parameter(DontShow)] [String] $kvaName = (New-Guid).Guid, [parameter()] [String] $kvaPodCIDR = $global:defaultPodCidr, [parameter(DontShow)] [Switch] $kvaSkipWaitForBootstrap, [parameter()] [ValidateRange(1,65535)] [int] $nodeAgentPort = $global:defaultNodeAgentPort, [parameter()] [ValidateRange(1,65535)] [int] $nodeAgentAuthorizerPort = $global:defaultNodeAuthorizerPort, [parameter()] [ValidateRange(1,65535)] [int] $cloudAgentPort = $global:defaultCloudAgentPort, [parameter()] [ValidateRange(1,65535)] [int] $cloudAgentAuthorizerPort = $global:defaultCloudAuthorizerPort, [parameter()] # [ValidateScript({Test-ValidClusterName -name $_})] [String] $clusterRoleName, [parameter()] # [ValidateScript({ # $isValidIpv4 = Validate-IPV4Address -ip $_ # if (!$isValidIpv4 -and (-not ($_ -cmatch $regexPatternCidrFormat))){ # throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ipv4_address, $_)) # } # return $true # })] [Alias("cloudServiceCidr")] [String] $cloudServiceIP = "", [parameter()] [ProxySettings] $proxySettings = $null, [parameter()] [String] $sshPublicKey, [parameter(DontShow)] [Switch] $skipUpdates, [parameter(DontShow)] [Switch] $skipHostLimitChecks, [parameter(DontShow)] [Switch] $skipRemotingChecks, [parameter(DontShow)] [Switch] $forceDnsReplication, [parameter()] # [ValidateScript({Test-ValidMacPoolAddress -macPoolAddress $_})] [String] $macPoolStart, [parameter()] # [ValidateScript({Test-ValidMacPoolAddress -macPoolAddress $_})] [String] $macPoolEnd, [parameter(DontShow)] [switch] $useStagingShare, [parameter(DontShow)] [ContainerRegistry] $containerRegistry = $null, [parameter(DontShow)] [String] $catalog = $script:catalogName, [parameter(DontShow)] [String] $ring = $script:ringName, [parameter(DontShow)] [String] $deploymentId = [Guid]::NewGuid().ToString(), [parameter(DontShow)] [int] $operatorTokenValidity = $global:operatorTokenValidity, [parameter(DontShow)] [int] $addonTokenValidity = $global:addonTokenValidity, [parameter(DontShow)] [float] $certificateValidityFactor = $global:certificateValidityFactor, [Parameter(Mandatory=$true, ParameterSetName='networkcontroller')] [Switch] $useNetworkController, [Parameter(Mandatory=$true, ParameterSetName='networkcontroller')] [string] $networkControllerFqdnOrIpAddress, [Parameter(Mandatory=$false, ParameterSetName='networkcontroller')] [string] $networkControllerClientCertificateName, [Parameter(Mandatory=$true, ParameterSetName='networkcontroller')] [string] $networkControllerLbSubnetRef, [Parameter(Mandatory=$true, ParameterSetName='networkcontroller')] [string] $networkControllerLnetRef, [parameter(DontShow)] [float] $caCertificateValidityFactor = $global:caCertificateValidityFactor, [parameter(DontShow)] [Switch] $enablePreview, [parameter(DontShow)] [Switch] $skipCleanOnFailure, [parameter(DontShow)] [float] $nodeCertificateValidityFactor = $global:nodeCertificateValidityFactor, [parameter(DontShow)] [int] $caCertRotationThreshold = $global:caCertRotationThreshold, [Parameter(Mandatory=$false)] [Int] $concurrentDownloads = 10, [Parameter(Mandatory=$false)] [VipPoolSettings] $vipPool, [Parameter()] [ValidateSet($true, $false)] [bool] $offlineDownload = $false, [Parameter()] [bool] $offsiteTransferCompleted = $false, [Parameter()] [Switch] $skipValidationCheck, [Parameter()] [OfflineDownloadMode] $mode = "full", [Parameter()] [Switch] $useHTTPSForDownloads ) $startCmdletTime = Get-Date $configCmdletParams = @{version= $version; catalog= $catalog; ring= $ring; moduleName= $moduleName; kvaName= $kvaName; enablePreview= $enablePreview; offlineDownload= $offlineDownload} trap { $tc = Get-DefaultTraceConfigDetails -catalog $catalog -ring $ring -deploymentId $deploymentId Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -ConfigDetails $tc ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams throw $_ } $currentState = Get-InstallState -module $moduleName if ($currentState) { switch ($currentState) { ([InstallState]::NotInstalled) { # Fresh install break } Default { Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_current_state, $currentState)) throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_no_new_config_in_current_state, $moduleName, $currentState)) } } } Confirm-Configuration ` -useStagingShare:$useStagingShare.IsPresent -stagingShare $stagingShare -vnet $vnet -vipPool $vipPool -useNetworkController:$useNetworkController.IsPresent Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName if ($enablePreview.IsPresent) { $catalog = "aks-hci-stable-catalogs-ext" $ring = "earlyaccesspreview" } if ($sshPublicKey) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.akshci_depricated_parameter_please_use, "sshPublicKey", "ssh")) } # Optional diagnostic data is forced off until we meed privacy requirements to allow post-install changes to consent $diagnosticConfigValue = $false Set-AksHciConfigValue -name "enableOptionalDiagnosticData" -value $diagnosticConfigValue # Normalizing directory path $workingDir = Update-DirectoryPath -directoryPath $workingDir $imageDir = Update-DirectoryPath -directoryPath $imageDir $stagingShare = Update-DirectoryPath -directoryPath $stagingShare $cloudConfigLocation = Update-DirectoryPath -directoryPath $cloudConfigLocation $nodeConfigLocation = Update-DirectoryPath -directoryPath $nodeConfigLocation try { $currentMocState = Get-InstallState -module $global:MocModule Set-AksHciConfigValue -name "manifestCache" -value ([io.Path]::Combine($workingDir, $("$catalog.json"))) Set-AksHciConfigValue -name "stagingShare" -value $stagingShare Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Set-AKsHciConfigValue -name "offlineDownload" -value $offlineDownload if ($currentMocState -eq [InstallState]::NotInstalled) { Set-MocConfig -activity $activity -workingDir $workingDir -imageDir $imageDir -isolateImageDir:$isolateImageDir -stagingShare $stagingShare ` -cloudConfigLocation $cloudConfigLocation -nodeConfigLocation $nodeConfigLocation ` -vnet $vnet -cloudLocation $cloudLocation ` -nodeAgentPort $nodeAgentPort -nodeAgentAuthorizerPort $nodeAgentAuthorizerPort ` -cloudAgentPort $cloudAgentPort -cloudAgentAuthorizerPort $cloudAgentAuthorizerPort -version $version ` -clusterRoleName $clusterRoleName -cloudServiceIP $cloudServiceIP -skipUpdates:$skipUpdates.IsPresent ` -skipHostLimitChecks:$skipHostLimitChecks.IsPresent ` -forceDnsReplication:$forceDnsReplication.IsPresent ` -useStagingShare:$useStagingShare.IsPresent -macPoolStart $macPoolStart -macPoolEnd $macPoolEnd ` -sshPublicKey $sshPublicKey -ssh $ssh -skipRemotingChecks:$skipRemotingChecks.IsPresent ` -proxySettings $proxySettings -catalog $catalog -ring $ring -createAutoConfigContainers $createAutoConfigContainers -isolateAutoConfiguredContainerLike:$isolateAutoConfiguredContainerLike ` -deploymentId $deploymentId -certificateValidityFactor $certificateValidityFactor ` -nodeCertificateValidityFactor $nodeCertificateValidityFactor -caCertificateValidityFactor $caCertificateValidityFactor ` -useNetWorkController:$useNetWorkController.IsPresent ` -networkControllerFqdnOrIpAddress $networkControllerFqdnOrIpAddress ` -networkControllerLbSubnetRef $networkControllerLbSubnetRef ` -networkControllerLnetRef $networkControllerLnetRef ` -networkControllerClientCertificateName $networkControllerClientCertificateName ` -vipPool $vipPool -skipValidationCheck:$skipValidationCheck.IsPresent ` -offlineDownload $offlineDownload -offsiteTransferCompleted $offsiteTransferCompleted -skipHostAgentInstall ` -useHTTPSForDownloads:$useHTTPSForDownloads.IsPresent # Install pre-requisites # Incase of offline download, download the bits here. Bits are needed to install moc if ($offlineDownload) { Write-Host $AksHciLocMessage.akshci_wait_for_offline_download if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if (-not $version) { # If no version is specified, use the latest $version = Get-AksHciLatestVersion Set-AksHciConfigValue -name "version" -value $version } } else { Get-AksHciLatestVersion | out-null # This clears the cache Get-ProductRelease -Version $version -module $moduleName | Out-Null Set-AksHciConfigValue -name "version" -value $version } if (-not $offsiteTransferCompleted) { try { Get-AksHciRelease -mode $mode } catch { Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() throw $_ } $offsiteTransferCompleted = $true } } Write-Host $AksHciLocMessage.akshci_wait_for_prerequisites Install-Moc -activity $activity Set-AksHciConfigValue -name "mocInstalledByAksHci" -value $true $mocVersion = Get-MocVersion Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_moc_install_success, $mocVersion)) } elseif ($currentMocState -eq [InstallState]::Installed) { # Try and get the target AksHci version and check if the installed version of Moc is compatible. if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if (-not $version) { # If no version is specified, use the latest $version = Get-AksHciLatestVersion Set-AksHciConfigValue -name "version" -value $version } } # Throw an error if installed version of Moc is not compatible with target AksHci version Test-AksHciSupportedMocVersion -aksHciVersion $version } Set-KvaConfig -activity $activity -workingDir $workingDir -imageDir $imageDir -stagingShare $stagingShare ` -kvaName $kvaName -kvaPodCIDR $kvaPodCIDR -kvaSkipWaitForBootstrap:$kvaSkipWaitForBootstrap.IsPresent ` -controlplaneVmSize $controlplaneVmSize ` -vnet $vnet -cloudLocation $cloudLocation ` -skipUpdates:$skipUpdates.IsPresent ` -useStagingShare:$useStagingShare.IsPresent -version $version -macPoolStart $macPoolStart -macPoolEnd $macPoolEnd ` -proxySettings $proxySettings -containerRegistry:$containerRegistry ` -catalog $catalog -ring $ring ` -cloudAgentPort $cloudAgentPort -cloudAgentAuthorizerPort $cloudAgentAuthorizerPort ` -deploymentId $deploymentId -operatorTokenValidity $operatorTokenValidity -addonTokenValidity $addonTokenValidity ` -concurrentDownloads $concurrentDownloads ` -offlineDownload $offlineDownload -offsiteTransferCompleted $offsiteTransferCompleted -skipValidationCheck:$skipValidationCheck.IsPresent` -enableOptionalDiagnosticData $diagnosticConfigValue -useHTTPSForDownloads:$useHTTPSForDownloads.IsPresent Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_creating_config, $moduleName)) Set-AksHciConfigValue -name "workingDir" -value $workingDir New-Item -ItemType Directory -Force -Path $workingDir | out-null Set-AksHciConfigValue -name "moduleVersion" -value $moduleVersion Set-AksHciConfigValue -name "installState" -value ([InstallState]::NotInstalled) Set-AksHciConfigValue -name "skipUpdates" -value $skipUpdates.IsPresent Set-AksHciConfigValue -name "useStagingShare" -value $useStagingShare.IsPresent Set-AKsHciConfigValue -name "deploymentId" -value $deploymentId Set-AksHciConfigValue -name "skipCleanOnFailure" -value $skipCleanOnFailure.IsPresent Set-AKsHciConfigValue -name "caCertRotationThreshold" -value $caCertRotationThreshold Set-AksHciConfigValue -name "offsiteTransferCompleted" -value $offsiteTransferCompleted Set-AksHciConfigValue -name "useHTTPSForDownloads" -value $useHTTPSForDownloads.IsPresent $latestVersion = Get-AksHciLatestVersion # This clears the cache if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if (-not $version) { # If no version is specified, use the latest $version = $latestVersion Set-AksHciConfigValue -name "version" -value $version } } else { Get-ProductRelease -Version $version -module $moduleName | Out-Null Set-AksHciConfigValue -name "version" -value $version } $commands = Get-ConfigurationValue -Name "commands" -module $moduleName if (-not $commands) { # If no commands are found, initialize with current command $currentCommand = [Command]::new($activity, [System.Diagnostics.Process]::GetCurrentProcess().Id, $(hostname), $(get-date)) $commands = @($currentCommand) Set-AksHciConfigValue -name "commands" -value $commands } Test-ModuleCompatibility -Version $version | Out-Null Set-CachedVersions # Failing the call if Currently Installed PS module Version is less than Minimum supported PS version or we are installing the latest AksHci release and not using latest available PS module if ([Version]$version -eq [Version]$latestVersion) { $MinPSVersion = Get-AksHciConfigValue -name "cachedLatestPSVersion" } else { $productRelease = Get-ProductRelease -version $version -module $moduleName if($productRelease.CustomData.MinSupportedPSVersion) { $MinPSVersion = $productRelease.CustomData.MinSupportedPSVersion } } if (![String]::IsNullOrEmpty($MinPSVersion) -and ([version]$moduleVersion -lt [version]$MinPSVersion)) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_incompatible_PS_module_version, $MinPSVersion))), ([ErrorTypes]::IsUserErrorFlag)) } $installationPackageDir = [io.Path]::Combine($workingDir, $version) Set-AksHciConfigValue -name "installationPackageDir" -value $installationPackageDir New-Item -ItemType Directory -Force -Path $installationPackageDir | Out-Null Save-ConfigurationDirectory -moduleName $moduleName -WorkingDir $workingDir Save-Configuration -moduleName $moduleName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_saved_config) # Pushing the configuration to telemetry #ToDo: Add it back; Once redaction is enabled from telemetrySdk for all config fields <# $configCmdletParams += @{ config = Get-AksHciConfig; }#> Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } catch { Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() $currentMocState = Get-InstallState -module $global:MocModule $mocInstalledByAksHci = Get-AksHciConfigValue -name "mocInstalledByAksHci" if ($mocInstalledByAksHci -And !$skipCleanOnFailure.IsPresent) { Uninstall-Moc -activity $activity -Confirm:$false Set-AksHciConfigValue -name "mocInstalledByAksHci" -value $false } elseif ($currentMocState -eq [InstallState]::NotInstalled) { Reset-Configuration -moduleName $global:MocModule } throw $_ } } function Set-CachedVersions { <# .DESCRIPTION Cache the latest AksHci release versions. .PARAMETER date Date when versions were cached. If not provided, will use current date. #> param ( [String]$date ) if ([String]::IsNullOrEmpty($date)) { $date = Get-date } $latestmodule = Find-Module -Name AksHci $latestPSVersion = $latestmodule.Version.ToString() $LatestAksHciRelease = Get-LatestRelease -moduleName $moduleName $LatestAksHciVersion = $LatestAksHciRelease.Version $minAksHciVersion = $LatestAksHciRelease.CustomData.MinSupportedAkshciVersion Set-AksHciConfigValue -name "cachedLatestPSVersion" -value $latestPSVersion Set-AksHciConfigValue -name "cachedLatestAksHciVersion" -value $LatestAksHciVersion Set-AksHciConfigValue -name "latestVersionsCachedOn" -value $date Set-AksHciConfigValue -name "cachedMinAksHciVersion" -value $minAksHciVersion } function Set-AksHciConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER value Value to be persisted #> param ( [String] $name, [Object] $value ) Set-ConfigurationValue -name $name -value $value -module $moduleName } function Get-AksHciConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value #> param ( [String] $name ) return Get-ConfigurationValue -name $name -module $moduleName } function Get-AksHciConfig { <# .SYNOPSIS List the current configuration settings for the Azure Kubernetes Service host. .DESCRIPTION List the current configuration settings for the Azure Kubernetes Service host. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) throw $_ } Import-AksHciConfig -activity $activity Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_get_config, $moduleName)) $global:config[$modulename]["installState"] = Get-InstallState -module $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $global:config } function Import-AksHciConfig { <# .DESCRIPTION Loads a configuration from persisted storage. If no configuration is present then a default configuration can be optionally generated and persisted. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter()] [Switch] $createIfNotPresent, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $($AksHciLocMessage.akshci_import_config) # Check if configuration exists if (Test-Configuration -moduleName $moduleName) { # 1. Trigger an import of the dependent configurations Get-MocConfig | Out-Null Get-KvaConfig | Out-Null Import-Configuration -moduleName $moduleName } else { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_cannot_deploy, $moduleName))), ([ErrorTypes]::IsUserErrorFlag)) } Write-StatusWithProgress -activity $activity -module $moduleName -status $($AksHciLocMessage.akshci_import_config_complete) } function Install-AksHci { <# .SYNOPSIS Install the Azure Kubernetes Service on Azure Stack HCI agents/services and host. .DESCRIPTION Install the Azure Kubernetes Service on Azure Stack HCI agents/services and host. .PARAMETER timeoutMinutes Timeout in minutes for the installation to complete. Default is 120 minutes. .PARAMETER credential credential is a PSCredential holding a user's Service Principal. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$false)] [int]$timeoutMinutes, [Parameter(Mandatory=$false)] [PSCredential] $credential, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -createConfigIfNotPresent -skipMgmtKubeConfig -skipInstallationCheck -activity $activity Test-KvaAzureConnection Install-AksHciInternal -activity $activity -timeoutMinutes $timeoutMinutes -credential $credential Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Enable-AksHciPreview { <# .SYNOPSIS Enable AKSHCI catalog and ring configuration to expose early access preview builds. .DESCRIPTION Enable AKSHCI catalog and ring configuration to expose early access preview builds. .PARAMETER activity Activity name to use when updating progress .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter(DontShow)] [String] $ring = "earlyaccesspreview" ) $activity = $MyInvocation.MyCommand.Name $startCmdletTime = Get-Date $cmdletParams = @{catalog= $catalog; ring= $ring} trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity #Set MocConfig for early access preview Enable-MocPreview -catalog $catalog -ring $ring #Set KvaConfig for early access preview Enable-KvaPreview -catalog $catalog -ring $ring #Set AksHCiConfig for early access preview Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_enable_preview, $moduleName)) Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_config, $moduleName)) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Write-Warning $($AksHciLocMessage.akshci_preview_warning) Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Disable-AksHciPreview { <# .SYNOPSIS Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .DESCRIPTION Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .PARAMETER activity Activity name to use when updating progress. .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter(DontShow)] [String] $ring = "stable" ) $activity = $MyInvocation.MyCommand.Name $startCmdletTime = Get-Date $cmdletParams = @{catalog= $catalog; ring= $ring} trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity #Set MocConfig Disable-MocPreview -catalog $catalog -ring $ring #Set KvaConfig Disable-KvaPreview -catalog $catalog -ring $ring #Set AksHCiConfig Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_disable, $moduleName)) Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_config_update, $moduleName)) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Reinstall-AksHci { <# .SYNOPSIS Reisntall Azure Kubernetes Service on Azure Stack HCI and remove all deployed Kubernetes clusters. .DESCRIPTION Reinstalling Azure Kubernetes Service on Azure Stack HCI will remove all of your Kubernetes clusters if any, and the Azure Kubernetes Service host. It will also uninstall the Azure Kubernetes Service on Azure Stack HCI agents and services from the nodes. It will then go back through the original install process steps until the host is recreated. The Azure Kubernetes Service on Azure Stack HCI configuration that you configured via Set-AksHciConfig and the downloaded VHDX images are preserved. .PARAMETER timeoutMinutes Timeout in minutes for the installation to complete. Default is 120 minutes. .PARAMETER activity1 Activity name to use when updating progress .PARAMETER credential credential is a PSCredential holding a user's Service Principal. #> [CmdletBinding (PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$false)] [int]$timeoutMinutes, [Parameter(Mandatory=$false)] [PSCredential] $credential ) Write-Warning $($AksHciLocMessage.akshci_reinstall_userprompt_warning) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if(-not $PSCmdlet.ShouldProcess("Azure Stack HCI deployment", "Reinstall-AksHci" )) { Write-Warning $($AksHciLocMessage.akshci_reinstall_abort_warning) return } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck # If AksHci was not responsible for installing Moc, it will not uninstall it. Uninstall-AksHci -SkipConfigCleanup -activity $activity $currentMocState = Get-InstallState -module $global:MocModule if ($currentMocState -eq [InstallState]::NotInstalled) { Install-Moc -activity $activity Set-AksHciConfigValue -name "mocInstalledByAksHci" -value $true } Install-AksHciInternal -activity $activity -timeoutMinutes $timeoutMinutes -credential $credential Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Uninstall-AksHci { <# .SYNOPSIS Removes Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Removes Azure Kubernetes Service on Azure Stack HCI. If PowerShell commands are run on a cluster where Windows Admin Center was previously used to deploy, the PowerShell module checks the existence of the Windows Admin Center configuration file. Windows Admin Center places the Windows Admin Center configuration file across all nodes. .PARAMETER SkipConfigCleanup Skips removal of the configurations after uninstall. After Uninstall, you have to Set-AksHciConfig to install again. .PARAMETER SkipMocCleanup Skips cleaning up Moc resources and entities .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding (PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter()] [Switch] $SkipConfigCleanup, [Parameter(DontShow)] [Switch] $SkipMocCleanup, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) Write-Warning $($AksHciLocMessage.akshci_uninstall_userprompt_warning) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity throw $_ } if(-not $PSCmdlet.ShouldProcess("Azure Stack HCI deployment", "Uninstall-AksHci" )) { Write-Warning $($AksHciLocMessage.akshci_uninstall_abort_warning) return } $isAksHciInstalling = $false try { Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity $installAksHciCommand = "Install-AksHci" $isAksHciInstalling = Test-Command-Runnable -currentCommand $activity ` -cannotRunWithInstallState Installing ` -cannotRunWithCommand $installAksHciCommand if ($isAksHciInstalling) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_install_in_progress, $moduleName)), ([ErrorTypes]::IsUserErrorFlag)) } $configDetails = Get-TraceConfigDetails -moduleName $moduleName $aksHciRegistration = Get-AksHciRegistration if (-not [string]::IsNullOrWhiteSpace($aksHciRegistration.azureResourceGroup)) { try { Test-KvaAzureConnection } catch [Exception] { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_azure_connection_warning) } } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Uninstalling) try { $clusters = Get-AksHciCluster foreach($cluster in $clusters) { try { Remove-AksHciCluster -Name $cluster.Name -Confirm:$false } catch [Exception] { Write-Status -moduleName $moduleName -msg $($GenericLocMessage.generic_exception) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() Write-SubStatus -moduleName $moduleName -msg $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_cannot_delete_target_cluster, $moduleName)) } } } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } } catch [Exception] { if ($isAksHciInstalling) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_install_in_progress, $moduleName)), ([ErrorTypes]::IsUserErrorFlag)) } # If AksHci is not installed, you would reach here Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } try { Uninstall-Kva -SkipConfigCleanup:$SkipConfigCleanup.IsPresent -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } #ConfigMap details are not available after kva is uninstalled try { $mocInstalledByAksHci = Get-AksHciConfigValue -name "mocInstalledByAksHci" if (!$SkipMocCleanup.IsPresent -and $mocInstalledByAksHci) { $confirmValue = $true if ($PSBoundParameters.ContainsKey('Confirm')) { $confirmValue = $PSBoundParameters['Confirm'] } Uninstall-Moc -SkipConfigCleanup:$SkipConfigCleanup.IsPresent -activity $activity -Confirm:$confirmValue Set-AksHciConfigValue -name "mocInstalledByAksHci" -value $false } } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } Trace-Cmdlet -ConfigDetails $configDetails -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys Set-AksHciConfigValue -name "installState" -value ([InstallState]::NotInstalled) Uninitialize-AksHciEnvironment -activity $activity if (!$SkipConfigCleanup.IsPresent) { Reset-Configuration -moduleName $moduleName } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed } function Get-AksHciKubernetesVersion { <# .SYNOPSIS List the available versions for creating a managed Kubernetes cluster. .DESCRIPTION List the available versions for creating a managed Kubernetes cluster. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_kube_versions) Get-AvailableKubernetesVersions -moduleName $moduleName Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciRequiredKubernetesVersion { <# .DESCRIPTION Returns the kva required kubernetes versions that are supported by the specified AksHci release .PARAMETER akshciVersion AksHci Release version. Defaults to the version of the current deployment .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [String] $akshciVersion, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ([string]::IsNullOrEmpty($akshciVersion)) { $akshciVersion = Get-AksHciVersion } # Get the Manifest for the specified Version $productRelease = Get-ProductRelease -version $akshciVersion -module $moduleName $k8sVersion = $productRelease.CustomData.ManagementNodeImageK8sVersion return $("v"+$k8sVersion) } function Get-AksHciVmSize { <# .SYNOPSIS Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_vm_size) $result = @() foreach($definition in $global:vmSizeDefinitions) { $size = [ordered]@{'VmSize' = $definition[0]; 'CPU' = $definition[1]; 'MemoryGB' = $definition[2]; 'GPU Type' = $definition[3]; 'GPU Count' = $definition[4]} $result += New-Object -TypeName PsObject -Property $size } Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -BoundParameterKeys $PSBoundParameters.Keys return $result } function Set-AksHciCluster { <# .SYNOPSIS Scale the number of control plane nodes or worker nodes in a cluster. Also enable/disable AutoScaler for the cluster. .DESCRIPTION Scale the number of control plane nodes or worker nodes in a cluster. The control plane nodes and the worker nodes must be scaled independently. Also enable or disable AutoScaler for the cluster. .PARAMETER Name Name of the cluster .PARAMETER controlPlaneNodeCount The number of control plane nodes to scale to .PARAMETER controlPlaneVMSize The VM size for the control plane nodes .PARAMETER linuxNodeCount The number of Linux worker nodes to scale to .PARAMETER windowsNodeCount The number of Windows worker nodes to scale to .PARAMETER EnableAutoScaler Enable or disable the cluster autoscaler. .PARAMETER AutoScalerProfileName The name of the AutoScalerProfile used to configure the cluster autoscaler. To change the profile on a cluster that has autoscaler enabled, set this parameter but do not set the EnableAutoScaler parameter. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter(ParameterSetName='controlplane')] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount, [Parameter(ParameterSetName='controlplane')] [VmSize] $controlPlaneVMSize, [Parameter(Mandatory=$true, ParameterSetName='worker')] [ValidateRange(0,250)] [int] $linuxNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [ValidateRange(0,250)] [int] $windowsNodeCount, [Parameter(Mandatory=$true, ParameterSetName='toggleautoscaler')] [ValidateSet($true, $false)] [Boolean] $EnableAutoScaler, [Parameter(ParameterSetName='toggleautoscaler')] [Parameter(Mandatory=$true, ParameterSetName='changeautoscalerprofile')] [String] $AutoScalerProfileName, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; controlPlaneNodeCount= $controlPlaneNodeCount; linuxNodeCount= $linuxNodeCount; windowsNodeCount=$windowsNodeCount; EnableAutoScaler=$EnableAutoScaler; AutoScalerProfileName=$AutoScalerProfileName } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } # Since the scale parameter set has no mandatory parameters, make sure that at least one was provided # If not, throw a generic error message if ($PSCmdlet.ParameterSetName -eq "controlplane" -and -not ($PSBoundParameters.ContainsKey("controlplanevmsize") -or $PSBoundParameters.ContainsKey("controlplanenodecount"))) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_set_cluster_missing_parameters, $Name))), ([ErrorTypes]::IsUserErrorFlag)) } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity $mgmtCluster = (Get-KvaConfig)["kvaName"] if ($Name -ieq $mgmtCluster) { throw [CustomException]::new(($($AksHciLocMessage.akshci_scaling_unsupported)), ([ErrorTypes]::IsUserErrorFlag)) } switch ($PSCmdlet.ParameterSetName) { ("controlplane") { # Based on the above check, at least one of the parameters should be present if ($PSBoundParameters.ContainsKey("controlplanevmsize") -and $PSBoundParameters.ContainsKey("controlplanenodecount")) { Set-KvaClusterNodeCount -Name $Name -controlPlaneNodeCount $controlPlaneNodeCount -controlPlaneVMSize $controlPlaneVMSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("controlplanevmsize")) { Set-KvaClusterNodeCount -Name $Name -controlPlaneVMSize $controlPlaneVMSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("controlplanenodecount")) { Set-KvaClusterNodeCount -Name $Name -controlPlaneNodeCount $controlPlaneNodeCount -activity $activity } break } ("worker") { Write-Output @" WARNING: Set-AksHciCluster can now only be used to scale either the control plane node count or the worker count of the default nodepools that were created as a part of the older cluster creation workflow. Please consider using Set-AksHciNodePool to manage your node pool worker count as it can be used for any nodepool. Examples: - Get a list of cluster1's nodepools: `tGet-AksHciNodePool -ClusterName "cluster1" - Scale "nodepool1" to 2 worker nodes: `tSet-AksHciNodePool -ClusterName "cluster1" -Name "nodepool1" -Count 2 "@ if ($windowsNodeCount -gt 0) { $cluster = Get-KvaCluster -Name $Name -activity $activity Test-SupportedKubernetesVersion -imageType Windows -k8sVersion $cluster.KubernetesVersion } Set-KvaClusterNodeCount -Name $Name -linuxNodeCount $linuxNodeCount -windowsNodeCount $windowsNodeCount -activity $activity break } ("toggleautoscaler") { if (![string]::IsNullOrEmpty($AutoScalerProfileName)) { Set-KvaClusterAutoScaler -Name $Name -Enable $EnableAutoScaler -ProfileName $AutoScalerProfileName -activity $activity } else { Set-KvaClusterAutoScaler -Name $Name -Enable $EnableAutoScaler -activity $activity } break } ("changeautoscalerprofile") { Set-KvaClusterAutoScaler -Name $Name -ProfileName $AutoScalerProfileName -activity $activity break } } Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function New-AksHciAutoScalerProfile { <# .SYNOPSIS Creates a new AutoScalerProfile. .DESCRIPTION Creates a new AutoScalerProfile. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AutoScalerProfileConfig Hashtable containing AutoScalerProfile config keys and their values .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .INPUTS An AutoScalerProfile config is comprised of the following keys. Note: keys that are not provided will fall back to its default value - min-node-count - max-node-count - max-nodes-total - scale-down-enabled - scan-interval - scale-down-delay-after-add - scale-down-delay-after-delete - scale-down-delay-after-failure - scale-down-unneeded-time - scale-down-unready-time - scale-down-utilization-threshold - max-graceful-termination-sec - balance-similar-node-groups - expander - skip-nodes-with-local-storage - skip-nodes-with-system-pods - max-empty-bulk-delete - new-pod-scale-up-delay - max-total-unready-percentage - max-node-provision-time - ok-total-unready-count #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidAutoScalerProfileConfig -AutoScalerProfileConfig $_ })] [hashtable] $AutoScalerProfileConfig, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; AutoScalerProfileConfig=$AutoScalerProfileConfig } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_creating_autoscalerprofile, $Name)) New-KvaAutoScalerProfile -Name $Name -AutoScalerProfileConfig $AutoScalerProfileConfig -activity $activity Get-AksHciAutoScalerProfile -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciAutoScalerProfile { <# .SYNOPSIS Retrieve AutoScalerProfiles and their settings. .DESCRIPTION Retrieve AutoScalerProfiles and their settings. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_autoscalerprofile_info) Get-KvaAutoScalerProfile -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Set-AksHciAutoScalerProfile { <# .SYNOPSIS Updates an existing AutoScalerProfile. .DESCRIPTION Updates an existing AutoScalerProfile. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AutoScalerProfileConfig Hashtable containing AutoScalerProfile config keys and their values .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .INPUTS An AutoScalerProfile config is comprised of the following keys. Note: keys' values in the profile will not change if not specified in the argument to this cmdlet. - min-node-count - max-node-count - max-nodes-total - scale-down-enabled - scan-interval - scale-down-delay-after-add - scale-down-delay-after-delete - scale-down-delay-after-failure - scale-down-unneeded-time - scale-down-unready-time - scale-down-utilization-threshold - max-graceful-termination-sec - balance-similar-node-groups - expander - skip-nodes-with-local-storage - skip-nodes-with-system-pods - max-empty-bulk-delete - new-pod-scale-up-delay - max-total-unready-percentage - max-node-provision-time - ok-total-unready-count #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidAutoScalerProfileConfig -AutoScalerProfileConfig $_ })] [hashtable] $AutoScalerProfileConfig, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; AutoScalerProfileConfig=$AutoScalerProfileConfig } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_updating_autoscalerprofile, $Name)) Set-KvaAutoScalerProfile -Name $Name -AutoScalerProfileConfig $AutoScalerProfileConfig -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Remove-AksHciAutoScalerProfile { <# .SYNOPSIS Removes the AutoScalerProfile. .DESCRIPTION Removes the AutoScalerProfile. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_deleting_autoscalerprofile, $Name)) Remove-KvaAutoScalerProfile -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function New-AksHciCluster { <# .SYNOPSIS Create a new managed Kubernetes cluster. .DESCRIPTION Create a new Azure Kubernetes Service on Azure Stack HCI cluster. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneNodeCount The number of control plane (master) nodes .PARAMETER linuxNodeCount The number of Linux worker nodes .PARAMETER windowsNodeCount The number of Windows worker nodes .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER nodePoolName The name of the node pool .PARAMETER nodeCount The number of worker nodes in the node pool .PARAMETER nodeMaxPodCount The maximum number of pods that can run on a worker node .PARAMETER taints A list of taints to put on each worker node .PARAMETER nodeVmSize The VM size to use for the worker nodes in the node pool .PARAMETER osType The OS type for the worker nodes in the node pool .PARAMETER enableAutoScaler Enable the horizontal Nodepool AutoScaler for this cluster .PARAMETER autoScalerProfileName The name of the AutoScalerProfile to use .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER enableMonitoring Enable deploying the monitoring once cluster creation is complete. .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .PARAMETER primaryNetworkPlugin Primary network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". .PARAMETER secondaryNetworkPlugins Secondary network plugin (CNI) definitions created by calls to New-AksHciMocCNISetting. .PARAMETER clusterStorageContainer Storage container that is associated to the Cluster. .PARAMETER loadBalancerSettings LoadBalancer object specifying the type and other params of the loadbalancer .PARAMETER enableAzureRBAC Whether to enable Azure RBAC during creation of AKS HCI workload cluster .PARAMETER appId server app id for azure AD RBAC .PARAMETER appSecret server app secret for azure AD RBAC .PARAMETER aadClientId client app id for azure AD kubeconfig .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER vipPoolName Name of VIP pool to use for the cluster. .PARAMETER customLocationsOid object id of custom locations app. .PARAMETER enableAvailabilitySet flag to enable availability set. Disabled by default. #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName = 'twonodepools')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [String] $kubernetesVersion = (Get-AksHciRequiredKubernetesVersion), [Parameter()] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [ValidateRange(0,250)] [int] $linuxNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [ValidateRange(0,250)] [int] $windowsNodeCount = 0, [Parameter()] [String] $controlplaneVmSize = $global:defaultControlPlaneVmSize, [Parameter()] [String] $loadBalancerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $linuxNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $windowsNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [Parameter(ParameterSetName = 'onenodepool')] # [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $nodePoolName = $global:defaultNodePoolName, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [Parameter(ParameterSetName = 'onenodepool')] [ValidateRange(0,250)] [int] $nodeCount = $global:defaultWorkerNodeCount, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [Parameter(ParameterSetName = 'onenodepool')] [ValidateRange(0,250)] [int] $nodeMaxPodCount = 0, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [Parameter(ParameterSetName = 'onenodepool')] # [ValidateScript({Test-ValidTaints -taints $_})] [String[]] $taints, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [Parameter(ParameterSetName = 'onenodepool')] [VmSize] $nodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [Parameter(ParameterSetName = 'onenodepool')] [OsType] $osType = $global:defaultWorkerNodeOS, [Parameter(ParameterSetName = 'onenodepool')] [OsSku] $osSku = $(If ($OSType -eq $global:defaultWorkerNodeOS) {[OsSku]::CBLMariner} Else {[OsSku]::Windows2019}), [Parameter()] [Switch] $enableAutoScaler, [Parameter()] # To-Do: confirm auto Scaler profile name restrictions [String] $autoScalerProfileName, [Parameter()] [Switch]$enableADAuth, [Parameter()] [Switch]$enableMonitoring, [Parameter()] [VirtualNetwork]$vnet, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity, [Parameter()] [ValidateScript({return $true})] #Note: ValidateScript automatically constructs the NetworkPlugin object, therefore validates the parameter [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter(Mandatory=$false)] [SecondaryNetworkPlugin[]] $secondaryNetworkPlugins, [Parameter()] [String] $clusterStorageContainer = $global:cloudStorageContainer, [Parameter(Mandatory=$false)] [LoadBalancerSettings] $loadBalancerSettings, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAAD')] [Switch] $enableAzureRBAC, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAAD')] [String] $appId, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAAD')] [String] $appSecret, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAAD')] [String] $aadClientId, [Parameter(ParameterSetName = 'onenodepoolAADAzure')] [Parameter(ParameterSetName = 'onenodepoolAAD')] [String] $customLocationsOid, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName = 'onenodepoolAADAzure')] [String] $location, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidNetworkName -name $_})] [String] $vipPoolName, [Parameter(Mandatory=$false)] [Switch] $enableAvailabilitySet ) $startCmdletTime = Get-Date $cmdletParams = @{nodeVmSize= $nodeVmSize; secondaryNetworkPluginName= $secondaryNetworkPlugins.Name; secondaryNetworkPluginCniType= $secondaryNetworkPlugins.CniType; secondaryNetworkPluginEnableDpdk= $secondaryNetworkPlugins.EnableDpdk; secondaryNetworkPluginEnableSriov= $secondaryNetworkPlugins.EnableSriov; subscriptionId= $subscriptionId; tenantId= $tenantId; resourceGroup= $resourceGroup; Name= $Name; kubernetesVersion= $kubernetesVersion; controlPlaneNodeCount= $controlPlaneNodeCount; linuxNodeCount= $linuxNodeCount; windowsNodeCount= $windowsNodeCount; controlplaneVmSize= $controlplaneVmSize; loadBalancerVmSize= $loadBalancerVmSize; linuxNodeVmSize= $linuxNodeVmSize; windowsNodeVmSize= $windowsNodeVmSize; nodePoolName= $nodePoolName; nodeCount= $nodeCount; nodeMaxPodCount= $nodeMaxPodCount; taints= $taints; osType= $osType; osSku= $osSku; enableAutoScaler= $enableAutoScaler; autoScalerProfileName= $autoScalerProfileName; enableADAuth= $enableADAuth; enableMonitoring= $enableMonitoring; clusterStorageContainer= $clusterStorageContainer; LoadBalancerSettings= $LoadBalancerSettings; enableAzureRBAC= $enableAzureRBAC; location= $location; enableAvailabilitySet = $enableAvailabilitySet; } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } if (($linuxNodeVmSize -ne "") -and (Test-PrivatePreviewSku $linuxNodeVmSize)) { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_feature_sku, $linuxNodeVmSize)) } if (($nodeVmSize -ne "") -and (Test-PrivatePreviewSku $nodeVmSize)) { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_feature_sku, $nodeVmSize)) } if ($secondaryNetworkPlugins.IsPresent) { Write-Host $($AksHciLocMessage.akshci_preview_feature_secondary_network_plugins) } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Confirm-ClusterVipPoolConfiguration -vipPoolName $vipPoolName if ($enableAzureRBAC.IsPresent) { #Both ad auth and azure RBAC cannot be enabled together if($enableADAuth.IsPresent) { throw $($AksHciLocMessage.akshci_adauth_aad_conflict) } # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } } if ($loadBalancerVmSize -and $loadBalancerSettings) { throw [CustomException]::new($($AksHciLocMessage.akshci_loadbalancer_vmsize_multiple_parameters), ([ErrorTypes]::IsUserErrorFlag)) } if (-not $loadBalancerSettings) { # Backward compatibility if ((Get-MocConfig)["useNetworkController"] -eq $true) { $loadBalancerSettings = New-AksHciLoadBalancerSetting -name "sdnLB" -LoadBalancerSku SDNLoadBalancer } else { if (-not $loadBalancerVmSize) { $loadBalancerVmSize = $global:defaultLoadBalancerVmSize } $loadBalancerSettings = New-AksHciLoadBalancerSetting -name "haProxyLB" -LoadBalancerSku HAProxy -vmSize $loadBalancerVmSize -loadBalancerCount 1 } } Initialize-AksHciEnvironment -activity $activity $updateAksHciCommand = "Update-AksHci" $updateAksHciCommandRegex = "^$updateAksHciCommand$" $isAksHciUpdateRunning = Test-Command-Runnable -currentCommand $activity ` -cannotRunWithInstallState Updating ` -cannotRunWithCommand $updateAksHciCommand ` -cannotRunWithCommandRegex $updateAksHciCommandRegex if ($isAksHciUpdateRunning) { Uninitialize-AksHciEnvironment -activity $activity throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_update_in_progress, $moduleName)), ([ErrorTypes]::IsUserErrorFlag)) } if ($PSCmdlet.ParameterSetName -ieq "twonodepools") { Write-Output @" WARNING: In a future release, New-AksHciCluster will change how node pools are created. Currently, a user can only set a node count for two default node pools using the parameters linuxNodeCount and windowsNodeCount. This behavior will change giving users more control over the default node pool created after a cluster is deployed. Please consider using the new parameters to manage node pools below. Parameters to be deprecated: `t-linuxNodeCount `t-linuxNodeVmSize `t-windowsNodeCount `t-windowsNodeVmSize New parameters: `t-nodePoolName `t-nodeCount `t-nodeVmSize `t-osType Examples: - Create a cluster that has a node pool with 1 Windows worker node: `tNew-AksHciCluster -Name "cluster1" -osType Windows - Create a cluster that has a node pool with 2 Linux worker nodes and a custom name: `tNew-AksHciCluster -Name "cluster1" -nodePoolName "example-nodepool" -nodeCount 2 "@ Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_linux_kube_version) Test-SupportedKubernetesVersion -imageType Linux -k8sVersion $kubernetesVersion if ($windowsNodeCount -gt 0) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_windows_kube_version) Test-SupportedKubernetesVersion -imageType Windows -k8sVersion $kubernetesVersion } New-KvaCluster ` -Name $Name -kubernetesVersion $kubernetesVersion ` -controlPlaneNodeCount $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -linuxNodeCount $linuxNodeCount -linuxNodeVmSize $linuxNodeVmSize ` -windowsNodeCount $windowsNodeCount -windowsNodeVmSize $windowsNodeVmSize ` -enableAutoScaler:$enableAutoScaler.IsPresent -autoScalerProfileName $autoScalerProfileName ` -enableADAuth:$enableADAuth.IsPresent ` -enableAzureRBAC:$enableAzureRBAC.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin.Name ` -secondaryNetworkPlugins $secondaryNetworkPlugins ` -vnet $vnet ` -activity $activity -loadBalancerSettings $loadBalancerSettings ` -enableAvailabilitySet:$enableAvailabilitySet.IsPresent } # onenodepool, onenodepoolAAD or onenodepoolAADAzure elseif ($PSCmdlet.ParameterSetName -like "onenodepool*") { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_other_os_kube_version, $osType)) Test-SupportedKubernetesVersion -imageType $osType -k8sVersion $kubernetesVersion -osSku $osSku New-KvaCluster ` -Name $Name -kubernetesVersion $kubernetesVersion ` -controlPlaneNodeCount $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -nodePoolName $nodePoolName -nodeCount $nodeCount -nodeMaxPodCount $nodeMaxPodCount -taints $taints ` -nodeVmSize $nodeVmSize -osType $osType -osSku $osSku ` -enableAutoScaler:$enableAutoScaler.IsPresent -autoScalerProfileName $autoScalerProfileName ` -enableADAuth:$enableADAuth.IsPresent ` -enableAzureRBAC:$enableAzureRBAC.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin.Name ` -secondaryNetworkPlugins $secondaryNetworkPlugins ` -vnet $vnet ` -activity $activity -loadBalancerSettings $loadBalancerSettings ` -enableAvailabilitySet:$enableAvailabilitySet.IsPresent } Get-AksHciCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName ## If enableMonitoring is enabled then install the monitoring with default values. if ($enableMonitoring.IsPresent) { Install-AksHciMonitoring -Name $Name -storageSizeGB 100 -retentionTimeHours 240 } if ($enableAzureRBAC.IsPresent) { if ($PSCmdlet.ParameterSetName -ieq "onenodepoolAADAzure") { New-KvaArcConnection -Name $Name -enableAzureRBAC:$enableAzureRBAC.IsPresent -appId $appId -appSecret $appSecret -aadClientId $aadClientId -customLocationsOid $customLocationsOid -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -activity $activity } else { New-KvaArcConnection -Name $Name -enableAzureRBAC:$enableAzureRBAC.IsPresent -appId $appId -appSecret $appSecret -aadClientId $aadClientId -customLocationsOid $customLocationsOid -activity $activity } } Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciCluster { <# .SYNOPSIS List Kubernetes managed clusters including the Azure Kubernetes Service host. .DESCRIPTION List Kubernetes managed clusters including the Azure Kubernetes Service host. .PARAMETER Name Name of the cluster .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] # [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_cluster_info) Get-KvaCluster -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Remove-AksHciCluster { <# .SYNOPSIS Delete a managed Kubernetes cluster. .DESCRIPTION Delete a managed Kubernetes cluster. .PARAMETER Name Name of the cluster .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($Name, "Delete the managed Kubernetes cluster")) { if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity Remove-KvaCluster -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } } function Get-AksHciClusterUpdates { <# .SYNOPSIS Get the available Kubernetes upgrades for an Azure Kubernetes Service cluster. .DESCRIPTION Get the available Kubernetes upgrades for an Azure Kubernetes Service cluster. .PARAMETER Name Name of the cluster. .PARAMETER activity Activity name to use when updating progress. #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $upgrades = Get-KvaClusterUpgrades -Name $Name -activity $activity $upgrades.AvailableUpgrades Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Update-AksHciCluster { <# .SYNOPSIS Update a managed Kubernetes cluster to a newer Kubernetes or OS version. .DESCRIPTION Update a managed Kubernetes cluster to a newer Kubernetes or OS version. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to upgrade to .PARAMETER operatingSystem Perform an operating system upgrade instead of a kubernetes version upgrade .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'Low')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] # [ValidateScript({Test-ValidVersionNumber -VersionNumber $_})] [String] $kubernetesVersion, [Parameter()] [Switch] $operatingSystem, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; kubernetesVersion= $kubernetesVersion; operatingSystem= $operatingSystem } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity $updateAksHciCommand = "Update-AksHci" $updateAksHciCommandRegex = "^$updateAksHciCommand$" $isAksHciUpdateRunning = Test-Command-Runnable -currentCommand $activity ` -cannotRunWithInstallState Updating ` -cannotRunWithCommand $updateAksHciCommand ` -cannotRunWithCommandRegex $updateAksHciCommandRegex if ($isAksHciUpdateRunning) { Uninitialize-AksHciEnvironment -activity $activity throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_update_in_progress, $moduleName)), ([ErrorTypes]::IsUserErrorFlag)) } $cluster = Get-KvaCluster -Name $Name -activity $activity if ($operatingSystem.IsPresent -and $kubernetesVersion -ne "") { # operating system is updated when kubernetes version is upgraded. # if user specifies both, just turn the switch off, because we will internally # update the OS. $operatingSystem = $false } $nextVersion = $null if (-not $operatingSystem.IsPresent) { if ([string]::IsNullOrEmpty($kubernetesVersion)) { # no version was requested. just try to make the highest jump. $nextVersion = Get-NextKubernetesVersionForUpgrade -Name $Name -activity $activity } else { $nextVersion = Get-CleanInputKubernetesVersion -KubernetesVersion $kubernetesVersion } if ([string]::IsNullOrEmpty($nextVersion)) { $nextVersion = Get-CleanInputKubernetesVersion -KubernetesVersion $cluster.KubernetesVersion } Test-SupportedKubernetesVersion -imageType "Linux" -k8sVersion $nextVersion } if ($PSCmdlet.ShouldProcess($Name, "Update the managed Kubernetes cluster")) { $confirmValue = $true if ($PSBoundParameters.ContainsKey('Confirm')) { $confirmValue = $PSBoundParameters['Confirm'] } Update-KvaCluster -Name $Name -activity $activity -operatingSystem:$operatingSystem.IsPresent -nextVersion $nextVersion -Confirm:$confirmValue } Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function New-AksHciNodePool { <# .SYNOPSIS Create a new nodepool under a cluster. .DESCRIPTION Creates a new nodepool under a cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count The number of worker nodes in the nodepool .PARAMETER OSType OS type of the node pool. Defaults to Linux .PARAMETER VMSize The VM size to use for the worker nodes. Defaults to Standard_K8S3_v1 .PARAMETER MaxPodCount The maximum number of pods that can run on a worker node .PARAMETER Taints A list of taints to put on each worker node .PARAMETER DisableAutoScaler Prevent the horizontal nodepool AutoScaler from managing this nodepool. If AutoScaler is enabled on the cluster, all new nodepools will be managed by it. This flag can be used to override that for the new nodepool. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER LinuxOsConfig Linux O/S configuration object created by a call to New-LinuxOsConfig .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [ValidateRange(0,250)] [int] $Count = $global:defaultWorkerNodeCount, [Parameter()] [VmSize] $VMSize = $global:defaultWorkerVmSize, [Parameter()] [OsType] $OSType = $global:defaultWorkerNodeOS, [Parameter()] [OsSku] $OsSku = $(If ($OSType -eq $global:defaultWorkerNodeOS) {[OsSku]::CBLMariner} Else {[OsSku]::Windows2019}), [Parameter()] [ValidateRange(0,250)] [int] $MaxPodCount = 0, [Parameter()] # [ValidateScript({Test-ValidTaints -taints $_})] [String[]] $Taints, [Parameter()] [Switch] $DisableAutoScaler, [Parameter()] [Switch] $AsJob, [Parameter(Mandatory=$false)] [LinuxOsConfig] $LinuxOsConfig, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName; Name=$Name; Count=$Count; VMSize=$VMSize; OSType=$OSType; OSSku=$OsSku; MaxPodCount=$MaxPodCount; Taints=$Taints; LinuxOsConfig=$LinuxOsConfig } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } if (Test-PrivatePreviewSku $VMSize) { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_feature_sku, $VMSize)) } if ($LinuxOsConfig) { Write-Host $($AksHciLocMessage.akshci_preview_feature_linux_os_config) } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_create_node_pool) -moduleName $moduleName $cluster = Get-KvaCluster -Name $ClusterName -activity $activity Test-SupportedKubernetesVersion -imageType $OSType -k8sVersion $cluster.KubernetesVersion -osSku $OsSku New-KvaClusterNodePool ` -ClusterName $ClusterName -Name $Name ` -MaxPodCount $MaxPodCount -Taints $Taints -Count $Count ` -VMSize $VMSize -OSType $OSType -OsSku $OsSku ` -DisableAutoScaler:$DisableAutoScaler.IsPresent ` -LinuxOsConfig $LinuxOsConfig ` -activity $activity Get-AksHciNodePool -ClusterName $ClusterName -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciNodePool { <# .SYNOPSIS List a Kubernetes managed cluster's nodepools. .DESCRIPTION List a Kubernetes managed cluster's nodepools. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter()] # [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName; Name=$Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_cluster_node_pool_info) Get-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Set-AksHciNodePool { <# .SYNOPSIS Scale a Kubernetes managed cluster's nodepool. .DESCRIPTION Scale a Kubernetes managed cluster's nodepool. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count Node count to scale to .PARAMETER VMSize VM size to for the nodepool nodes .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER AutoScaler If set to true, allow the cluster AutoScaler to horizontally scale this nodepool if it is enabled on the cluster. Else, if set to false, autoscaling will be disabled on this nodepool. .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter(ParameterSetName='scale')] [ValidateRange(0,250)] [int] $Count, [Parameter(ParameterSetName='scale')] [VmSize] $VMSize, [Parameter(Mandatory=$true, ParameterSetName='autoscaler')] [ValidateSet($true, $false)] [Boolean] $AutoScaler, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName; Name=$Name; Count=$Count; VMSize=$VMSize; AutoScaler=$AutoScaler } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } # Since the scale parameter set has no mandatory parameters, make sure that at least one was provided # If not, throw a generic error message if ($PSCmdlet.ParameterSetName -eq "scale" -and -not ($PSBoundParameters.ContainsKey("vmsize") -or $PSBoundParameters.ContainsKey("count"))) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_set_nodepool_missing_parameters, $Name, $ClusterName))), ([ErrorTypes]::IsUserErrorFlag)) } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity switch ($PSCmdlet.ParameterSetName) { ("scale") { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_scaling_node_pool) # Based on the above check, at least one of the parameters should be present if ($PSBoundParameters.ContainsKey("vmsize") -and $PSBoundParameters.ContainsKey("count")) { Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Count $Count -VmSize $VmSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("vmsize")) { Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -VmSize $VmSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("count")) { Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Count $Count -activity $activity } break } ("autoscaler") { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_updating_node_pool_autoscaler) Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -AutoScaler $AutoScaler -activity $activity break } } Get-AksHciNodePool -ClusterName $ClusterName -Name $Name -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Remove-AksHciNodePool { <# .SYNOPSIS Delete a nodepool in a managed Kubernetes cluster. .DESCRIPTION Delete a nodepool in a managed Kubernetes cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName; Name=$Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($name, "Delete the node pool in the managed Kubernetes cluster")) { if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_delete_node_pool) Remove-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Confirm:$false -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciLogs { <# .SYNOPSIS Create a zipped folder with logs from all your pods. .DESCRIPTION Create a zipped folder with logs from all your pods. This command will create an output zipped folder called akshcilogs.zip in your AKS on Azure Stack HCI working directory. The full path to the akshcilogs.zip file will be the output after running Get-AksHciLogs (for example, C:\AksHci\0.9.6.3\akshcilogs.zip, where 0.9.6.3 is the AKS on Azure Stack HCI release number). .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .PARAMETER zipName Zip path and name to use for storing logs .PARAMETER VirtualMachineLogs Switch to get only the logs from the vm's (LB vm if unstacked deployment and management-cluster vm) .PARAMETER AgentLogs Switch to get only logs of the wssdagent and wssdcloudagent on all nodes .PARAMETER EventLogs Switch to get only Windows Event Logson all nodes .PARAMETER Detail Switch to get more detailed logs instead of minimum required logs, currently only effective on Get-MocLogs Feel free to utlize this switch for KvaLogs, DownloadsdkLogs and BillingRecords .PARAMETER KvaLogs Switch to get only the logs from KVA .PARAMETER DownloadSdkLogs Switch to get only the logs from DownloadSdk .PARAMETER BillingRecords Switch to get only the billing records #> param ( [Parameter()] [Switch]$AsJob, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$false)] [String]$zipName, [Parameter(Mandatory=$false)] [Switch]$VirtualMachineLogs, [Parameter(Mandatory=$false)] [Switch]$AgentLogs, [Parameter(Mandatory=$false)] [Switch]$EventLogs, [Parameter(Mandatory=$false)] [Switch]$Detail, [Parameter(Mandatory=$false)] [Switch]$KvaLogs, [Parameter(Mandatory=$false)] [Switch]$DownloadSdkLogs, [Parameter(Mandatory=$false)] [Switch]$BillingRecords ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } $allswitch = $true # Note: $Detail only affects the collection of detailed moc logs in the Get-MocLogs function. # Therefore, if the KvaLogs, DownloadSdkLogs, or BillingRecords functions also use this variable, it should be removed from the condition." if ($VirtualMachineLogs.IsPresent -or $AgentLogs.IsPresent -or $EventLogs.IsPresent -or $Detail.IsPresent -or $KvaLogs.IsPresent -or $DownloadSdkLogs.IsPresent -or $BillingRecords.IsPresent) { $allswitch = $false } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck $logName = $("akshcilogs" + [io.Path]::GetRandomFileName()) $logDir = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], $logName) if ($VirtualMachineLogs.IsPresent -or $AgentLogs.IsPresent -or $EventLogs.IsPresent -or $Detail.IsPresent -or $allswitch) { try { Get-MocLogs -path $logDir -activity $activity ` -VirtualMachineLogs:$VirtualMachineLogs.IsPresent ` -AgentLogs:$AgentLogs.IsPresent ` -NodeVirtualizationLogs:$EventLogs.IsPresent ` -Detail:$Detail.IsPresent | Out-Null } catch [Exception] { } } if ($allswitch -or $KvaLogs.IsPresent) { try { Get-KvaLogs -path $logDir -activity $activity } catch [Exception] { } } if ($allswitch -or $DownloadSdkLogs.IsPresent) { try { Get-DownloadSdkLogs -Path $logDir } catch [Exception] { } } if ($allswitch -or $BillingRecords.IsPresent) { New-Item -ItemType Directory -Force -Path $logDir | Out-Null try { Get-KvaBillingRecords -activity $activity -outputformat "json" | ConvertFrom-Json | Format-List * > ($logDir + "\AksHciBillingRecords.log") } catch [Exception]{ Write-Status -moduleName $moduleName -msg $($AksHciLocMessage.akshci_billing_collection_failed) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() } } $akshcilogDir = [io.Path]::Combine($logDir, "akshci") New-Item -ItemType Directory -Force -Path $akshcilogDir | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_collecting_info, $moduleName)) $global:config[$moduleName] > $akshcilogDir"\AksHciConfig.txt" Get-AksHciEventLog | Format-List * > $akshcilogDir"\AksHciPS.log" $modulelist = @("AksHci", "Kva", "Moc", "DownloadSdk", "Az.Resources", "Az.Accounts", "AzureAD", "TraceProvider" ) foreach ($module in $modulelist) { Get-Command -Module $module | Sort-Object -Property Source >> $($akshcilogDir+"\moduleinfo.txt") } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_compressing_logs) if ([string]::IsNullOrEmpty($zipName)) { $zipName = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], "$logName.zip") } try { Compress-Directory -ZipFilename $zipName -SourceDir $logDir } catch [Exception] { Write-Status -moduleName $moduleName -msg $($GenericLocMessage.generic_exception) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() Write-SubStatus -moduleName $moduleName -msg $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_cannot_compress, $zipName)) Write-Status -moduleName $moduleName -msg $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_log_path, $logDir)) Uninitialize-AksHciEnvironment -activity $activity return $logDir } Remove-Item -Path $logDir -Force -Recurse -ErrorAction Continue Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_zip_path, $zipName)) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $zipName } function Get-AksHciEventLog { <# .SYNOPSIS Gets all the event logs from the Azure Kubernetes Service on Azure Stack HCI PowerShell module. .DESCRIPTION Gets all the event logs from the Azure Kubernetes Service on Azure Stack HCI PowerShell module. #> $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime throw $_ } $logs = Get-WinEvent -ProviderName $moduleName -ErrorAction Ignore $logs += Get-KvaEventLog $logs += Get-MocEventLog $logs += Get-DownloadSdkEventLog Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $logs } function Enable-AksHciArcConnection { <# .SYNOPSIS Connects an AKS on Azure Stack HCI workload cluster to Azure Arc for Kubernetes. .DESCRIPTION Connects an AKS on Azure Stack HCI workload cluster to Azure Arc for Kubernetes. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER customLocationsOid object id of custom locations app .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [Parameter()] [String] $customLocationsOid, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; tenantId= $tenantId; subscriptionId= $subscriptionId; resourceGroup= $resourceGroup; location= $location; customLocationsOid= $customLocationsOid } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } # just to ensure the cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { New-KvaArcConnection -Name $Name -customLocationsOid $customLocationsOid -activity $activity } else { New-KvaArcConnection -Name $Name -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -customLocationsOid $customLocationsOid -activity $activity } Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_arc_installed) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Disable-AksHciArcConnection { <# .DESCRIPTION Helper function to remove the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; tenantId= $tenantId; subscriptionId= $subscriptionId; resourceGroup= $resourceGroup; location= $location } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } # just to ensure the cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Remove-KvaArcConnection -Name $Name -activity $activity } else { Remove-KvaArcConnection -Name $Name -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -activity $activity } Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_arc_uninstalled) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Install-AksHciAdAuth { <# .SYNOPSIS Install Active Directory authentication. .DESCRIPTION Install Active Directory authentication. .PARAMETER Name Cluster Name .PARAMETER keytab Path to the kerberos keytab corresponding to the current password on the local machine. Must be named current.keytab .PARAMETER previousKeytab Path to the kerberos keytab corresponding to the previous password on the local machine. Must be named previous.keytab .PARAMETER SPN SPN registered for the Active Directory account to be used with the api-server. .PARAMETER TTL Time to live (in hours) for previous keytab file if supplied. Default is 10 hours .PARAMETER adminUser The user name to be given cluster-admin permissions. Machine must be domain joined. .PARAMETER adminGroup The group name to be given cluster-admin permissions. Machine must be domain joined. .PARAMETER adminUserSID The user SID to be given cluster-admin permissions. .PARAMETER adminGroupSID The group SID to be given cluster-admin permissions. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='domainjoin')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter(Mandatory=$true)] [String] $keytab, [Parameter(Mandatory=$false)] [String] $previousKeytab, [Parameter(Mandatory=$true)] [String] $SPN, [Parameter(Mandatory=$false)] [int] $TTL, [Parameter(Mandatory=$false, ParameterSetName='domainjoin')] [String] $adminUser, [Parameter(Mandatory=$false, ParameterSetName='domainjoin')] [String] $adminGroup, [Parameter(Mandatory=$false, ParameterSetName='workplacejoin')] [String] $adminUserSID, [Parameter(Mandatory=$false, ParameterSetName='workplacejoin')] [String] $adminGroupSID, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $capiCluster = Get-KvaCapiCluster -Name $Name $canInstallWebhook = $false foreach($feature in $capiCluster.additionalfeatures ) { if ($feature.FeatureName -eq "ad-auth-webhook") { $canInstallWebhook = $true } } if (-not $canInstallWebhook) { Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_adauth_disabled, $Name)) Uninitialize-AksHciEnvironment -activity $activity return } if(-not $adminUser -and -not $adminUserSID -and -not $adminGroup -and -not $adminGroupSID) { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_addon_enable_req) Uninitialize-AksHciEnvironment -activity $activity return } try { if (![string]::IsNullOrEmpty($adminUser)) { $adminUserSIDYAML = (New-Object System.Security.Principal.NTAccount($adminUser)).Translate([System.Security.Principal.SecurityIdentifier]).value } if (![string]::IsNullOrEmpty($adminGroup)) { $adminGroupSIDYAML = (New-Object System.Security.Principal.NTAccount($adminGroup)).Translate([System.Security.Principal.SecurityIdentifier]).value } } catch { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_sid_translation_failed) Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -ChildStageName "NameToSIDTranslationFailed" ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity return } if ($PSCmdlet.ParameterSetName -ieq "workplacejoin") { $adminUserSIDYAML = $adminUserSID $adminGroupSIDYAML = $adminGroupSID } if (![string]::IsNullOrEmpty($previousKeytab)) { $prevKtSt = "--from-file=`"$previousKeytab`"" } $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: ad-auth-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: ad-auth-webhook targetNamespace: kube-system templateType: yaml providerVariables: - key: AD_AUTH_SPN value: "$SPN" - key: ADMIN_USER value: "$adminUserSIDYAML" - key: ADMIN_GROUP value: "$adminGroupSIDYAML" - key: TICKET_LIFETIME value: "$TTL" - key: keytab valueFrom: secret: name: keytab-$Name "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-ad-auth-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KubeCtl -arguments $("create secret generic keytab-$Name --from-file=`"$keytab`" $prevKtSt") Invoke-Kubectl -arguments $("apply -f ""$yamlFile"" ") Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_active_dir_sso) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Uninstall-AksHciAdAuth { <# .SYNOPSIS Uninstall Active Directory authentication. .DESCRIPTION Uninstall Active Directory authentication. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: ad-auth-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: ad-auth-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-ad-auth-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("delete -f ""$yamlFile"" ") Remove-Item $yamlFile Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_active_dir_sso_uninstalled) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Install-AksHciGMSAWebhook { <# .DESCRIPTION Installs gMSA webhook for an AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Set-KvaGMSAWebhook -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_installed) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Uninstall-AksHciGMSAWebhook { <# .DESCRIPTION Uninstalls gmsa-webhook addon for an AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Reset-KvaGMSAWebhook -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_uninstalled) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Add-AksHciGMSACredentialSpec { <# .DESCRIPTION Helper function to add a credentials spec for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecFilePath File Path of the JSON cred spec file .PARAMETER credSpecName Name of the Kubernetes credential spec object the user would like to designate This will be the name the deployment yaml reference for the field gmsaCredentialSpec .PARAMETER secretName Name of the Kubernetes secret object storing the Active Directory user credentials and gMSA domain .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER serviceAccount Name of the Kubernetes service account assigned to read the Kubernetes gMSA credspec object .PARAMETER clusterRoleName Name of the Kubernetes clusterrole assigned to use the Kubernetes gMSA credspec object .PARAMETER overwrite Overwrites existing Kubernetes credential spec object .PARAMETER activity Activity name to use when updating progress .EXAMPLE Add-AksHciGMSACredentialSpec -Name test1 -credSpecFilePath .\credspectest.json -credSpecName credspec-test1 -secretName secret-test1 -clusterRoleName clusterrole-test1 Creates a GMSACredentialSpec object called credspec-test1 from the JSON credential spec file credspectest.json on a target cluster named test1. The object credspec-test1 references the default namespaced secret secret-test1 created by the user for Active Directory user credentials. The cmdlet also creates a cluster role named clusterrole-test1 that binds to the default service account along with a rolebinding that resides in the default namespace. .EXAMPLE Add-AksHciGMSACredentialSpec -Name test1 -credSpecFilePath .\credspectest.json -credSpecName credspec-test1 -secretName secret-test1 -secretNamespace secret-namespace -clusterRoleName clusterrole-test1 -serviceAccount svc1 -overwrite Creates a GMSACredentialSpec object called credspec-test1 from the JSON credential spec file credspectest.json on a target cluster named test1. The object credspec-test1 references the secret secret-test1 residing in the namespace secret-namespace. Both the secret and the namespace secret-namespace are created by the user. The also cmdlet creates a cluster role named clusterrole-test1 that binds to the user-created service account svc1 along with a rolebinding that resides in the secret-namespace namespace. The overwrite parameter checks for existing GMSACredentialSpec, clusterrole, and rolebinding objects with the same names as the ones specified by the cmdlet parameters and overwrites them with the new setup based on the new parameters. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$Name, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecFilePath')] [ValidateScript({Test-ValidCredSpecJsonPath $jsonPath $_})] [String]$credSpecFilePath, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecName')] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$credSpecName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$secretName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$clusterRoleName, [Parameter()] [switch]$overwrite, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $Name} if (-not $activity) { $activity = $MyInvocation.MyCommand.Name } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Set-KvaGMSACredentialSpec -Name $Name -credSpecFilePath $credSpecFilePath -credSpecName $credSpecName ` -secretName $secretName -secretNamespace $secretNamespace -serviceAccount $serviceAccount ` -clusterRoleName $clusterRoleName -overwrite:$overwrite.isPresent -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_cred_spec_installed) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Remove-AksHciGMSACredentialSpec { <# .DESCRIPTION Helper function to remove a credentials spec for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecName Name of the Kubernetes credential spec object the user would like to designate .PARAMETER serviceAccount Kubernetes service account assigned to read the Kubernetes gMSA credential spec object .PARAMETER clusterRoleName Name of the Kubernetes clusterrole assigned to use the Kubernetes gMSA credential spec object .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER activity Activity name to use when updating progress .EXAMPLE Remove-AksHciGMSACredentialSpec -Name test1 -credSpecName credspec-test1 -clusterRoleName clusterrole-test1 Removes the GMSACredentialSpec object credspec-test1 and the clusterrole object clusterrole-test1 along with the rolebinding object binding clusterrole-test1 to the default service account from a target cluster named test1 .EXAMPLE Remove-AksHciGMSACredentialSpec -Name test1 -credSpecName credspec-test1 -serviceAccount svc1 -secretNamespace secret-namespace -clusterRoleName clusterrole-test1 Removes a GMSACredentialSpec object credspec-test1 and the clusterrole object clusterrole-test1 from the target cluster test1. The rolebinding object binding clusterrole-test1 to the service account svc1 is also removed from the secret-namespace namespace in the target cluster named test1. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$Name, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecName')] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$credSpecName, [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String]$clusterRoleName, [Parameter()] [String]$secretNamespace = "default", [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $Name} if (-not $activity) { $activity = $MyInvocation.MyCommand.Name } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Reset-KvaGMSACredentialSpec -Name $Name -credSpecName $credSpecName -serviceAccount $serviceAccount ` -clusterRoleName $clusterRoleName -secretNamespace $secretNamespace -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_cred_spec_uninstalled) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciCredential { <# .SYNOPSIS Access your cluster using kubectl. .DESCRIPTION Access your cluster using kubectl. This will use the specified cluster's kubeconfig file as the default kubeconfig file for kubectl. .PARAMETER Name Name of the cluster to obtain the credential/kubeconfig for. .PARAMETER configPath Location to output the credential/kubeconfig file to. .PARAMETER adAuth To get the Active Directory SSO version of the kubeconfig. .PARAMETER aadAuth To get the Azure RBAC Kubeconfig .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [string] $Name, [Parameter()] [string] $configPath = $($env:USERPROFILE+"\.kube\config"), [Parameter(Mandatory=$false)] [Switch] $adAuth, [Parameter(Mandatory=$false)] [Switch] $aadAuth, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $Name; adAuth= $adAuth; aadAuth= $aadAuth } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($Name, $("Retrieve and write the cluster kubeconfig file to $configPath"))) { Initialize-AksHciEnvironment -activity $activity Get-KvaClusterCredential -Name $Name -outputLocation $configPath -adAuth:$adAuth.IsPresent -aadAuth:$aadAuth.IsPresent -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } } function Repair-AksHciClusterCerts { <# .DESCRIPTION Attempts to repair failed TLS on a cluster/cloudagent .PARAMETER Name Name of the node/cluster to fix .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER $fixCloudCredentials Fix cloud tls in a cluster .PARAMETER $fixKubeletCredentials Fix failed TLS on a cluster .PARAMETER $patchLoadBalancer Patch load balancer certificates .PARAMETER force Force repair(without checks) .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(DefaultParameterSetName = 'cloud')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [string] $Name, [Parameter()] [string] $sshPrivateKeyFile, [Parameter(Mandatory=$true, ParameterSetName='cloud')] [Switch] $fixCloudCredentials, [Parameter(ParameterSetName='cloud')] [Switch] $patchLoadBalancer, [Parameter(Mandatory=$true, ParameterSetName='kubelet')] [Switch] $fixKubeletCredentials, [Parameter(ParameterSetName='cloud')] [Switch] $force, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $Name; fixCloudCredentials= $fixCloudCredentials; patchLoadBalancer= $patchLoadBalancer; fixKubeletCredentials= $fixKubeletCredentials; force= $force } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity if (-not $sshPrivateKeyFile) { $sshPrivateKeyFile = Get-UserSSHPrivateKey } if ($PSCmdlet.ParameterSetName -ieq "cloud") { Repair-KvaCerts -Name $Name -sshPrivateKeyFile $sshPrivateKeyFile -patchLoadBalancer:$patchLoadBalancer.IsPresent -force:$force.IsPresent -activity $activity } if($PSCmdlet.ParameterSetName -ieq "kubelet") { Repair-KvaCluster -Name $Name -sshPrivateKeyFile $sshPrivateKeyFile -fixCertificates -activity $activity } Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys } function Repair-AksHciCerts { <# .DESCRIPTION Attempts to repair failed TLS on a cluster . .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [string] $sshPrivateKeyFile, [Parameter()] [Switch] $force, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name)" } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity if (-not $sshPrivateKeyFile) { $sshPrivateKeyFile = Get-UserSSHPrivateKey } # Rotate NodeAgent tokens Repair-Moc # Rotate KVA and Management cluster tokens Repair-KvaCerts -sshPrivateKeyFile $sshPrivateKeyFile -force:$force.IsPresent -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Sync-AksHciBilling { <# .DESCRIPTION Sync Aks-Hci billing. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $syncResult = Sync-KvaBilling -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $syncResult } function Get-AksHciBillingStatus { <# .DESCRIPTION Get Aks-Hci billing status. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $statusResult = Get-KvaBillingStatus -activity $activity -outputformat "json" | ConvertFrom-Json Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $statusResult } function New-AksHciClusterNetwork { <# .DESCRIPTION Create network settings to be used for the target clusters. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNetDHCP = New-AksHciClusterNetwork -name e1 -vswitchName External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE $clusterVNetStatic = New-AksHciClusterNetwork -name e1 -vswitchName External -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNetworkName -Name $_})] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [ValidateRange(0, 4094)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIPPrefix -ipprefix $_})] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $gateway, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidDNSServers -dnsservers $_})] [String[]] $dnsservers, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $vippoolstart, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $vippoolend, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $k8snodeippoolend, [Parameter()] [String] $activity ) $startCmdletTime = Get-Date $networkdetailsCmdletParams = @{name= $name; vswitchName= $vswitchName } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime -CmdletParameters $networkdetailsCmdletParams Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity $isVipPoolPresent = ((-not [string]::IsNullOrWhiteSpace($vippoolstart)) -or (-not [string]::IsNullOrWhiteSpace($vippoolend))) if (-not $isVipPoolPresent) { $defaultVipPool = (Get-MocConfig)["defaultVipPoolName"] if ([string]::IsNullOrWhiteSpace($defaultVipPool)) { throw [CustomException]::new($($AksHciLocMessage.akshci_no_vippools_configured), ([ErrorTypes]::IsUserErrorFlag)) } } $vnet = $(New-KvaClusterNetwork -name $name -vswitchname $vswitchname -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vlanID $vlanID -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend -activity $activity) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $networkdetailsCmdletParams -BoundParameterKeys $PSBoundParameters.Keys Uninitialize-AksHciEnvironment -activity $activity return $vnet } function Get-AksHciClusterNetwork { <# .DESCRIPTION Gets the VirtualNetwork object for a target cluster given either the vnet name or the cluster name. If no parameter is given, all vnet's are returned. .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidNetworkName -Name $_})] [string] $name, [Parameter(Mandatory=$false)] # [ValidateScript({Test-ValidClusterName -Name $_})] [string] $clusterName, [Parameter()] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $name; clusterName= $clusterName} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity $vnet = $(Get-KvaClusterNetwork -name $name -clusterName $clusterName -activity $activity) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys Uninitialize-AksHciEnvironment -activity $activity return $vnet } function Remove-AksHciClusterNetwork { <# .DESCRIPTION Remove a virtual network object for a target cluster .PARAMETER name The name of the vnet .NOTES The cmdlet will throw an exception if the network is still being used. The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE Remove-AksHciClusterNetwork -name e1 #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNetworkName -Name $_})] [string] $name, [Parameter()] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{name= $name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity $vnet = Remove-KvaClusterNetwork -name $name -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys Uninitialize-AksHciEnvironment -activity $activity return $vnet } function Install-AksHciMonitoring { <# .DESCRIPTION Installs monitoring infrastructure on AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER storageSizeGB Amount of storage for Prometheus in GB .PARAMETER retentionTimeHours metrics retention time in hours. (min 2 hours, max 876000 hours(100 years)) .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter(Mandatory=$true)] [ValidateRange(1, 2147483647)] [int] $storageSizeGB, [Parameter(Mandatory=$true)] [ValidateRange(2,876000)] # see kva.psm1 [int] $retentionTimeHours, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name; storageSizeGB= $storageSizeGB; retentionTimeHours= $retentionTimeHours } if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_installing_monitoring) Set-KvaHciMonitoring -Name $Name -storageSizeGB $storageSizeGB -retentionTimeHours $retentionTimeHours -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_installed_monitoring) Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_monitoring_progress) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Uninstall-AksHciMonitoring { <# .DESCRIPTION Uninstalls monitoring from an AKS-HCI cluster. .PARAMETER Name cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_uninstalling_monitoring) Reset-KvaHciMonitoring -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_uninstalled_monitoring) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Add-AksHciNode { <# .DESCRIPTION Add new node to the Moc stack during a Failure Replacement Unit scenario .PARAMETER nodeName The name of the node in the Failover Cluster, the node is already expected to have been added to the failover cluster .PARAMETER activity Activity name to use when updating progress .EXAMPLE Add-AksHciNode -nodeName "node1" #> param ( [String]$nodeName, [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{nodeName= $nodeName} trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_adding_node) $workingDir = $global:config[$modulename]["workingDir"] Save-ConfigurationDirectoryNode -nodeName $nodeName -moduleName $moduleName -WorkingDir $workingDir Initialize-KvaNode -nodeName $nodeName New-MocPhysicalNode -nodeName $nodeName -activity $activity Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Remove-AksHciNode { <# .DESCRIPTION Remove a failed node from the Moc stack during a Failure Replacement Unit scenario .PARAMETER nodeName The name of the node in Failover Cluster .PARAMETER activity Activity name to use when updating progress .NOTES If the physical machine is shut down or removed or unreachable on the network prior to the cmdlet this guarntees that it is removed from the cloud-agent maps but not a complete cleaup of that node. .EXAMPLE Remove-AksHciNode -nodeName "node1" #> param ( [String]$nodeName, [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{nodeName= $nodeName} trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_removing_node) Remove-MocPhysicalNode -nodeName $nodeName -activity $activity Uninitialize-KvaNode -nodeName $nodeName Delete-ConfigurationDirectoryNode -nodeName $nodeName -moduleName $moduleName Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function New-AksHciProxySetting { <# .DESCRIPTION Create proxy settings to be used for the Aks Hci deployment .PARAMETER name A name to associate with the proxy settings .PARAMETER http HTTP proxy server configuration .PARAMETER https HTTPS proxy server configuration .PARAMETER noProxy Proxy server exemption/bypass list .PARAMETER certFile Path to a CA certificate file used to establish trust with a HTTPS proxy server .PARAMETER credential Proxy server credentials (for basic authentication) .OUTPUTS Proxy Settings object .EXAMPLE $credential = Get-Credential $proxySetting = New-AksHciProxySetting -http http://contosoproxy:8080 -https https://contosoproxy:8080 -noProxy "localhost,127.0.0.1,.svc,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" -credential $credential -certFile c:\proxyca.crt #> param ( [Parameter()] [String] $name, [Parameter()] [String] $http, [Parameter()] [String] $https, [Parameter()] [String] $noProxy = $global:defaultProxyExemptions, [Parameter()] [String] $certFile, [Parameter()] [PSCredential] $credential = [PSCredential]::Empty ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime throw $_ } Test-ProxyConfiguration -http $http -https $https -noProxy $noProxy -certFile $certFile Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return [ProxySettings]::new($credential, $name, $http, $https, $noProxy, $certFile) } function Set-AksHciProxySetting { <# .DESCRIPTION Update proxy settings noProxy list .PARAMETER noProxy Proxy server exemption/bypass list .PARAMETER certFile Root CA cert file path to install CA cert .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $noProxy, [Parameter()] [String] $certFile, [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } if ([string]::IsNullOrEmpty($noProxy) -and [string]::IsNullOrEmpty($certFile)) { throw [CustomException]::new($($AksHciLocMessage.akshci_set_proxysettings_missing_parameters), ([ErrorTypes]::IsUserErrorFlag)) } Initialize-AksHciEnvironment -activity $activity $proxySettings = Get-AksHciProxySetting if ($proxySettings.HTTP -eq "" -and $proxySettings.HTTPS -eq "" -and $noProxy -ne "") { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_proxy_update_failed) Uninitialize-AksHciEnvironment -activity $activity return } if (-not ([string]::IsNullOrEmpty($noProxy))) { $proxySettings.noProxy = $noProxy } if (-not ([string]::IsNullOrEmpty($certFile))) { $proxySettings.CertFile = $certFile } Test-ProxyConfiguration -http $proxySettings.HTTP -https $proxySettings.https -noProxy $proxySettings.noProxy -certFile $proxySettings.CertFile Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $global:MocModule Set-KvaProxySetting -proxySettings $proxySettings Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciProxySetting { <# .DESCRIPTION Returns AksHci proxy settings .PARAMETER activity Activity name to use when updating progress .OUTPUTS Proxy Settings object #> param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $http = $global:config[$moduleName]["proxyServerHTTP"] $https = $global:config[$moduleName]["proxyServerHTTPS"] $noProxy = $global:config[$moduleName]["proxyServerNoProxy"] $certFile = $global:config[$moduleName]["proxyServerCertFile"] $credentials = [PSCredential]::Empty if ($($global:config[$moduleName]["proxyServerUsername"]) -and $($global:config[$moduleName]["ProxyServerPassword"])) { $securePass = $($global:config[$moduleName]["ProxyServerPassword"]) | ConvertTo-SecureString -Key $global:credentialKey $credentials = New-Object System.Management.Automation.PSCredential -ArgumentList $($global:config[$moduleName]["proxyServerUsername"]), $securePass } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys Uninitialize-AksHciEnvironment -activity $activity return [ProxySettings]::new($credentials, "", $http, $https, $noProxy, $certFile) } function New-AksHciContainerRegistry { <# .DESCRIPTION Create container registry settings to be used for the Aks Hci deployment .PARAMETER server The container registry server name .PARAMETER credential Credential to connect to the container registry (if required) .OUTPUTS Container Registry object .EXAMPLE $credential = Get-Credential $registry = New-AksHciContainerRegistry -server "ecpacr.azurecr.io" -credential $credential #> param ( [Parameter(Mandatory=$true)] [String] $server, [Parameter()] [PSCredential] $credential = [PSCredential]::Empty ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return [ContainerRegistry]::new($credential, $server) } function Invoke-AksHciRotateCACertificate { <# .DESCRIPTION Rotate cloudagent CA certificate .PARAMETER activity Activity name to use when updating progress #> param ( [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_rotate_moc_ca_certificate) Invoke-MocRotateCACertificate -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_cloud_certificates) Invoke-AksHciPatchCloudCertificates -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys Uninitialize-AksHciEnvironment -activity $activity } function Invoke-AksHciPatchCloudCertificates { <# .DESCRIPTION Patch all the cloudagent certificate in the cluster .PARAMETER activity Activity name to use when updating progress #> param ( [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_nodeagent_certificates) Repair-AksHciCerts -force Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_mgmt_certificates) $clusters = Get-AksHciCluster foreach($cluster in $clusters) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_patching_cluster_certificates, $cluster.Name)) Repair-AksHciClusterCerts -Name $cluster.Name -fixCloudCredentials -patchLoadBalancer -force } Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_cloud_certificates_complete) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } #endregion #region Installation and Provisioning functions function Install-AksHciInternal { <# .DESCRIPTION The main deployment method for AksHci. This function is responsible for installing MOC stack and the management appliance/cluster. .PARAMETER credential credential is a PSCredential holding a user's Service Principal. .PARAMETER timeoutMinutes Timeout in minutes for the installation to complete .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$false)] [int]$timeoutMinutes, [Parameter(Mandatory=$false)] [PSCredential] $credential, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $uninstallAksHciCommand = "Uninstall-AksHci" $isAksHciUnInstalling = Test-Command-Runnable -currentCommand $activity ` -cannotRunWithInstallState Uninstalling ` -cannotRunWithCommand $uninstallAksHciCommand if ($isAksHciUnInstalling) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_uninstall_in_progress, $moduleName)), ([ErrorTypes]::IsUserErrorFlag)) } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installing) $startCmdletTime = Get-Date try { $currentMocState = Get-InstallState -module $global:MocModule switch ($currentMocState) { ([InstallState]::Installed) { # No-Op. Precheck ensures that installed Moc version is valid. break } Default { Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_moc_current_state, $currentMocState)) throw [CustomException]::new( $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_moc_not_installed, $currentMocState)), ([ErrorTypes]::IsUserErrorFlag)) } } Get-AksHciPackage -Version (Get-AksHciVersion) Install-Kva -activity $activity -timeoutMinutes $timeoutMinutes } catch { $errorMessage = "" try { $skipClean = Get-AksHciConfigValue -name "skipCleanOnFailure" if (-not $skipClean) { # Save the logs to temp $zipName = [io.Path]::GetTempFileName() + ".zip" Get-AksHciLogs -zipName $zipName | Out-Null $errorMessage += "`r`n Logs are available at $zipName" # Uninstall will not happen if installState is in Installing state Set-AksHciConfigValue -name "installState" -value ([InstallState]::InstallFailed) Uninstall-AksHci -SkipConfigCleanup:$True -activity $activity -Confirm:$false } } catch { Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } Set-AksHciConfigValue -name "installState" -value ([InstallState]::InstallFailed) $errorMessage = Write-ModuleEventException -message $errorMessage -exception $_ -moduleName $modulename throw [CustomException]::new([System.Exception]::new($errorMessage, $_.Exception), ([ErrorTypes]::IsErrorFlag)) } Write-Status -moduleName $moduleName $($AksHciLocMessage.akshci_installation_complete) Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installed) } function Initialize-AksHciEnvironment { <# .DESCRIPTION Executes steps to prepare the environment for AksHci operations. .PARAMETER createConfigIfNotPresent Whether the call should create a new AksHci deployment configuration if one is not already present. .PARAMETER skipMgmtKubeConfig Whether the call should skip a check to ensure that a appliance/management kubeconfig is present. .PARAMETER activity Activity name to use when updating progress #> param ( [Switch]$createConfigIfNotPresent, [Switch]$skipMgmtKubeConfig, [Switch]$skipInstallationCheck, [Parameter(Mandatory=$true)] [String]$activity ) Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_initializing_environment) -moduleName $moduleName Import-AksHciConfig -createIfNotPresent:($createConfigIfNotPresent.IsPresent) -activity $activity Initialize-Environment -checkForUpdates:$false -moduleName $script:moduleName -activity $activity $date = Get-AksHciConfigValue -name "latestVersionsCachedOn" if($date.GetType().Name -ne "DateTime"){ $date=Get-date Set-AksHciConfigValue -name "latestVersionsCachedOn" -value $date } $futureDate = $date.AddDays(2).Date $todaysDate = Get-date $LatestAksHciVersion = Get-AksHciConfigValue -name "cachedLatestAksHciVersion" $latestPSVersion = Get-AksHciConfigValue -name "cachedLatestPSVersion" $minAksHciVersion = Get-AksHciConfigValue -name "cachedMinAksHciVersion" $CurrentAksHciVersion = $global:config[$modulename]["version"] if(($todaysDate -gt $futureDate) -or ($LatestAksHciVersion -eq "") -or ($latestPSVersion -eq "") -or ($minAksHciVersion -eq "")) { Set-CachedVersions -date $todaysDate $LatestAksHciVersion = Get-AksHciConfigValue -name "cachedLatestAksHciVersion" $latestPSVersion = Get-AksHciConfigValue -name "cachedLatestPSVersion" $minAksHciVersion = Get-AksHciConfigValue -name "cachedMinAksHciVersion" } try { #Warn if Current PS module version is not latest if ([version]$latestPSVersion -gt [version]$moduleVersion){ Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_latest_PS_module_warning, $latestVersion)) } #Warn If Current AksHci Version is not latest if ([version]$CurrentAksHciVersion -lt [version]$LatestAksHciVersion){ Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_latest_version_warning, $LatestAksHciVersion, $CurrentAksHciVersion)) } #warn for minimum compatible AKSHCI version if($minAksHciVersion -and ([version]$CurrentAksHciVersion -le [version]$minAksHciVersion)){ Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_minsupported_version_warning, $minAksHciVersion)) } } catch [Exception] {} if (-not $skipInstallationCheck.IsPresent) { if (-not (Test-IsProductInstalled -moduleName $moduleName -activity $activity)) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_module_not_installed, $moduleName))), ([ErrorTypes]::IsUserErrorFlag)) } } if (-not ($skipMgmtKubeConfig.IsPresent)) { Get-Kva -activity $activity | Out-Null } Initialize-MocEnvironment -activity $activity Uninitialize-MocEnvironment -activity $activity if($global:config) { $data = (($global:config).Values.Values).Where({$_.Length -lt 64}).Where({![string]::IsNullOrWhiteSpace($_)}).Where({$_ -notmatch "^\d+$"}) Initialize-TraceCmdlet -Data $data } } function Uninitialize-AksHciEnvironment { <# .DESCRIPTION Executes steps to teardown the environment for AksHci operations. .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String]$activity ) Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_uninitializing_environment) -moduleName $moduleName Uninitialize-Environment -moduleName $script:moduleName -activity $activity } function Get-AksHciVersion { <# .SYNOPSIS Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError ` -BoundParameterKeys $PSBoundParameters.Keys ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys Uninitialize-AksHciEnvironment -activity $activity return $global:config[$modulename]["version"] } function Update-AksHci { <# .SYNOPSIS Update the Azure Kubernetes Service host to the latest Kubernetes version. .DESCRIPTION Update the Azure Kubernetes Service host to the latest Kubernetes version. Performed as step updates, updating to the next available version until latest version is achieved. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) <# 1. Check if versions later than current version are available a. If yes, prompt for upgrade b. If no, return silenty, printing a message 2. If upgrade is requested, do the following check a. 3. In case of multiple available newer versions a. Run upgrade from current version to next, repeating until latest available version is achieved b. Upon failure, return to last successful version to be upgraded to 4. Handle No target cluster scenarios 5. Handle No Target and Mgmt cluster scenarios 6. Handle scenario when the product is not installed #> $startCmdletTime = Get-Date $startingVersion = "" $versionsToUpgrade = @() $targetAksHciVersion = "" $updateAksHciCorrelationId = "" $stepUpdateAksHciCorrelationId = "" trap { # Set the installState to failed, no matter what the exception is Set-AksHciConfigValue -name "installState" -value ([InstallState]::UpdateFailed) $cmdletparms = $PSBoundParameters; $cmdletparms += @{config = Get-AksHciConfig} #This is original PowerShell Error event which is same for all cmdlets Trace-CmdletError ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -BoundParameterKeys $PSBoundParameters.Keys ` -CmdletParameters $cmdletparms ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -AksHciTargetVersion $targetAksHciVersion #This is new error event specific to Update cmdlet #This is the 'PowerShell.UpdateAksHci.StepUpdate.Complete' Error Event Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -IsSuccess $false ` -IsStepUpdateAksHciComplete $true ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId ` -StepUpdateAksHciCorrelationId $stepUpdateAksHciCorrelationId #This is the 'PowerShell.UpdateAksHci.Complete' Error Event Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -IsSuccess $false ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId Uninitialize-AksHciEnvironment -activity $activity throw $_ } #Get the initial/starting Akshci version $startingVersion = Get-AksHciVersion if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity $updateAksHciCommand = "Update-AksHci" $updateAksHciCommandRegex = "^$updateAksHciCommand$" $isAksHciUpdateRunning = Test-Command-Runnable -currentCommand $activity ` -cannotRunWithInstallState Updating ` -cannotRunWithCommand $updateAksHciCommand ` -cannotRunWithCommandRegex $updateAksHciCommandRegex if ($isAksHciUpdateRunning) { Uninitialize-AksHciEnvironment -activity $activity throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_update_in_progress, $moduleName)), ([ErrorTypes]::IsUserErrorFlag)) } $newAksHciClusterCommand = "New-AksHciCluster" $isAksHciClusterCreateRunning = Test-Command-Runnable ` -currentCommand $activity ` -cannotRunWithCommand $newAksHciClusterCommand if ($isAksHciClusterCreateRunning) { Uninitialize-AksHciEnvironment -activity $activity throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_already_running_command_with_suggestion, $newAksHciClusterCommand, $activity)), ([ErrorTypes]::IsUserErrorFlag)) } $updateAksHciTargetClusterCommand = "Update-AksHciCluster" $isAksHciTargetClusterUpdateRunning = Test-Command-Runnable ` -currentCommand $activity ` -cannotRunWithCommand $updateAksHciTargetClusterCommand if ($isAksHciTargetClusterUpdateRunning) { Uninitialize-AksHciEnvironment -activity $activity throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_already_running_command_with_suggestion, $updateAksHciTargetClusterCommand, $activity)), ([ErrorTypes]::IsUserErrorFlag)) } Set-CachedVersions $MinPSVersion = Get-AksHciConfigValue -name "cachedLatestPSVersion" if (![String]::IsNullOrEmpty($MinPSVersion) -and ([version]$moduleVersion -lt [version]$MinPSVersion)) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_incompatible_PS_module_version, $MinPSVersion))), ([ErrorTypes]::IsUserErrorFlag)) } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_updating) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_current_version, $(Get-AksHciVersion))) $targetAksHciVersion = (Get-LatestRelease -moduleName $moduleName).Version ## Set caCertRotationThreshold value if not set $certThreshold = Get-ConfigurationValue -Name "caCertRotationThreshold" -module $moduleName if ((-not $certThreshold) -or ($certThreshold -eq 0)) { Set-ConfigurationValue -name "caCertRotationThreshold" -value $global:caCertRotationThreshold -module $moduleName } #Fetch deploymentID to pass to Update Moc during Upgrade. This can be removed once May2022 release is obselete $deploymentId = Get-ConfigurationValue -Name "deploymentId" -module $moduleName Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_suitable_version) #If current PS module Version is less than Minimum supported PS version, then failing the update call. $productRelease = Get-ProductRelease -version $targetAksHciVersion -module $moduleName if($productRelease.CustomData.MinSupportedPSVersion){ $MinPSVersion = $productRelease.CustomData.MinSupportedPSVersion if ([version]$moduleVersion -lt [version]$MinPSVersion){ throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_incompatible_PS_module_version, $MinPSVersion))), ([ErrorTypes]::IsUserErrorFlag)) } } #One event for each instance of [Update-AksHci] (i.e. before entering in update loop capture Start update) #This Update Start event should have a correlation id that is the same for all events in a single execution instance of [Update-AksHci] $updateAksHciCorrelationId = $(New-Guid) Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId ` -IsUpdateAksHciStarted $true Set-AksHciConfigValue -name "installState" -value ([InstallState]::Updating) # Trigger the moc update to latest # if update-moc failed to update to latest, it will revert it back to current version # Fail the update if moc was installed by a different entity # mocInstalledByAksHci does not exist for older releases of AksHci, update Moc to latest $mocInstalledByAksHci = Get-AksHciConfigValue -name "mocInstalledByAksHci" if ([string]::IsNullOrEmpty($mocInstalledByAksHci) -or $mocInstalledByAksHci) { Update-Moc -activity $activity -deploymentId $deploymentId -skipHostAgentUpdate $true Set-AksHciConfigValue -name "mocInstalledByAksHci" -value $true } else { Test-AksHciSupportedMocVersion -aksHciVersion $targetAksHciVersion } $testResult = Test-UpdateAksHci if ($testResult.TestResult -eq "Failed") { Write-Warning $($testResult.Details) throw [CustomException]::new($($testResult.Recommendation), ([ErrorTypes]::IsUserErrorFlag)) # if (-not $PSCmdlet.ShouldProcess("Azure Stack HCI deployment", "Update-AksHci" )) # { # Write-Warning $($AksHciLocMessage.akshci_update_abort_warning) # return # } } $nextUpdate = Get-AksHciLatestUpdate $nextVersion = $nextUpdate.Version while(-not [string]::IsNullOrWhiteSpace($nextVersion)) { Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_upgrade_version, $nextVersion)) $currentInstallationPath = $global:config[$modulename]["installationPackageDir"] # We found a version to Upgrade to # 1. Download the package Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_package_version, $nextVersion)) Get-AksHciPackage -Version $nextVersion #Before starting an update, Capture this event, in case update is hung; Get the update details for next possible update #For every new update, generate a new updateCorrelationId so that start and end events can be correlated $stepUpdateAksHciCorrelationId = $(New-Guid) $stepUpdateStartTime = Get-Date Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -AksHciInitialVersion $(Get-AksHciVersion) ` -AksHciTargetVersion $nextVersion ` -StepUpdateAksHciCorrelationId $stepUpdateAksHciCorrelationId ` -IsStepUpdateAksHciStarted $true ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId try { $newInstallationPath = [io.Path]::Combine($global:config[$modulename]["workingDir"], $nextVersion) Set-AksHciConfigValue -name "installationPackageDir" -value $newInstallationPath New-Item -ItemType Directory -Force -Path $newInstallationPath | Out-Null # Trigger the appliance update - What happens when appliance update fails. Update-Kva -activity $activity -version $nextVersion # Set the version, once successful Set-AksHciConfigValue -name "version" -value $nextVersion #After every successful update, send a Success Event Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $stepUpdateStartTime ` -IsSuccess $true ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -StepUpdateAksHciCorrelationId $stepUpdateAksHciCorrelationId ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId ` -IsStepUpdateAksHciComplete $true } catch { Set-AksHciConfigValue -name "installState" -value ([InstallState]::UpdateFailed) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_update_failed, $_.Exception.Message)) Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_cleaning_up_updates) # Cleanup and Revert Set-AksHciConfigValue -name "installationPackageDir" -value $currentInstallationPath # We will not revert the MOC if the kva update failed # Do we need to cleanup the downloaded package - keep it, so we customers may attempt to update again throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.generic_update_failed, $_)), ([ErrorTypes]::IsErrorFlag)) } # When upgrading from 1.0.1.10628 to 1.0.2.10719, there is a bug which can cause billing to go out of policy. # To prevent this from happening, in a chained upgrade case, force a sync inbetween upgrades. try { Sync-AksHciBilling | Out-Null } catch [Exception] { $errorMessageString = $_.Exception.Message.ToString() Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_failed_billing_sync, $errorMessageString)) } $nextUpdate = Get-AksHciLatestUpdate $nextVersion = $nextUpdate.Version } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName $currentVersion = (Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName).Version if ($currentVersion -ine $targetAksHciVersion) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_update_not_latest, $currentVersion))), ([ErrorTypes]::IsUserErrorFlag)) } $isCertificateRotationRequired = Test-CloudCACertificateNearingExpiry -expiryThresholdDays $global:config[$moduleName]["caCertRotationThreshold"] if ($isCertificateRotationRequired) { Invoke-AksHciRotateCACertificate -activity $activity } Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -AksHciTargetVersion $targetAksHciVersion ` -BoundParameterKeys $PSBoundParameters.Keys -CmdletParameters @{config = Get-AksHciConfig} Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -IsSuccess $true ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId } #endregion #region Helper Functions function Get-AksHciPackage { <# .DESCRIPTION Downloads the package of the specified AksHCI Version .PARAMETER Version Version #> param ( [Parameter(Mandatory=$true)] [String]$Version ) # Validate the version Get-ProductRelease -Version $Version -moduleName $moduleName | Out-Null } function Get-AksHciLatestVersion { <# .DESCRIPTION Get the latest AksHci version #> $catalog = Get-LatestCatalog -moduleName $moduleName return $catalog.ProductStreamRefs[0].ProductReleases[0].Version } function Get-AksHciLatestUpdate { $updates = Get-AksHciUpdates if ($updates.Count -eq 0) { return } $latestUpdate = $updates[-1] ForEach ($tmpUpdate in $updates.values) { if ($tmpUpdate.CanUpgradeTo -and [version]$tmpUpdate.Version -gt [version]$latestUpdate.Version) { $latestUpdate = $tmpUpdate } } if (-not $latestUpdate.CanUpgradeTo) { return } return $latestUpdate } function Get-AksHciUpdates { <# .SYNOPSIS List the available Kubernetes updates for Azure Kubernetes Service on Azure Stack HCI in order. .DESCRIPTION List the available Kubernetes updates for Azure Kubernetes Service on Azure Stack HCI in order from latest to earliest. #> [CmdletBinding()] param () $startCmdletTime = Get-Date $activity = "$($MyInvocation.MyCommand.Name) - $Name" trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity $latestRelease = Get-LatestRelease -moduleName $moduleName $currentRelease = Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName $latestVersion = $latestRelease.Version $currentVersion = $currentRelease.Version $currentKvaK8sVersion = $currentRelease.CustomData.ManagementNodeImageK8sVersion $upgradePath = [ordered]@{} if ($latestVersion -ieq $currentVersion) { Uninitialize-AksHciEnvironment -activity $activity return } # There may be more updates that users might have not applied. # Show them the complete list, so they are aware of what will be updated # Assumption here is that product releases would be returned in order $updateReleases = Get-ProductReleasesUptoVersion -Version $currentVersion -moduleName $moduleName $targetKubernetesVersions = Get-TargetClusterKubernetesVersions $updateReleases | ForEach-Object { $tmp = $_ $tmpVersion = $tmp.Version $supportedK8sVersions = Get-AvailableKubernetesVersions -akshciVersion $tmpVersion -moduleName $moduleName $computedRelease = @{ Version = $tmpVersion; SupportedKubernetesVersions = $supportedK8sVersions; CanUpgradeTo = $false; } if ($tmpVersion -ieq $currentVersion) { $computedRelease += @{ Comments = "This is your CURRENT Version"; } } if ($tmpVersion -ieq $latestVersion) { $computedRelease += @{ Comments = "This is the LATEST Version"; } } if ([System.Version]$tmpVersion -gt [System.Version]$currentVersion) { $script:canupgrade = $true try { # Validate that this powershell module is compatible with the proposed upgrade Test-ModuleCompatibility -Version $tmpVersion | Out-Null } catch [Exception] { $computedRelease += @{ Recommendation = $_.Exception.Message.ToString(); } $script:canupgrade = $false Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime } if ($script:canupgrade) { # Need to check if we can upgrade from current KVA k8s version to next $productRelease = Get-ProductRelease -version $tmpVersion -moduleName $moduleName $targetKvaK8sVersion = $productRelease.CustomData.ManagementNodeImageK8sVersion if (-not (Test-KubernetesVersionUpdate -currentVersion $currentKvaK8sVersion -targetVersion $targetKvaK8sVersion)) { $computedRelease += @{ Recommendation = "Management Cluster Kubernetes Version $currentKvaK8sVersion cannot be updated to Management Cluster Kubernetes Version (" + $targetKvaK8sVersion + ") for $tmpVersion. Please upgrade to earlier release if possible, otherwise manual intervention is required."; } $script:canupgrade = $false } } # Validate that the current target cluster k8s versions are still supported by the proposed upgrade if ($script:canupgrade) { $outOfSupportVersions = Get-TargetClusterVersionsOutOfSupport -version $tmpVersion if ($outOfSupportVersions -and $outOfSupportVersions.Count -gt 0) { foreach ($outOfSupportVersion in $outOfSupportVersions.GetEnumerator()) { $kubernetesVersion = $outOfSupportVersion.Name $targetClusters = $outOfSupportVersion.Value if ($targetClusters.Count -gt 1) { $computedRelease += @{ Recommendation = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_clusters_on_unsupported_version, $targetClusters -join ",", $kubernetesVersion)) } } else { $computedRelease += @{ Recommendation = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_cluster_on_unsupported_version, [System.String]$targetClusters, $kubernetesVersion)) } } } $script:canupgrade = $false } } if ($script:canupgrade) { $computedRelease.CanUpgradeTo = $true $computedRelease += @{ Recommendation = "You can upgrade to AksHci Version [$tmpVersion]"; } } } $upgradePath[$tmpVersion] = $computedRelease; } Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $upgradePath } function Test-KubernetesVersionUpdate { <# .DESCRIPTION Test if the specified current kubernetes version is able to upgrade to the specified target version .PARAMETER currentVersion Current k8s version .PARAMETER targetVersion K8s version to upgrade to #> param ( [Parameter(Mandatory=$true)] [String] $currentVersion, [Parameter(Mandatory=$true)] [String] $targetVersion ) $current = [Version]$currentVersion.TrimStart("v") $target = [Version]$targetVersion.TrimStart("v") if ($current.Major -ne $target.Major) { return $false } if ($current.Minor -eq $target.Minor -or $current.Minor+1 -eq $target.Minor) { return $true } return $false } function Test-SupportedKubernetesVersion { <# .DESCRIPTION Test if the specified kubernetes version is supported by the current deployment .PARAMETER K8sVersion Kubernetes version to test .PARAMETER imageType Image type can be Windows or Linux .PARAMETER osSku SKU of the image can be CBLMariner, Windows2019 or Windows2022 #> param ( [Parameter(Mandatory=$true)] [String] $K8sVersion, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String] $imageType, [ValidateSet("CBLMariner", "Windows2019", "Windows2022")] [OsSku] $osSku = $(If ($imageType -eq "Linux") {[OsSku]::CBLMariner} Else {[OsSku]::Windows2019}) ) $availableVersions = Get-AvailableKubernetesVersions -moduleName $moduleName foreach($version in $availableVersions) { if (($version.OS -ieq $imageType) -and ($version.OrchestratorVersion -ieq $k8sVersion) -and ([string]::IsNullOrEmpty($version.SKU) -or ($version.SKU -ieq $osSku))) { return } } throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_unsupported_k8s, $k8sVersion, $imageType))), ([ErrorTypes]::IsUserErrorFlag)) } function Get-TargetClusterVersionsOutOfSupportOnLatest { <# .DESCRIPTION Return a table of out of support k8s versions for a the latest AksHci version to target clusters using them. Nothing is returned if no out of support versions are being used. #> $startCmdletTime = Get-Date $activity = "$($MyInvocation.MyCommand.Name) - $Name" trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity $latestRelease = Get-LatestRelease -moduleName $moduleName $outOfSupportVersions = Get-TargetClusterVersionsOutOfSupport -version $latestRelease.Version -activity $activity Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $outOfSupportVersions } function Get-TargetClusterVersionsOutOfSupport { <# .DESCRIPTION Return a table of out of support k8s versions for a specified AksHci version to target clusters using them. Nothing is returned if no out of support versions are being used. .PARAMETER version AksHci release version .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $version, [parameter(DontShow)] [String] $activity ) $versionClusters = Get-TargetClusterKubernetesReferences $outOfSupportVersions = @{} if ($versionClusters -and $versionClusters.Count -gt 0) { $earliestVersion = Get-EarliestKubernetesVersion -akshciVersion $version -moduleName $moduleName foreach ($versionCluster in $versionClusters.GetEnumerator()) { $kubernetesVersion = $versionCluster.Name $targetClusters = $versionCluster.Value if ([Version]$kubernetesVersion.TrimStart("v") -lt [Version]$earliestVersion.TrimStart("v") -and -not (Test-KubernetesVersionUpdate -currentVersion $kubernetesVersion -targetVersion $earliestVersion)) { $outOfSupportVersions[$kubernetesVersion] = $targetClusters } } } return $outOfSupportVersions } function Test-UpdateAksHci { <# .DESCRIPTION Validate if an update to the latest version of AksHci is possible - Check if any target clusters are out of support in the latest AksHci version #> $startCmdletTime = Get-Date $activity = "$($MyInvocation.MyCommand.Name) - $Name" trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity $result = [pscustomobject]@{ 'TestName' = "Validate AksHci Update"; 'Category' = "AksHci"; 'TestResult' = "Success"; 'Details' = ""; 'Recommendation' = '' } $outOfSupportVersions = Get-TargetClusterVersionsOutOfSupportOnLatest if ($outOfSupportVersions -and $outOfSupportVersions.Count -gt 0) { $details = "" foreach ($outOfSupportVersion in $outOfSupportVersions.GetEnumerator()) { $kubernetesVersion = $outOfSupportVersion.Name $targetClusters = $outOfSupportVersion.Value if ($targetClusters.Count -gt 1) { $details += $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_clusters_on_unsupported_version, $targetClusters -join ",", $kubernetesVersion)) } else { $details += $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_cluster_on_unsupported_version, [System.String]$targetClusters, $kubernetesVersion)) } } $result.TestResult = "Failed" $result.Details = $details $result.Recommendation = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_out_of_support_target_cluster_version)) } Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $result } function Get-NextKubernetesVersionForUpgrade { <# .DESCRIPTION Get the next Kubernetes Version for Upgrade. .PARAMETER Name Cluster name .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $upgrades = Get-KvaClusterUpgrades -Name $Name -activity $activity if ($upgrades.AvailableUpgrades.Count -eq 0) { return $null } $kubernetesVersionArray = @() foreach($availableVersion in $upgrades.AvailableUpgrades) { $kubernetesVersionArray += Get-CleanInputKubernetesVersion -KubernetesVersion $availableVersion.kubernetesVersion -Semver } $sorted = $kubernetesVersionArray | ForEach-Object { new-object System.Version ($_) } | Sort-Object -Descending $highestUpgradeAvailable = $sorted[0].ToString() return "v$highestUpgradeAvailable" } function Get-CleanInputKubernetesVersion { <# .DESCRIPTION Cleans the input kubernetes verison .PARAMETER KubernetesVersion KubernetesVersion string to be cleaned. .PARAMETER Semver Semver switch to enforce semver valid output. #> param ( [Parameter(Mandatory=$true)] [String]$KubernetesVersion, [Parameter()] [Switch] $Semver ) $splitVersion = $KubernetesVersion.Split("-") if ($Semver.IsPresent) { $cleanVersion = $splitVersion[0] -replace '[v]','' } else { $cleanVersion = $splitVersion[0] } return $cleanVersion } function Confirm-Configuration { <# .DESCRIPTION Validates the configuration .PARAMETER useStagingShare Requests a staging share to be used for downloading binaries and images (for private testing) .PARAMETER stagingShare The staging share endpoint to use when useStagingShare is requested .PARAMETER vnet Vnet to be validated, if provided .PARAMETER vipPool vipPool to be validated, if provided #> param ( [Switch] $useStagingShare, [String] $stagingShare, [VirtualNetwork] $vnet, [VipPoolSettings] $vipPool, [Switch] $useNetWorkController ) if ($useStagingShare.IsPresent -and [string]::IsNullOrWhiteSpace($stagingShare)) { throw [CustomException]::new($($GenericLocMessage.generic_staging_share_unspecified), ([ErrorTypes]::IsUserErrorFlag)) } # Test networking on hosts if ($vnet) { # Commenting this to unblock customers and proper fix be made for Nov # Confirm-Vnet -vnet $vnet } $isVnetVipPoolPresent = $vnet -ne $null -and ((-not [string]::IsNullOrEmpty($vnet.vipPoolStart)) -or (-not [string]::IsNullOrEmpty($vnet.vipPoolEnd))) if ($useNetworkController.IsPresent -and $isVnetVipPoolPresent) { # SDN does not support vnet vip pool. throw [CustomException]::new($($AksHciLocMessage.akshci_networkcontroller_vnet_vippool_notsupported), ([ErrorTypes]::IsUserErrorFlag)) } if ($isVnetVipPoolPresent -and ($vipPool -ne $null)) { # Vnet and global vip pools cannot be specified together. throw [CustomException]::new($($AksHciLocMessage.akshci_vnet_vippool_and_global_vippool_present), ([ErrorTypes]::IsUserErrorFlag)) } if ((-not $isVnetVipPoolPresent) -and ($vipPool -eq $null)) { # Either a vnet or global vip pool must be configured throw [CustomException]::new($($AksHciLocMessage.akshci_no_vippools_configured), ([ErrorTypes]::IsUserErrorFlag)) } } function Set-AksHciRegistration { <# .DESCRIPTION Register an AksHci with Azure. Calls Connect-AzAccount under the covers. .PARAMETER SubscriptionId SubscriptionId is an azure subscription id. .PARAMETER TenantId TenantId is an azure tenant id. .PARAMETER ArmAccessToken ArmAccessToken is the token for accessing arm. .PARAMETER GraphAccessToken GraphAccessToken is the token for accessing the graph. .PARAMETER AccountId AccountId is an azure account id. .PARAMETER EnvironmentName EnvironmentName is the intented public cloud. .PARAMETER Credential Credential is a PSCredential holding a user's Service Principal. .PARAMETER ResourceGroupName ResourceGroupName is the name of the azure resource group to place arc resources. .PARAMETER Region Region is the name of the azure resource group to place arc resources. .PARAMETER UseDeviceAuthentication UseDeviceAuthentication outputs a code to be used in the browser. .PARAMETER SkipLogin SkipLogin skips the Connect-AzAccount call. Useful in automation or when running from a connected shell. .PARAMETER activity Activity name to use when updating progress. .PARAMETER skipValidationCheck Skips running validation checks if the flag is passed. .PARAMETER skipAzureValidationCheck Skips running validation checks if the flag is passed. #> param( [Parameter(Mandatory = $true)] [string] $SubscriptionId, [Parameter(Mandatory = $false)] [string] $TenantId, [Parameter(Mandatory = $false)] [string] $ArmAccessToken, [Parameter(Mandatory = $false)] [string] $GraphAccessToken, [Parameter(Mandatory = $false)] [string] $AccountId, [Parameter(Mandatory = $false)] [string] $EnvironmentName = $global:azureCloud, [Parameter(Mandatory = $true)] [string] $ResourceGroupName, [Parameter(Mandatory = $false)] [string] $Region, [Parameter(Mandatory = $false)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [Switch] $UseDeviceAuthentication, [Parameter(Mandatory = $false)] [Switch] $SkipLogin, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter()] [Switch] $skipValidationCheck ) $startCmdletTime = Get-Date $cmdletParams = @{ subscriptionId = $SubscriptionId; region = $Region; ResourceGroupName= $ResourceGroupName; UseDeviceAuthentication= $UseDeviceAuthentication; SkipLogin= $SkipLogin; skipValidationCheck= $skipValidationCheck; skipAzureValidationCheck= $skipAzureValidationCheck; } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck if (-not $SkipLogin.IsPresent) { Write-Host "`n`n" #Needed for output formatting Set-AzureLogin -SubscriptionId $SubscriptionId -TenantId $TenantId -ArmAccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId -EnvironmentName $EnvironmentName -Credential $Credential -UseDeviceAuthentication:$UseDeviceAuthentication.IsPresent } $kubernetesProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes $kubernetesConfigProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration # The RPs should always exist but just in case arm is down, bail out. if (($null -eq $kubernetesProvider) -or ($null -eq $kubernetesProvider)) { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ChildStageName "KubernetesProvidersNotExists" ` -ErrorMessage "Unable to check registered Resource Providers" ` -StartCmdletTime $startCmdletTime throw [CustomException]::new(($($AksHciLocMessage.akshci_resource_provider_err)), ([ErrorTypes]::IsErrorFlag)) } if (($kubernetesProvider[0].RegistrationState -ne "Registered") -or ($kubernetesConfigProvider[0].RegistrationState -ne "Registered")) { Write-Status -moduleName $moduleName -Verbose -msg " Kubernetes Resource Providers are not registered for the current logged in tenant. Please run the following commands. With the azure cli: az provider register --namespace Microsoft.Kubernetes az provider register --namespace Microsoft.KubernetesConfiguration With Azure Powershell: Register-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes Register-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration Registration is an asynchronous process and may take approximately 10 minutes. You can monitor the registration process with the following commands: With the azure cli: az provider show -n Microsoft.Kubernetes -o table az provider show -n Microsoft.KubernetesConfiguration -o table With Azure Powershell: Get-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes Get-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration " throw [CustomException]::new(($($AksHciLocMessage.akshci_k8s_err)), ([ErrorTypes]::IsErrorFlag)) } if ($Region -eq "") { $rg = Get-AzResourceGroup -Name $ResourceGroupName $Region = $rg.Location.ToLower().replace(' ', '') } $isValidLocation = $false # in the case of an invalid location, build a string of all the locations to return to the user. $locationErrorString = "" foreach($location in $kubernetesProvider.Locations) { $cleanLocation = $location.ToLower().replace(' ', '') if ($Region -eq $cleanLocation) { $isValidLocation = $true } $locationErrorString += "$cleanLocation," } if (-not $isValidLocation) { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ChildStageName "RegionNotValid" ` -ErrorMessage "$Region is not a valid Region for AksHci" ` -StartCmdletTime $startCmdletTime throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_invalid_region, $Region, $locationErrorString))), ([ErrorTypes]::IsUserErrorFlag)) } Set-KvaRegistration -azureResourceGroup $ResourceGroupName -azureLocation $Region -runValidationCheck:(!($skipValidationCheck.IsPresent)) Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciRegistration { <# .DESCRIPTION Gets the Registration for AksHci. #> $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -StartCmdletTime $startCmdletTime throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime return Get-KvaRegistration } function Set-AzureLogin { <# .DESCRIPTION Performs an Azure Login. Calls Connect-AzAccount under the covers. .PARAMETER SubscriptionId SubscriptionId .PARAMETER TenantId TenantId .PARAMETER ArmAccessToken ArmAccessToken .PARAMETER GraphAccessToken GraphAccessToken .PARAMETER AccountId AccountId .PARAMETER EnvironmentName EnvironmentName .PARAMETER Credential Credential .PARAMETER UseDeviceAuthentication UseDeviceAuthentication .PARAMETER activity Activity name to use when updating progress #> param( [Parameter(Mandatory = $true)] [string] $SubscriptionId, [Parameter(Mandatory = $false)] [string] $TenantId, [Parameter(Mandatory = $false)] [string] $ArmAccessToken, [Parameter(Mandatory = $false)] [string] $GraphAccessToken, [Parameter(Mandatory = $false)] [string] $AccountId, [Parameter(Mandatory = $false)] [string] $EnvironmentName, [Parameter(Mandatory = $false)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [Switch] $UseDeviceAuthentication, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) if($EnvironmentName -eq $AzurePPE) { Add-AzEnvironment -Name $AzurePPE -PublishSettingsFileUrl "https://windows.azure-test.net/publishsettings/index" -ServiceEndpoint "https://management-preview.core.windows-int.net/" -ManagementPortalUrl "https://windows.azure-test.net/" -ActiveDirectoryEndpoint "https://login.windows-ppe.net/" -ActiveDirectoryServiceEndpointResourceId "https://management.core.windows.net/" -ResourceManagerEndpoint "https://api-dogfood.resources.windows-int.net/" -GalleryEndpoint "https://df.gallery.azure-test.net/" -GraphEndpoint "https://graph.ppe.windows.net/" -GraphAudience "https://graph.ppe.windows.net/" | Out-Null } Disconnect-AzAccount | Out-Null if($null -ne $Credential) { if ([string]::IsNullOrEmpty($TenantId)) { throw [CustomException]::new(($($AksHciLocMessage.akshci_empty_tenantid)), ([ErrorTypes]::IsUserErrorFlag)) } else { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -Credential $Credential -ServicePrincipal | Out-Null } } elseif([string]::IsNullOrEmpty($ArmAccessToken) -or [string]::IsNullOrEmpty($GraphAccessToken) -or [string]::IsNullOrEmpty($AccountId)) { # Interactive login $IsIEPresent = Test-Path "$env:SystemRoot\System32\ieframe.dll" if([string]::IsNullOrEmpty($TenantId)) { if($IsIEPresent -and (-not $UseDeviceAuthentication)) { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId | Out-Null } else # Use -UseDeviceAuthentication as IE Frame is not available to show Azure login popup { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId -UseDeviceAuthentication | Out-Null } } else { if($IsIEPresent -and (-not $UseDeviceAuthentication)) { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId | Out-Null } else # Use -UseDeviceAuthentication as IE Frame is not available to show Azure login popup { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -UseDeviceAuthentication | Out-Null } } } else { # Not an interactive login if([string]::IsNullOrEmpty($TenantId)) { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId -AccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId | Out-Null } else { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -AccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId | Out-Null } } } function New-AksHciStorageContainer { <# .DESCRIPTION Creates a new cloud storage container .PARAMETER activity Activity name to use when updating progress .PARAMETER Name The name of the new storage container .PARAMETER Path The path where the vhds will be stored #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidDirectoryPath -dirPath $_})] [String]$Path ) $startCmdletTime = Get-Date $cmdletParams = @{StorageContainerName = $Name} trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_new_storage_container) -moduleName $moduleName $cloudLocation = (Get-MocConfig)["cloudLocation"] New-MocContainer -name $Name -path $Path -location $cloudLocation Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_storage_container_created) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Get-AksHciStorageContainer { <# .DESCRIPTION Gets the storage containers .PARAMETER activity Activity name to use when updating progress .PARAMETER Name The name of the storage container, if not present returns all #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter()] [String]$Name ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $Name} trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_storage_container_info) $cloudLocation = (Get-MocConfig)["cloudLocation"] $result = Get-MocContainer -name $Name -location $cloudLocation Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys return $result } function Install-AksHciCsiSmb { <# .DESCRIPTION Installs csi smb plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_install_csi_smb_plugin) Set-KvaCsiSmb -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_smb_plugin_installed) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Install-AksHciCsiNfs { <# .DESCRIPTION Installs csi nfs plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_install_csi_nfs_plugin) Set-KvaCsiNfs -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_nfs_plugin_installed) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Uninstall-AksHciCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_uninstall_csi_smb_plugin) Reset-KvaCsiSmb -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_smb_plugin_uninstalled) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Uninstall-AksHciCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date $cmdletParams = @{ClusterName= $ClusterName} if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -ErrorMessage $_ -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_uninstall_csi_nfs_plugin) Reset-KvaCsiNfs -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_nfs_plugin_uninstalled) Uninitialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -CmdletParameters $cmdletParams -StartCmdletTime $startCmdletTime -BoundParameterKeys $PSBoundParameters.Keys } function Confirm-Vnet { <# .DESCRIPTION Validate if the vnet configuration is valid #> param( [VirtualNetwork] $vnet ) $isMultinode = Test-MultiNodeDeployment $vSwitchName = $vnet.VswitchName $vipPoolStart = $vnet.VipPoolStart $vipPoolEnd = $vnet.VipPoolEnd if ($isMultinode) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Test-VipPoolAgainstVnicAddressPrefix -switchName $vSwitchName -multiNode -nodeName $_.Name -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd -vlanID $vnet.Vlanid } } else { Test-VipPoolAgainstVnicAddressPrefix -switchName $vSwitchName -nodeName ($env:computername) -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd -vlanID $vnet.Vlanid } } function Get-AksHciReleaseKubernetesVersion { <# .DESCRIPTION Show all the kubernetes versions which will be downloaded in offline download scenario. .PARAMETER activity Activity name to use when updating progress .PARAMETER catalog Catalog name to use .PARAMETER ring Ring name to use .PARAMETER mode Different modes for choosing different Linux kubernetes versions to download. #> [CmdletBinding()] param( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = $script:catalogName, [parameter(DontShow)] [String] $ring = $script:ringName, [Parameter()] [OfflineDownloadMode] $mode = "full" ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } $startVersion = (Get-AksHciVersion) $releaseVersions = @{} $updateReleases = Get-ProductReleasesUptoVersion -Version $startVersion -moduleName $moduleName $updateReleases | ForEach-Object { $version = $_.Version $result = @() if ([System.Version]$version -ge [System.Version]$startVersion) { $result += $_.CustomData.ManagementNodeImageK8sVersion foreach($releaseStream in $_.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { $vhdInfo = Get-ImageReleaseVhdInfo -release $subProductRelease if ($vhdInfo) { if ($vhdInfo.CustomData.BaseOSImage.OperatingSystem -ieq "Linux") { if($version -ge $global:convergedReleaseVersion){ foreach( $k8sPackage in $subProductRelease.ProductFiles.CustomData.K8sPackages ){ $result += $k8sPackage.Version } } else{ $k8sVersion = $subProductRelease.ProductFiles.CustomData.K8sPackages.Version if ($mode -ieq "full") { $result += $k8sVersion } } } else { if ($mode -ieq "full") { $result += $subProductRelease.ProductFiles.CustomData.BaseOSImage.SKU + "-" + $subProductRelease.ProductFiles.CustomData.BaseOSImage.imageVersion } } } } } $releaseVersions.Add($version, ($result | Sort-Object -Descending | Get-Unique)) } } return $releaseVersions | ConvertTo-Json } function Get-AksHciRelease { <# .DESCRIPTION Download the bits(images,windows-stable,mocstack-stable,kva-stable,client-cred-plugin-stable) for future use .PARAMETER activity Activity name to use when updating progress .PARAMETER catalog Catalog name to use .PARAMETER ring Ring name to use .PARAMETER mode Different modes for choosing different Linux kubernetes versions to download. #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = $script:catalogName, [parameter(DontShow)] [String] $ring = $script:ringName, [Parameter()] [OfflineDownloadMode] $mode = "full" ) $startCmdletTime = Get-Date $cmdletParams = @{catalog= $catalog; ring = $ring; mode= $mode } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams throw $_ } if ([string]::IsNullOrWhiteSpace($global:config[$modulename]["stagingShare"])) { throw $($GenericLocMessage.generic_staging_share_unspecified) } Set-AksHciConfigValue -name "offsiteTransferCompleted" -value $false Set-KvaOfflineOffsiteTransferCompleted -offsiteTransferCompleted $false Set-MocOfflineOffsiteTransferCompleted -offsiteTransferCompleted $false $startVersion = (Get-AksHciVersion) $updateReleases = Get-ProductReleasesUptoVersion -Version $startVersion -moduleName $moduleName $updateReleases | ForEach-Object { $version = $_.Version if ([System.Version]$version -ge [System.Version]$startVersion) { $destination = $global:config[$modulename]["stagingShare"] + "\" + $version if (Test-Path $destination) { Test-OfflineDownloadFiles -destination $destination -version $version } else { New-Item -Path $destination -ItemType Directory Get-ReleaseContent -version $version -activity $activity -destination $destination -moduleName $moduleName -mode $mode } } } $manifestFileName = [String]::Concat($global:config[$moduleName]["catalog"], "-", $global:config[$moduleName]["ring"], ".json") $manifestFile = [io.Path]::Combine($global:config[$modulename]["stagingShare"], $manifestFileName) $catalogManifest = Get-LatestCatalog -moduleName $moduleName $catalogJson = ConvertTo-Json -InputObject $catalogManifest -depth 100 Set-Content -path $manifestFile -value $catalogJson -Confirm:$false Set-AksHciConfigValue -name "offsiteTransferCompleted" -value $true Set-KvaOfflineOffsiteTransferCompleted -offsiteTransferCompleted $true Set-MocOfflineOffsiteTransferCompleted -offsiteTransferCompleted $true Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function Enable-AksHciOfflineDownload { <# .DESCRIPTION Turns on offline downloading. Changes the -offlineDownload flag in Set-AksHciConfig to true. .PARAMETER stagingShare Path that the bits will be downloaded to. .PARAMETER offsiteTransferCompleted Sets deployment to use artifacts downloaded offsite and transfered to deployment server. .PARAMETER activity Activity name to use when updating progress #> param( [Parameter()] [String] $stagingShare, [Parameter()] [bool] $offsiteTransferCompleted = $false, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity if (-not [string]::IsNullOrWhiteSpace($stagingShare)) { if ((-not [string]::IsNullOrWhiteSpace(($global:config[$modulename]["stagingShare"]))) -and ($global:config[$modulename]["stagingShare"] -ne $stagingShare)) { $errorMsg = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_stagingshare_updated, $global:config[$modulename]["stagingShare"], $stagingShare)) Write-Warning $errorMsg } $global:config[$modulename]["stagingShare"] = $stagingShare } if ([string]::IsNullOrWhiteSpace($global:config[$modulename]["stagingShare"])) { throw $($GenericLocMessage.generic_staging_share_unspecified) } Enable-MocOfflineDownload -stagingShare $global:config[$modulename]["stagingShare"] -offsiteTransferCompleted $offsiteTransferCompleted Enable-KvaOfflineDownload -stagingShare $global:config[$modulename]["stagingShare"] -offsiteTransferCompleted $offsiteTransferCompleted Set-AksHciConfigValue -name "offlineDownload" -value $true Set-AksHciConfigValue -name "offsiteTransferCompleted" -value $offsiteTransferCompleted Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -BoundParameterKeys $PSBoundParameters.Keys } function Disable-AksHciOfflineDownload { <# .DESCRIPTION Turns off offline downloading. Changes the -offlineDownload flag in Set-AksHciConfig to false .PARAMETER activity Activity name to use when updating progress #> param( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime Uninitialize-AksHciEnvironment -activity $activity throw $_ } Initialize-AksHciEnvironment -activity $activity Disable-MocOfflineDownload Disable-KvaOfflineDownload Set-AksHciConfigValue -name "offlineDownload" -value $false Uninitialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -BoundParameterKeys $PSBoundParameters.Keys } function Get-DefaultTraceConfigDetails { <# .DESCRIPTION Capture the config map details for the TraceCmdlet module. .PARAMETER catalog catalog name to use .PARAMETER ring ring name to use .PARAMETER deploymentId deploymentId name to use #> param( [parameter(DontShow)] [String] $catalog, [parameter(DontShow)] [String] $ring, [parameter(DontShow)] [String] $deploymentId ) [TraceConfigDetails]$traceConfig = [TraceConfigDetails]::new() $traceConfig.DeploymentId = $deploymentId $traceConfig.Catalog = $catalog $traceConfig.Audience = $ring $traceConfig.ModuleName = $moduleName $traceConfig.ModuleVersion = $moduleVersion return $traceConfig | ConvertTo-Json } function New-AksHciLoadBalancerSetting { <# .DESCRIPTION Create a object for load balancer. .PARAMETER name Name of the LoadBalancer .PARAMETER loadBalancerSku Choice of load balancer for kubernetes service none/haproxy/kubevip/metalb .PARAMETER loadBalancerVMSize Size of load balance VM .PARAMETER loadBalancerCount Number of load Balancer VMs .OUTPUTS LoadBalancer object .EXAMPLE New-AksHciLoadBalancerSetting -Name "lb1" -loadBalancerSku KubeVIP .EXAMPLE New-AksHciLoadBalancerSetting -Name "lb1" -loadBalancerSku HAProxy -loadBalancerVmSize Standard_A4_v2 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [LoadBalancerSku] $LoadBalancerSku, [Parameter(Mandatory=$false)] [Vmsize] $vmSize, [Parameter(Mandatory=$false)] [int] $loadBalancerCount ) $startCmdletTime = Get-Date $cmdletParams = @{Name= $name; LoadBalancerSku= $LoadBalancerSku; vmSize= $vmSize; loadBalancerCount= $loadBalancerCount } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) throw $_ } Confirm-LoadBalancerSettings -loadBalancerSku $LoadBalancerSku -loadBalancerCount $loadBalancerCount if ([string]::IsNullOrWhiteSpace($VmSize)) { $VmSize = $global:defaultLoadBalancerVmSize } if ( ($loadBalancerCount) -and ($loadBalancerCount -ge 1) -and ($LoadBalancerSku -ne [LoadBalancerSku]::HAProxy) ) { Write-Host $($AksHciLocMessage.akshci_scaling_loadbalancer_unsupported_for_non_haproxy_type) throw [CustomException]::new(($($AksHciLocMessage.akshci_scaling_loadbalancer_unsupported_for_non_haproxy_type)), ([ErrorTypes]::IsUserErrorFlag)) } if ( (-not $loadBalancerCount) -or ($loadBalancerCount -eq 0) ) { $loadBalancerCount = 1 } switch ($LoadBalancerSku) { ([LoadBalancerSku]::HAProxy) { #Today, if HAProxy is chosen both APi server and K8s service will use HAProxy $serviceLoadBalancerSku = [LoadBalancerSku]::HAProxy break } ([LoadBalancerSku]::None) { # Bring your LB $serviceLoadBalancerSku = $LoadBalancerSku $LoadBalancerSku = [LoadBalancerSku]::KubeVIP break } ([LoadBalancerSku]::SDNLoadBalancer) { # SDN Loadbalancer $serviceLoadBalancerSku = $LoadBalancerSku #There is no need to create LB VM's as Networkcontroller is already provisioned for LoadBalancing. $loadBalancerCount = 0 break } Default { $serviceLoadBalancerSku = $LoadBalancerSku $LoadBalancerSku = [LoadBalancerSku]::KubeVIP break } } if ($serviceLoadBalancerSku -eq [LoadBalancerSku]::None) { Write-Host "NOTE: None for load balancer implies the user will use their own Load Balancer, it will be user's responsibility to configure External Load Balancer correctly." } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys return [LoadBalancerSettings]::new($Name, $LoadBalancerSku, $serviceLoadBalancerSku, $VmSize, $loadBalancerCount) } function Test-CloudCACertificateNearingExpiry { <# .DESCRIPTION Validate that powershell remoting to a node is working. #> param ( [Parameter(Mandatory=$true)] [int] $expiryThresholdDays = $global:caCertRotationThreshold ) $startCmdletTime = Get-Date $cmdletParams = @{expiryThresholdDays= $expiryThresholdDays} trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) throw $_ } $result = Get-MocCertificate -name $global:cloudAgentCACertName -expiryDays $expiryThresholdDays | ConvertFrom-Json Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys if ($result.tags[0].NearingExpiry -eq "true") { return $true } return $false } function Set-AksHciLoadBalancer { <# .DESCRIPTION Updates a load balancer object specifically, the number of replicas .PARAMETER clusterName Name of the cluster hosting the loadblancer .PARAMETER loadBalancerVMSize Size of load balance VM .PARAMETER loadBalancerCount Number of load balancer replicas .EXAMPLE Set-AksHciLoadBalancer -clusterName "cluster1" -loadBalancerCount 2 #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [string] $clusterName, [Parameter(Mandatory=$false)] [int] $loadBalancerCount, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date $cmdletParams = @{clusterName= $clusterName; loadBalancerCount= $loadBalancerCount} trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError ` -BoundParameterKeys $PSBoundParameters.Keys ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } $mgmtCluster = (Get-KvaConfig)["kvaName"] if ($clusterName -ieq $mgmtCluster) { throw [CustomException]::new($($AksHciLocMessage.akshci_scaling_unsupported), ([ErrorTypes]::IsUserErrorFlag)) } Set-KvaLoadBalancer -clusterName $clusterName -loadBalancerCount $loadBalancerCount -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $cmdletParams ` -BoundParameterKeys $PSBoundParameters.Keys } function New-AksHciVipPoolSetting { <# .DESCRIPTION .PARAMETER name Name of the VIP pool .PARAMETER vipPoolStart Start of the vipPool .PARAMETER vipPoolEnd End of the vipPool .EXAMPLE New-AksHciVipPoolSetting -name "Pool" -vipPoolStart "20.0.0.100" -vipPoolEnd "20.0.0.110" #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNetworkName -Name $_})] [string] $name, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $vipPoolStart, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidIpv4Address -ipv4 $_})] [String] $vipPoolEnd ) return New-VipPoolSettings -name $name -vipPoolStart $vipPoolStart -vipPoolEnd $vipPoolEnd } function Confirm-LoadBalancerSettings { <# .DESCRIPTION Validates loadbalancer settings. .PARAMETER loadBalancerSku Choice of load balancer for kubernetes service none/haproxy/kubevip/metalb .PARAMETER loadBalancerCount Number of load Balancer VMs #> param ( [Parameter(Mandatory=$true)] [LoadBalancerSku] $loadBalancerSku, [Parameter(Mandatory=$false)] [int] $loadBalancerCount ) if ( ($loadBalancerCount) -and ($loadBalancerCount -ge 1) -and ($loadBalancerSku -ne [LoadBalancerSku]::HAProxy) ) { Write-Host $($AksHciLocMessage.akshci_scaling_loadbalancer_unsupported_for_non_haproxy_type) throw [CustomException]::new($($AksHciLocMessage.akshci_scaling_loadbalancer_unsupported_for_non_haproxy_type), ([ErrorTypes]::IsUserErrorFlag)) } # Error check: SDNLoadBalancer is only used with SDN $isSDNConfigured = (Get-MocConfig)["useNetworkController"] if ($isSDNConfigured -and ($loadBalancerSku -ne [LoadBalancerSku]::SDNLoadBalancer)) { Write-Host $($AksHciLocMessage.akshci_only_sdn_loadbalancer_supported_for_sdn) throw [CustomException]::new($($AksHciLocMessage.akshci_only_sdn_loadbalancer_supported_for_sdn), ([ErrorTypes]::IsUserErrorFlag)) } if ((-not $isSDNConfigured) -and $loadBalancerSku -eq [LoadBalancerSku]::SDNLoadBalancer) { Write-Host $($AksHciLocMessage.akshci_sdn_loadbalancer_unsupported_without_sdn) throw [CustomException]::new($($AksHciLocMessage.akshci_sdn_loadbalancer_unsupported_without_sdn), ([ErrorTypes]::IsUserErrorFlag)) } } function Confirm-ClusterVipPoolConfiguration { <# .DESCRIPTION Validates the vippool configuration for the cluster #> param( [string] $vipPoolName ) if (-not [string]::IsNullOrEmpty($vipPoolName)) { # Only a single vippool is supported at this time and the vippool parameter should match the default global vippool configured in MOC. $globalVipPool = (Get-MocConfig)["defaultvippoolname"] if ($vipPoolName -ne $globalVipPool) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_vippool_doestnot_match_global_pool, $vipPoolName, $globalVipPool)), ([ErrorTypes]::IsUserErrorFlag)) } return } } function New-AksHciMocCNISetting { <# .SYNOPSIS Create a new MOC CNI setting .DESCRIPTION Create a new MOC CNI to be attached to the target cluster. Creates an in memory object for CNI settings. Upon cluster creation the object is persisted in the cluster configuration. .PARAMETER name Name of the CNIA descriptive name for the settings object .PARAMETER network The cluster network to apply the MOC CNI settings. The switch that the network is attached to must fulfil the requirements for the attached network. i.e. if 'enableSRIOV' is set, the switch must have been created with '-sriov $true' value set. .PARAMETER enableDpdk Whether to enable DPDK .PARAMETER enableSRIOV Whether to enable SR-IOV .EXAMPLE New-AksHciMocCNISetting -name "CNISetting" -network $vnet -enableDpdk $true -enableSriov $true #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [virtualNetwork] $network, [Parameter(Mandatory=$false)] [bool] $enableDpdk = $false, [Parameter(Mandatory=$false)] [bool] $enableSriov = $false ) $startCmdletTime = Get-Date $cmdletParams = @{name= $name; enableDpdk= $enableDpdk; enableSriov= $enableSriov; networkName= $network.Name; networkVswitchName= $network.VswitchName } trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) throw $_ } Write-Host $($AksHciLocMessage.akshci_preview_feature_new_akshci_moc_cni_setting) [SecondaryNetworkPlugin] $plugin = [SecondaryNetworkPlugin]::new($name, $network, "moc", $enableDpdk, $enableSriov) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys return $plugin } function New-AksHciClusterLinuxOsConfig { <# .SYNOPSIS Create a new Linux O/S configuration object .DESCRIPTION Create a new Linux O/S configuration object that can be used in New-AksHciCluster. Creates an in memory object for Linux O/S configuration settings. Upon cluster creation the object is persisted in the cluster configuration. .PARAMETER name Name of the CNIA descriptive name for the configuration object .PARAMETER hugePages2M The number of 2MB hugepages to support. Cannot be set if hugePages1G is also set. .PARAMETER hugePages1G The number of 2MB hugepages to support. Cannot be set if hugePages2M is also set. .EXAMPLE New-LinuxOsConfig -name "2hugepages1g" -hugePages1G 2 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$false)] [int] $hugePages2M = 0, [Parameter(Mandatory=$false)] [int] $hugePages1G = 0 ) $startCmdletTime = Get-Date $cmdletParams = @{name= $name; hugePages2M=$hugePages2M; hugePages1G=$hugePages1G} trap { Trace-CmdletError -BoundParameterKeys $PSBoundParameters.Keys -ErrorMessage $_ ` -CmdletParameters $cmdletParams ` -StartCmdletTime $startCmdletTime ` -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) } Write-Host $($AksHciLocMessage.akshci_preview_feature_linux_os_config) [LinuxOsConfig] $linuxOsConfig = [LinuxOsConfig]::new($name, $hugePages2M, $hugePages1G) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails -moduleName $moduleName) -StartCmdletTime $startCmdletTime -CmdletParameters $cmdletParams -BoundParameterKeys $PSBoundParameters.Keys return $linuxOsConfig } function Test-Command-Runnable { <# .DESCRIPTION For the given current command, it will check - if the given installedState is match - and check if there is an active commands given in the cannotRunWithCommands parameter #> param( [Parameter(Mandatory=$true)] [String] $currentCommand, [Parameter(Mandatory=$false)] [InstallState] $cannotRunWithInstallState, [Parameter(Mandatory=$true)] [String] $cannotRunWithCommand, [Parameter(Mandatory=$false)] [String] $cannotRunWithCommandRegex ) if ([string]::IsNullOrEmpty($cannotRunWithCommandRegex)) { $cannotRunWithCommandRegex = $cannotRunWithCommand } $curState = Get-InstallState -module $moduleName if (($cannotRunWithInstallState -ne $null) -and ($curState -ne $cannotRunWithInstallState)) { return $false } $isConflictingCommandsRunning = $false $commands = Get-RunningPSCommands -Name $cannotRunWithCommandRegex -moduleName $moduleName Foreach ($command in $commands) { if ($command.processId -eq [System.Diagnostics.Process]::GetCurrentProcess().Id) { # skip current powershell process continue; } # there is a powershell running command specified in cannotRunWithCommand $isConflictingCommandsRunning = $true Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_already_running_command, $cannotRunWithCommand, $command.processId, $command.computerName)) Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_already_running_command_override, $currentCommand, $cannotRunWithCommand, $command.processId)) } # If $curState is equal to $cannotRunWithInstallState # And there are no conflicting commands running # it means, it is stuck in that state, we can proceed with the requested command # eg: While running Update-Akshci, user forcefully killed that process/powershell # installState is stuck in Updating state. # Again, user is trying to Update-Akshci # in this case, cannotRunWithInstallState is Updating (two update should not happen in parallel) # But, if there are no active Update-Akshci commands, probably user forcefully killed Update-Akshci # and made installState stuck in Updating state. So we can proceed with Update-Akshci if (($cannotRunWithInstallState -eq $curState) -and (!$isConflictingCommandsRunning)) { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_installstate_hung_and_proceeding, $curState, $currentCommand)) } return $isConflictingCommandsRunning } function Get-MinSupportedMocVersion { <# .DESCRIPTION Gets the minimum supported MOC version for specified version of AksHci .PARAMETER aksHciVersion The target version of AksHci to inspect for minimum compatible version of MOC .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidVersionNumber -VersionNumber $_})] [String]$aksHciVersion, [String]$activity = $MyInvocation.MyCommand.Name ) $minMocVersion = $null $productRelease = Get-ProductRelease -version $aksHciVersion -module $moduleName foreach($productStream in $productRelease.ProductStreamRefs) { if ($productStream.ProductReleases[0].ProductName -eq $global:mocProductName) { $minMocVersion = $productStream.ProductReleases[0].Version } } return $minMocVersion } function Test-AksHciSupportedMocVersion { <# .DESCRIPTION Test if the installed MOC version is at least the minimum compatible version. If installed Moc is incompatible, throws an error. If no minimum compatible version is defined, return true. .PARAMETER aksHciVersion The target version of AksHci to inspect for minimum compatible version of MOC .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidVersionNumber -VersionNumber $_})] [String]$aksHciVersion, [String]$activity = $MyInvocation.MyCommand.Name ) $minMocVersion = Get-MinSupportedMocVersion -aksHciVersion $aksHciVersion if ($minMocVersion) { $currentMocState = Get-InstallState -module $global:MocModule if ($currentMocState -eq [InstallState]::Installed) { $currMocVersion = $(Get-MocVersion) if ([version]$currMocVersion -ge [version]$minMocVersion) { return $true } throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_moc_version_not_supported, $currMocVersion, $minMocVersion)), ([ErrorTypes]::IsInfraErrorFlag)) } else { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_moc_not_installed, $minMocVersion)), ([ErrorTypes]::IsInfraErrorFlag)) } } return $true } function Test-OfflineDownloadFiles { <# .DESCRIPTION Check if there is any missing bits which is needed for akshci installation or upgrade in offline download scenario for specified version. If there is any missing files, delete version path and redownload all files, else skip this version. #> param( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$true)] [String]$version, [Parameter(Mandatory=$true)] [String]$destination ) $releaseFiles = Get-ReleaseFiles -version $version $downloadedFiles = Get-ChildItem -Recurse $destination -Exclude *.json $missingFiles = Compare-Object -ReferenceObject $releaseFiles.Name -DifferenceObject $downloadedFiles.Name if ($missingFiles.length -eq 0) { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_offlinedownload_path_exist, $destination, $version)) } else { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_offlinedownload_path_exist_but_missing_files, $destination, $version)) Remove-Item $destination New-Item -Path $destination -ItemType Directory Get-ReleaseContent -version $version -activity $activity -destination $destination -moduleName $moduleName } } function Get-ReleaseFiles { <# .DESCRIPTION Gets the list of release files for the specified Version #> param( [Parameter(Mandatory=$true)] [String]$version ) $releaseFiles = @() $productRelease = Get-ProductRelease -version $version -moduleName $moduleName $productRelease.ProductStreamRefs | ForEach-Object { $releaseFiles += $_.ProductReleases[0].ProductFiles | Where-Object {$_.Name -ne "manifest.cab"} } return $releaseFiles } function Test-PrivatePreviewSku { <# .DESCRIPTION Returns true if the SKU is in private preview #> param( [Parameter(Mandatory=$true)] [VmSize] $VMSize ) if ($VMSize -eq [VmSize]::Standard_NC4_A2 -or $VMSize -eq [VmSize]::Standard_NC8_A2 -or $VMSize -eq [VmSize]::Standard_NC16_A2 -or $VMSize -eq [VmSize]::Standard_NC32_A2 -or $VMSize -eq [VmSize]::Standard_NC4_A30 -or $VMSize -eq [VmSize]::Standard_NC8_A30 -or $VMSize -eq [VmSize]::Standard_NC16_A30 -or $VMSize -eq [VmSize]::Standard_NC32_A30) { return $true } return $false } #endregion # SIG # Begin signature block # MIIoRgYJKoZIhvcNAQcCoIIoNzCCKDMCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCAasSO5ajxX7MVo # qa0Ki5jW1jAh/5aJjMXEBcTkKokWGKCCDXYwggX0MIID3KADAgECAhMzAAAEBGx0 # Bv9XKydyAAAAAAQEMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjQwOTEyMjAxMTE0WhcNMjUwOTExMjAxMTE0WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQC0KDfaY50MDqsEGdlIzDHBd6CqIMRQWW9Af1LHDDTuFjfDsvna0nEuDSYJmNyz # NB10jpbg0lhvkT1AzfX2TLITSXwS8D+mBzGCWMM/wTpciWBV/pbjSazbzoKvRrNo # DV/u9omOM2Eawyo5JJJdNkM2d8qzkQ0bRuRd4HarmGunSouyb9NY7egWN5E5lUc3 # a2AROzAdHdYpObpCOdeAY2P5XqtJkk79aROpzw16wCjdSn8qMzCBzR7rvH2WVkvF # HLIxZQET1yhPb6lRmpgBQNnzidHV2Ocxjc8wNiIDzgbDkmlx54QPfw7RwQi8p1fy # 4byhBrTjv568x8NGv3gwb0RbAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQU8huhNbETDU+ZWllL4DNMPCijEU4w # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMjkyMzAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAIjmD9IpQVvfB1QehvpC # Ge7QeTQkKQ7j3bmDMjwSqFL4ri6ae9IFTdpywn5smmtSIyKYDn3/nHtaEn0X1NBj # L5oP0BjAy1sqxD+uy35B+V8wv5GrxhMDJP8l2QjLtH/UglSTIhLqyt8bUAqVfyfp # h4COMRvwwjTvChtCnUXXACuCXYHWalOoc0OU2oGN+mPJIJJxaNQc1sjBsMbGIWv3 # cmgSHkCEmrMv7yaidpePt6V+yPMik+eXw3IfZ5eNOiNgL1rZzgSJfTnvUqiaEQ0X # dG1HbkDv9fv6CTq6m4Ty3IzLiwGSXYxRIXTxT4TYs5VxHy2uFjFXWVSL0J2ARTYL # E4Oyl1wXDF1PX4bxg1yDMfKPHcE1Ijic5lx1KdK1SkaEJdto4hd++05J9Bf9TAmi # u6EK6C9Oe5vRadroJCK26uCUI4zIjL/qG7mswW+qT0CW0gnR9JHkXCWNbo8ccMk1 # sJatmRoSAifbgzaYbUz8+lv+IXy5GFuAmLnNbGjacB3IMGpa+lbFgih57/fIhamq # 5VhxgaEmn/UjWyr+cPiAFWuTVIpfsOjbEAww75wURNM1Imp9NJKye1O24EspEHmb # DmqCUcq7NqkOKIG4PVm3hDDED/WQpzJDkvu4FrIbvyTGVU01vKsg4UfcdiZ0fQ+/ # V0hf8yrtq9CkB8iIuk5bBxuPMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGiYwghoiAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAQEbHQG/1crJ3IAAAAABAQwDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIN6FHl7bZs+6Wuy8TZdIz+tz # w40qY0FRjfM2pqTE/75vMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAjHWyPcCDoIlMZp1xwahAGxOqqiPUNvcerDZN5n0HaytsiiEXaeI8QIOH # asitCqssSDcY+KUM3FoAeFdSezk+Sbaj3yJTlpPLV9nBRezbPq5wMpmG/K0npRys # rtVjuBUyV0rn+vMehCBZwLHgRP/yGq57n+XGUaB3IiVITl6lVtbiuNUvMt7EsC4P # 1Rh//5HCaT0lEwzNfvV5a31WA8jfA0BwIWQcWfY5RrCp5UYybA63EuGK3eG5pSPo # Jqphtz3a/hfV20yXnlQxnxII22M7DbDi9TETcfvJuYQ6hKP30l+VVfUi7ng0beE3 # 0zQ0YXSx+RE/gFqppNRoWETGYJAxm6GCF7AwghesBgorBgEEAYI3AwMBMYIXnDCC # F5gGCSqGSIb3DQEHAqCCF4kwgheFAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFaBgsq # hkiG9w0BCRABBKCCAUkEggFFMIIBQQIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCDgiJHzNYj5R1TK0QL3+nSOZNrss7fVWxVWSBxMod/POgIGZusiu9bE # GBMyMDI0MTEwNzE3NDEwOS4zNTlaMASAAgH0oIHZpIHWMIHTMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT # TjozNjA1LTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg # U2VydmljZaCCEf4wggcoMIIFEKADAgECAhMzAAAB91ggdQTK+8L0AAEAAAH3MA0G # CSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMB4XDTI0 # MDcyNTE4MzEwNloXDTI1MTAyMjE4MzEwNlowgdMxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9w # ZXJhdGlvbnMgTGltaXRlZDEnMCUGA1UECxMeblNoaWVsZCBUU1MgRVNOOjM2MDUt # MDVFMC1EOTQ3MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNl # MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0OdHTBNom6/uXKaEKP9r # PITkT6QxF11tjzB0Nk1byDpPrFTHha3hxwSdTcr8Y0a3k6EQlwqy6ROz42e0R5eD # W+dCoQapipDIFUOYp3oNuqwX/xepATEkY17MyXFx6rQW2NcWUJW3Qo2AuJ0HOtbl # SpItQZPGmHnGqkt/DB45Fwxk6VoSvxNcQKhKETkuzrt8U6DRccQm1FdhmPKgDzgc # fDPM5o+GnzbiMu6y069A4EHmLMmkecSkVvBmcZ8VnzFHTDkGLdpnDV5FXjVObAgb # SM0cnqYSGfRp7VGHBRqyoscvR4bcQ+CV9pDjbJ6S5rZn1uA8hRhj09Hs33HRevt4 # oWAVYGItgEsG+BrCYbpgWMDEIVnAgPZEiPAaI8wBGemE4feEkuz7TAwgkRBcUzLg # Q4uvPqRD1A+Jkt26+pDqWYSn0MA8j0zacQk9q/AvciPXD9It2ez+mqEzgFRRsJGL # tcf9HksvK8Jsd6I5zFShlqi5bpzf1Y4NOiNOh5QwW1pIvA5irlal7qFhkAeeeZqm # op8+uNxZXxFCQG3R3s5pXW89FiCh9rmXrVqOCwgcXFIJQAQkllKsI+UJqGq9rmRA # BJz5lHKTFYmFwcM52KWWjNx3z6odwz2h+sxaxewToe9GqtDx3/aU+yqNRcB8w0tS # XUf+ylN4uk5xHEpLpx+ZNNsCAwEAAaOCAUkwggFFMB0GA1UdDgQWBBTfRqQzP3m9 # PZWuLf1p8/meFfkmmDAfBgNVHSMEGDAWgBSfpxVdAF5iXYP05dJlpxtTNRnpcjBf # BgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3Bz # L2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcmww # bAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRwOi8vd3d3Lm1pY3Jvc29m # dC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0El # MjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBYGA1UdJQEB/wQMMAoGCCsGAQUF # BwMIMA4GA1UdDwEB/wQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAgEAN0ajafILeL6S # QIMIMAXM1Qd6xaoci2mOrpR8vKWyyTsL3b83A7XGLiAbQxTrqnXvVWWeNst5YQD8 # saO+UTgOLJdTdfUADhLXoK+RlwjfndimIJT9MH9tUYXLzJXKhZM09ouPwNsrn8YO # LIpdAi5TPyN8Cl11OGZSlP9r8JnvomW00AoJ4Pl9rlg0G5lcQknAXqHa9nQdWp1Z # xXqNd+0JsKmlR8tcANX33ClM9NnaClJExLQHiKeHUUWtqyLMl65TW6wRM7XlF7Y+ # PTnC8duNWn4uLng+ON/Z39GO6qBj7IEZxoq4o3avEh9ba43UU6TgzVZaBm8VaA0w # SwUe/pqpTOYFWN62XL3gl/JC2pzfIPxP66XfRLIxafjBVXm8KVDn2cML9IvRK02s # 941Y5+RR4gSAOhLiQQ6A03VNRup+spMa0k+XTPAi+2aMH5xa1Zjb/K8u9f9M05U0 # /bUMJXJDP++ysWpJbVRDiHG7szaca+r3HiUPjQJyQl2NiOcYTGV/DcLrLCBK2zG5 # 03FGb04N5Kf10XgAwFaXlod5B9eKh95PnXKx2LNBgLwG85anlhhGxxBQ5mFsJGkB # n0PZPtAzZyfr96qxzpp2pH9DJJcjKCDrMmZziXazpa5VVN36CO1kDU4ABkSYTXOM # 8RmJXuQm7mUF3bWmj+hjAJb4pz6hT5UwggdxMIIFWaADAgECAhMzAAAAFcXna54C # m0mZAAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UE # CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z # b2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZp # Y2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMy # MjVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQH # EwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNV # BAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0B # AQEFAAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51 # yMo1V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY # 6GB9alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9 # cmmvHaus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN # 7928jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDua # Rr3tpK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74 # kpEeHT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2 # K26oElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5 # TI4CvEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZk # i1ugpoMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9Q # BXpsxREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3Pmri # Lq0CAwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUC # BBYEFCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJl # pxtTNRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9y # eS5odG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUA # YgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU # 1fZWy4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2Ny # bC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIw # MTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0w # Ni0yMy5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/yp # b+pcFLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulm # ZzpTTd2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM # 9W0jVOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECW # OKz3+SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4 # FOmRsqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3Uw # xTSwethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPX # fx5bRAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVX # VAmxaQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGC # onsXHRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU # 5nR0W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEG # ahC0HVUzWLOhcGbyoYIDWTCCAkECAQEwggEBoYHZpIHWMIHTMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJzAlBgNVBAsTHm5TaGllbGQgVFNTIEVT # TjozNjA1LTA1RTAtRDk0NzElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAg # U2VydmljZaIjCgEBMAcGBSsOAwIaAxUAb28KDG/xXbNBjmM7/nqw3bgrEOaggYMw # gYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYD # VQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQsF # AAIFAOrW4CYwIhgPMjAyNDExMDcwNjQzNTBaGA8yMDI0MTEwODA2NDM1MFowdzA9 # BgorBgEEAYRZCgQBMS8wLTAKAgUA6tbgJgIBADAKAgEAAgIQAwIB/zAHAgEAAgIT # ODAKAgUA6tgxpgIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAow # CAIBAAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBCwUAA4IBAQBQFLMIuI49 # uXsmjOgz7uM/y7ujLs7lMPSR2ZIbTKKwf5hbi5EdsxLKKKlX3heOD3gOcybzXPo3 # JUbTlJVlLW8n1o/IUx4L9GzZK2vCSrGdEzdwMVsgR6fyxrk8GTv1in4IDxCnnZ8N # RJnnTC7B8POAnCxLuLc7MTpJofjRoQGUxL1W6DvlPjOsEMGKNNvhRT5HqTjHaRYc # zNjul75SRlzELhUF5tBc76MGddyFnHKCEy3cRh6o0lBAK7VX88JBJyaWdfAQJv54 # Ir52RG7RNlkZUNcFJQeddDyW0Z5ditn80n3kLwHHLemlW0nKxrsuPZODrMzprBdb # yO3NeNwIkbFTMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENB # IDIwMTACEzMAAAH3WCB1BMr7wvQAAQAAAfcwDQYJYIZIAWUDBAIBBQCgggFKMBoG # CSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0BCQQxIgQg5ZrZHT8q # w90rcoNQEpFskO+8HdlZng9ES9yqau0tUaMwgfoGCyqGSIb3DQEJEAIvMYHqMIHn # MIHkMIG9BCAh2pjaa3ca0ecYuhu60uYHP/IKnPbedbVQJ5SoIH5Z4jCBmDCBgKR+ # MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMT # HU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB91ggdQTK+8L0AAEA # AAH3MCIEIOvNpKyTNFzqzjMeGiUctzGHzLU+hp19F5H40gE5hsNVMA0GCSqGSIb3 # DQEBCwUABIICAF/SEfUcz4DygLZkCu9iYtyhd1OQmKzZ4HVl+4rpumoJ+ZMX6asy # 61UMuzWexz6ki5WtIq1l+Dkc0HVzthrmc8YyPsPQoYpislxvR4eiF+OmrUwr3PLw # jcHrn/U/bbiTZBtzxChczlYxY3l4djtuXjQa2B2y3qXuvbRvgzaZHE610GMJ+0Ph # QPKjXiRSapsuB1oj1BC+6zA0MMyaI9o4l4KAc/OyQtKAc8QQazjkQMrNHvzyQy4u # 7ybd20Bh8P4KVw1WN8O5bLpQA2EsRkFFBoy7/psvGJNziSZPuaoq57GXKxNrODZz # 4lZBrg1wCHRwxPx8k6f9Vsu0WBXmh0yZ11RJ+C3BzuVSlDC4cjLKCUJ6/1y+ruRV # Ka2QaZW/1P7TRjaMYHjT6E4h1K4RZQRrzonMz64NJkGIwwyA8n/NX6GvkDoVJ1uw # Z9zABCIbK0nyzy9+RdmUPRQuC+vPcG+mvDq7/2lKK69Y1jC8uArOJFXksRyaKeeJ # xTm/xBSnFJRuN9sTz4kzM0UOMz+hIz9jwKJPMuSLJFC/FCYeCyBHqALzyroDAuQE # /5HvFr9YPRgjfi70kI7Gel3KHI+hRx740mFvh6qbUK2aX492mzx7CXrIPThEPHrd # hPqbtO/c9AnKZCDIEGyXVvjOjUnhXcnZU7ywVI5nnlIG57LjQjt9yM40 # SIG # End signature block |