AksHci.psm1
|
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # AksHci Day 0/2 Operations # ######################################################################################### #requires -runasadministrator using module .\Common.psm1 #region Module Constants $moduleName = "AksHci" $moduleVersion = "1.1.9" #endregion #region Download catalog constants $catalogName = "aks-hci-stable-catalogs-ext" $ringName = "stable" # Major version is checked for compatibilty between PS and product $supportedProductVersion = "1.0.0" #endregion #region Aliases Set-Alias -Name Initialize-AksHciNode -Value Initialize-MocNode #endregion #region # Install Event Log New-ModuleEventLog -moduleName $moduleName #endregion #region to capture TraceCmdlet Configmap details class TraceConfigDetails{ [string] $DeploymentId [string] $Catalog [string] $Audience [string] $AksHciVersion [string] $ModuleName [string] $Offer [string] $ModuleVersion } #endregion #region Private Function function Initialize-AksHciConfiguration { <# .DESCRIPTION Initialize AksHci Configuration Wipes off any existing cached configuration #> if ($global:config.ContainsKey($moduleName)) { $global:config.Remove($moduleName) } $global:config += @{ $moduleName = @{ "installationPackageDir" = "" "installState" = [InstallState]::NotInstalled "manifestCache" = "" "moduleVersion" = $moduleVersion "skipUpdates" = $false "stagingShare" = "" "useStagingShare" = $false "version" = "" "workingDir" = "" "catalog" = "" "ring" = "" "proxyServerCertFile" = "" "proxyServerHTTP" = "" "proxyServerHTTPS" = "" "proxyServerNoProxy" = "" "proxyServerPassword" = "" "proxyServerUsername" = "" "deploymentId" = "" }; } } #endregion #region global config Initialize-AksHciConfiguration #endregion #region Exported Functions function New-AksHciNetworkSetting { <# .SYNOPSIS Create an object for a new virtual network. .DESCRIPTION Create a virtual network to set the DHCP or static IP address for the control plane, load balancer, agent endpoints, and a static IP range for nodes in all Kubernetes clusters. This cmdlet will return a VirtualNetwork object, which can be used later in the configuration steps. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER MacPoolName The name of the mac pool .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .EXAMPLE New-AksHciNetworkSetting -name External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE New-AksHciNetworkSetting -name "Defualt Switch" -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [String] $MacPoolName = $global:cloudMacPool, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend ) trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" ` -CmdletParameters "name $name vswitchName $vswitchName MacPoolName $MacPoolName gateway $gateway dnsservers $dnsservers vippoolstart $vippoolstart vippoolend $vippoolend k8snodeippoolstart $k8snodeippoolstart k8snodeippoolend $k8snodeippoolend" throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters "name $name vswitchName $vswitchName MacPoolName $MacPoolName gateway $gateway dnsservers $dnsservers vippoolstart $vippoolstart vippoolend $vippoolend k8snodeippoolstart $k8snodeippoolstart k8snodeippoolend $k8snodeippoolend" return New-VirtualNetwork -name $name -vswitchName $vswitchName -MacPoolName $MacPoolName -vlanID $vlanID -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend } function Test-ModuleCompatibility { <# .DESCRIPTION Tests if the requested product version is compatible with the version(s) understood by this Powershell module. .PARAMETER Version The AKS HCI product version to be tested for compatibility. #> param ( [String] $Version ) $result = Compare-Versions -Version $script:supportedProductVersion -ComparisonVersion $Version if ($result -eq 0) { return $true } $errorMsg = "The requested product version ($Version) is not compatible with this version of the $moduleName powershell module ($moduleVersion). " if ($result -lt 0) { $errorMsg += "Reason: The module is too old. Please update to a newer version of the $moduleName module." } else { $errorMsg += "Reason: The module is too new. Please try installing a newer version of the product or downgrade the $moduleName powershell module to proceed." } throw $errorMsg } function Set-AksHciConfig { <# .SYNOPSIS Set or update the configurations settings for the Azure Kubernetes Service host. .DESCRIPTION Set the configuration settings for the Azure Kubernetes Service host. If you're deploying on a 2-4 node Azure Stack HCI cluster or a Windows Server 2019 Datacenter failover cluster, you must specify the imageDir and cloudConfigLocation parameters. For a single node Windows Server 2019 Datacenter, all parameters are optional and set to their default values. However, for optimal performance, we recommend using a 2-4 node Azure Stack HCI cluster deployment. .PARAMETER workingDir This is a working directory for the module to use for storing small files. Defaults to %systemdrive%\akshci for single node deployments. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as c:\ClusterStorage\Volume2\ImageStore or an SMB share such as \\FileShare\ImageStore. .PARAMETER imageDir The path to the directory where Azure Kubernetes Service on Azure Stack HCI will store its VHD images. Defaults to %systemdrive%\AksHciImageStore for single node deployments. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as C:\ClusterStorage\Volume2\ImageStore or a SMB share such as \\fileshare\ImageStore. .PARAMETER version The version of Azure Kubernetes Service on Azure Stack HCI that you want to deploy. The default is the latest version. We do not recommend changing the default. .PARAMETER cloudConfigLocation The location where the cloud agent will store its configuration. Defaults to %systemdrive%\wssdcloudagent for single node deployments. The location can be the same as the path of -imageDir. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as C:\ClusterStorage\Volume2\ImageStore or an SMB share such as \\fileshare\ImageStore. The location needs to be on a highly available share so that the storage will always be accessible. .PARAMETER nodeConfigLocation The location where the node agents will store their configuration. Every node has a node agent, so its configuration is local to it. This location must be a local path. Defaults to %systemdrive%\programdata\wssdagent for all deployments. .PARAMETER cloudLocation This parameter provides a custom Microsoft Operated Cloud location name. The default name is "MocLocation". We do not recommend changing the default. .PARAMETER vnet A VirtualNetwork object created using the New-AksHciNetworkSetting cmdlet. .PARAMETER controlplaneVmSize The size of the VM to create for the control plane. To get a list of available VM sizes, use Get-AksHciVmSize. .PARAMETER kvaName Kubernetes Virtual Appliance name. We do not recommend changing the default. .PARAMETER kvaPodCIDR Configures the Kubernetes POD CIDR. We do not recommend changing the default. .PARAMETER nodeAgentPort The TCP/IP port number that node agents should listen on. Defaults to 45000. We do not recommend changing the default. .PARAMETER nodeAgentAuthorizerPort The TCP/IP port number that node agents should use for their authorization port. Defaults to 45001. We do not recommend changing the default. .PARAMETER cloudAgentPort The TCP/IP port number that cloud agent should listen on. Defaults to 55000. We do not recommend changing the default. .PARAMETER cloudAgentAuthorizerPort The TCP/IP port number that cloud agent should use for its authorization port. Defaults to 65000. We do not recommend changing the default. .PARAMETER clusterRoleName This specifies the name to use when creating cloud agent as a generic service within the cluster. This defaults to a unique name with a prefix of ca- and a guid suffix (for example: "ca-9e6eb299-bc0b-4f00-9fd7-942843820c26"). We do not recommend changing the default. .PARAMETER cloudServiceCidr This can be used to provide a static IP/network prefix to be assigned to the MOC CloudAgent service. This value should be provided using the CIDR format. (Example: 192.168.1.2/16). You may want to specify this to ensure that anything important on the network is always accessible because the IP address will not change. Default is none. .PARAMETER proxySettings A ProxySettings object created using the New-AksHciProxySetting cmdlet. .PARAMETER sshPublicKey Path to an SSH public key file. Using this public key, you will be able to log in to any of the VMs created by the Azure Kubernetes Service on Azure Stack HCI deployment. If you have your own SSH public key, you will pass its location here. If no key is provided, we will look for one under %systemdrive%\akshci\.ssh\akshci_rsa.pub. If the file does not exist, an SSH key pair in the above location will be generated and used. .PARAMETER skipHostLimitChecks Requests the script to skip any checks it does to confirm memory and disk space is available before allowing the deployment to proceed. We do not recommend using this setting. .PARAMETER skipRemotingChecks Requests the script to skip any checks it does to confirm remoting capabilities to both local and remote nodes. We do not recommend using this setting. .PARAMETER insecure Deploys Azure Kubernetes Service on Azure Stack HCI components such as cloud agent and node agent(s) in insecure mode (no TLS secured connections). We do not recommend using insecure mode in production environments. .PARAMETER forceDnsReplication DNS replication can take up to an hour on some systems. This will cause the deployment to be slow. To bypass this issue, try to use this flag. The -forceDnsReplication flag is not a guaranteed fix. If the logic behind the flag fails, the error will be hidden, and the command will carry on as if the flag was not provided. .PARAMETER macPoolStart This is used to specify the start of the MAC address of the MAC pool that you wish to use for the Azure Kubernetes Service host VM. The syntax for the MAC address requires that the least significant bit of the first byte should always be 0, and the first byte should always be an even number (that is, 00, 02, 04, 06...). A typical MAC address can look like: 02:1E:2B:78:00:00. Use MAC pools for long-lived deployments so that MAC addresses assigned are consistent. This is useful if you have a requirement that the VMs have specific MAC addresses. Default is none. .PARAMETER macPoolEnd This is used to specify the end of the MAC address of the MAC pool that you wish to use for the Azure Kubernetes Service host VM. The syntax for the MAC address requires that the least significant bit of the first byte should always be 0, and the first byte should always be an even number (that is, 00, 02, 04, 06...). The first byte of the address passed as the -macPoolEnd should be the same as the first byte of the address passed as the -macPoolStart. Use MAC pools for long-lived deployments so that MAC addresses assigned are consistent. This is useful if you have a requirement that the VMs have specific MAC addresses. Default is none. .PARAMETER useStagingShare Reserved for internal use. We do not recommend using this parameter. .PARAMETER containerRegistry Reserved for internal use. We do not recommend using this parameter. .PARAMETER catalog Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Reserved for internal use. We do not recommend using this parameter. .PARAMETER deploymentId Reserved for internal use. We do not recommend using this parameter. .PARAMETER skipUpdates Reserved for internal use. We do not recommend using this parameter. .PARAMETER stagingShare Reserved for internal use. We do not recommend using this parameter. .PARAMETER kvaSkipWaitForBootstrap Reserved for internal use. We do not recommend using this parameter. .PARAMETER deploymentType Reserved for internal use. We do not recommend using this parameter. .PARAMETER activity Reserved for internal use. We do not recommend using this parameter. .PARAMETER enablePreview Enable AKS HCI Early Access Preview feature on Azure. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter()] [String] $workingDir = $global:defaultWorkingDir, [parameter()] [String] $imageDir, [parameter()] [String] $version, [parameter(DontShow)] [String] $stagingShare = $global:defaultStagingShare, [parameter()] [String] $cloudConfigLocation = $global:defaultCloudConfigLocation, [parameter()] [String] $nodeConfigLocation = $global:defaultNodeConfigLocation, [parameter()] [String] $cloudLocation = $global:defaultCloudLocation, [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet, [parameter()] [VmSize] $controlplaneVmSize = $global:defaultMgmtControlPlaneVmSize, [parameter(DontShow)] [String] $kvaName = (New-Guid).Guid, [parameter()] [String] $kvaPodCIDR = $global:defaultPodCidr, [parameter(DontShow)] [Switch] $kvaSkipWaitForBootstrap, [parameter()] [int] $nodeAgentPort = $global:defaultNodeAgentPort, [parameter()] [int] $nodeAgentAuthorizerPort = $global:defaultNodeAuthorizerPort, [parameter()] [int] $cloudAgentPort = $global:defaultCloudAgentPort, [parameter()] [int] $cloudAgentAuthorizerPort = $global:defaultCloudAuthorizerPort, [parameter()] [String] $clusterRoleName = $($global:cloudAgentAppName + "-" + [guid]::NewGuid()), [parameter()] [Alias("cloudServiceIP")] [String] $cloudServiceCidr = "", [parameter()] [ProxySettings] $proxySettings = $null, [parameter()] [String] $sshPublicKey, [parameter(DontShow)] [Switch] $skipUpdates, [parameter(DontShow)] [Switch] $skipHostLimitChecks, [parameter(DontShow)] [Switch] $skipRemotingChecks, [parameter(DontShow)] [Switch] $insecure, [parameter(DontShow)] [Switch] $forceDnsReplication, [parameter()] [String] $macPoolStart, [parameter()] [String] $macPoolEnd, [parameter(DontShow)] [switch] $useStagingShare, [parameter(DontShow)] [ContainerRegistry] $containerRegistry = $null, [parameter(DontShow)] [String] $catalog = $script:catalogName, [parameter(DontShow)] [String] $ring = $script:ringName, [parameter(DontShow)] [String] $deploymentId = [Guid]::NewGuid().ToString(), [parameter(DontShow)] [int] $operatorTokenValidity = $global:operatorTokenValidity, [parameter(DontShow)] [int] $addonTokenValidity = $global:addonTokenValidity, [parameter(DontShow)] [float] $certificateValidityFactor = $global:certificateValidityFactor, [parameter(DontShow)] [Switch] $enablePreview ) trap { Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters "version $version proxySettings $proxySettings catalog $catalog ring $ring moduleName $moduleName kvaName $kvaName enablePreview $enablePreview" throw $_ } Confirm-Configuration ` -useStagingShare:$useStagingShare.IsPresent -stagingShare $stagingShare -vnet $vnet Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName if ($enablePreview.IsPresent) { $catalog = "aks-hci-stable-catalogs-ext" $ring = "earlyaccesspreview" } Set-MocConfig -activity $activity -workingDir $workingDir -imageDir $imageDir -stagingShare $stagingShare ` -cloudConfigLocation $cloudConfigLocation -nodeConfigLocation $nodeConfigLocation ` -vnet $vnet -cloudLocation $cloudLocation ` -nodeAgentPort $nodeAgentPort -nodeAgentAuthorizerPort $nodeAgentAuthorizerPort ` -cloudAgentPort $cloudAgentPort -cloudAgentAuthorizerPort $cloudAgentAuthorizerPort -version $version ` -clusterRoleName $clusterRoleName -cloudServiceCidr $cloudServiceCidr -skipUpdates:$skipUpdates.IsPresent ` -skipHostLimitChecks:$skipHostLimitChecks.IsPresent -insecure:$insecure.IsPresent ` -forceDnsReplication:$forceDnsReplication.IsPresent ` -useStagingShare:$useStagingShare.IsPresent -macPoolStart $macPoolStart -macPoolEnd $macPoolEnd ` -sshPublicKey $sshPublicKey -skipRemotingChecks:$skipRemotingChecks.IsPresent ` -proxySettings $proxySettings -catalog $catalog -ring $ring ` -deploymentId $deploymentId -certificateValidityFactor $certificateValidityFactor Set-KvaConfig -activity $activity -workingDir $workingDir -imageDir $imageDir -stagingShare $stagingShare ` -kvaName $kvaName -kvaPodCIDR $kvaPodCIDR -kvaSkipWaitForBootstrap:$kvaSkipWaitForBootstrap.IsPresent ` -controlplaneVmSize $controlplaneVmSize ` -vnet $vnet -cloudLocation $cloudLocation ` -skipUpdates:$skipUpdates.IsPresent -insecure:$insecure.IsPresent ` -useStagingShare:$useStagingShare.IsPresent -version $version -macPoolStart $macPoolStart -macPoolEnd $macPoolEnd ` -proxySettings $proxySettings -containerRegistry:$containerRegistry ` -catalog $catalog -ring $ring ` -cloudAgentPort $cloudAgentPort -cloudAgentAuthorizerPort $cloudAgentAuthorizerPort ` -deploymentId $deploymentId -operatorTokenValidity $operatorTokenValidity -addonTokenValidity $addonTokenValidity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Creating configuration for $moduleName" Set-AksHciConfigValue -name "workingDir" -value $workingDir Set-AksHciConfigValue -name "manifestCache" -value ([io.Path]::Combine($workingDir, $("$catalog.json"))) New-Item -ItemType Directory -Force -Path $workingDir | out-null Set-AksHciConfigValue -name "moduleVersion" -value $moduleVersion Set-AksHciConfigValue -name "installState" -value ([InstallState]::NotInstalled) Set-AksHciConfigValue -name "stagingShare" -value $stagingShare Set-AksHciConfigValue -name "skipUpdates" -value $skipUpdates.IsPresent Set-AksHciConfigValue -name "useStagingShare" -value $useStagingShare.IsPresent Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Set-AKsHciConfigValue -name "deploymentId" -value $deploymentId if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if (-not $version) { # If no version is specified, use the latest $version = Get-AksHciLatestVersion Set-AksHciConfigValue -name "version" -value $version } } else { Get-AksHciLatestVersion | out-null # This clears the cache Get-ProductRelease -Version $version -module $moduleName | Out-Null Set-AksHciConfigValue -name "version" -value $version } Test-ModuleCompatibility -Version $version | Out-Null $installationPackageDir = [io.Path]::Combine($workingDir, $version) Set-AksHciConfigValue -name "installationPackageDir" -value $installationPackageDir New-Item -ItemType Directory -Force -Path $installationPackageDir | Out-Null Save-ConfigurationDirectory -moduleName $moduleName -WorkingDir $workingDir Save-Configuration -moduleName $moduleName Write-SubStatus -moduleName $moduleName "New configuration has been saved`n" Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters "version $version proxySettings $proxySettings catalog $catalog ring $ring moduleName $moduleName kvaName $kvaName enablePreview $enablePreview" } function Set-AksHciConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER value Value to be persisted #> param ( [String] $name, [Object] $value ) Set-ConfigurationValue -name $name -value $value -module $moduleName } function Get-AksHciConfig { <# .SYNOPSIS List the current configuration settings for the Azure Kubernetes Service host. .DESCRIPTION List the current configuration settings for the Azure Kubernetes Service host. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) trap { Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters throw $_ } Import-AksHciConfig -activity $activity Write-Status -moduleName $moduleName "Getting configuration for $moduleName" $global:config[$modulename]["installState"] = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $global:config } function Import-AksHciConfig { <# .DESCRIPTION Loads a configuration from persisted storage. If no configuration is present then a default configuration can be optionally generated and persisted. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter()] [Switch] $createIfNotPresent, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status "Importing Configuration" # Check if configuration exists if (Test-Configuration -moduleName $moduleName) { # 1. Trigger an import of the dependent configurations Get-MocConfig | Out-Null Get-KvaConfig | Out-Null Import-Configuration -moduleName $moduleName } else { throw "This machine does not appear to be configured for deployment." } Write-StatusWithProgress -activity $activity -module $moduleName -status "Importing Configuration Completed" } function Install-AksHci { <# .SYNOPSIS Install the Azure Kubernetes Service on Azure Stack HCI agents/services and host. .DESCRIPTION Install the Azure Kubernetes Service on Azure Stack HCI agents/services and host. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $activity = $MyInvocation.MyCommand.Name trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -createConfigIfNotPresent -skipMgmtKubeConfig -skipInstallationCheck -activity $activity Test-KvaAzureConnection Install-AksHciInternal -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } function Enable-AksHciPreview { <# .SYNOPSIS Enable AKSHCI catalog and ring configuration to expose early access preview builds. .DESCRIPTION Enable AKSHCI catalog and ring configuration to expose early access preview builds. .PARAMETER activity Activity name to use when updating progress .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter(DontShow)] [String] $ring = "earlyaccesspreview" ) $activity = $MyInvocation.MyCommand.Name trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } #Set MocConfig for early access preview Enable-MocPreview -catalog $catalog -ring $ring #Set KvaConfig for early access preview Enable-KvaPreview -catalog $catalog -ring $ring #Set AksHCiConfig for early access preview Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Enabling Preview for $moduleName" Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName "Preview Configuration for module $moduleName has been updated`n" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Write-Warning "Warning: Preview features are intended to be used for development and testing purposes only." Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Disable-AksHciPreview { <# .SYNOPSIS Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .DESCRIPTION Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .PARAMETER activity Activity name to use when updating progress. .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter(DontShow)] [String] $ring = "stable" ) $activity = $MyInvocation.MyCommand.Name trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } #Set MocConfig Disable-MocPreview -catalog $catalog -ring $ring #Set KvaConfig Disable-KvaPreview -catalog $catalog -ring $ring #Set AksHCiConfig Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Disabling preview for $moduleName" Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName "Configuration for module $moduleName has been updated`n" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Restart-AksHci { <# .SYNOPSIS Restart Azure Kubernetes Service on Azure Stack HCI and remove all deployed Kubernetes clusters. .DESCRIPTION Restarting Azure Kubernetes Service on Azure Stack HCI will remove all of your Kubernetes clusters if any, and the Azure Kubernetes Service host. It will also uninstall the Azure Kubernetes Service on Azure Stack HCI agents and services from the nodes. It will then go back through the original install process steps until the host is recreated. The Azure Kubernetes Service on Azure Stack HCI configuration that you configured via Set-AksHciConfig and the downloaded VHDX images are preserved. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck Uninstall-AksHci -SkipConfigCleanup -activity $activity Install-AksHciInternal -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } function Uninstall-AksHci { <# .SYNOPSIS Removes Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Removes Azure Kubernetes Service on Azure Stack HCI. If PowerShell commands are run on a cluster where Windows Admin Center was previously used to deploy, the PowerShell module checks the existence of the Windows Admin Center configuration file. Windows Admin Center places the Windows Admin Center configuration file across all nodes. .PARAMETER SkipConfigCleanup skips removal of the configurations after uninstall. After Uninstall, you have to Set-AksHciConfig to install again. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [Switch] $SkipConfigCleanup, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) try { Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity $configDetails = Get-TraceConfigDetails $aksHciRegistration = Get-AksHciRegistration if (-not [string]::IsNullOrWhiteSpace($aksHciRegistration.azureResourceGroup)) { try { Test-KvaAzureConnection } catch [Exception] { Write-SubStatus -moduleName $moduleName "Warning: Install-AksHci was setup with an Azure Connection, but the connection has expired. Uninstall will continue but will result in leaked Azure resources." } } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Uninstalling) try { $clusters = Get-AksHciCluster foreach($cluster in $clusters) { try { Remove-AksHciCluster -Name $cluster.Name -Confirm:$false } catch [Exception] { Write-Status -moduleName $moduleName -msg "Exception caught!!!" Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() Write-SubStatus -moduleName $moduleName -msg "Could not delete target cluster ""$cluster.Name""" } } } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } } catch [Exception] { # If AksHci is not installed, you would reach here Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } try { Uninstall-Kva -SkipConfigCleanup:$SkipConfigCleanup.IsPresent -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } #ConfigMap details are not available after kva is uninstalled try { Uninstall-Moc -SkipConfigCleanup:$SkipConfigCleanup.IsPresent -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } Trace-Cmdlet -ConfigDetails $configDetails Set-AksHciConfigValue -name "installState" -value ([InstallState]::NotInstalled) if (!$SkipConfigCleanup.IsPresent) { Reset-Configuration -moduleName $moduleName } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed } function Get-AksHciKubernetesVersion { <# .SYNOPSIS List the available versions for creating a managed Kubernetes cluster. .DESCRIPTION List the available versions for creating a managed Kubernetes cluster. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Write-StatusWithProgress -activity $activity -status "Retrieving Kubernetes versions" -moduleName $moduleName Get-AvailableKubernetesVersions Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } function Get-AksHciVmSize { <# .SYNOPSIS Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Write-StatusWithProgress -activity $activity -status "Retrieving VM Sizes" -moduleName $moduleName $result = @() foreach($definition in $global:vmSizeDefinitions) { $size = [ordered]@{'VmSize' = $definition[0]; 'CPU' = $definition[1]; 'MemoryGB' = $definition[2]} $result += New-Object -TypeName PsObject -Property $size } Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters return $result } function Set-AksHciCluster { <# .SYNOPSIS Scale the number of control plane nodes or worker nodes in a cluster. .DESCRIPTION Scale the number of control plane nodes or worker nodes in a cluster. The control plane nodes and the worker nodes must be scaled independently. .PARAMETER Name Name of the cluster .PARAMETER controlPlaneNodeCount The number of control plane nodes to scale to .PARAMETER linuxNodeCount The number of Linux worker nodes to scale to .PARAMETER windowsNodeCount The number of Windows worker nodes to scale to .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='controlplane')] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [int] $linuxNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [int] $windowsNodeCount, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity $mgmtCluster = (Get-KvaConfig)["kvaName"] if ($Name -ieq $mgmtCluster) { throw "Scaling of the management cluster is not supported at this time." } if ($PSCmdlet.ParameterSetName -ieq "controlplane") { Set-KvaClusterNodeCount -Name $Name -controlPlaneNodeCount $controlPlaneNodeCount -activity $activity } elseif ($PSCmdlet.ParameterSetName -ieq "worker") { Write-Output @" WARNING: Set-AksHciCluster can now only be used to scale either the control plane node count or the worker count of the default nodepools that were created as a part of the older cluster creation workflow. Please consider using Set-AksHciNodePool to manage your node pool worker count as it can be used for any nodepool. Examples: - Get a list of cluster1's nodepools: `tGet-AksHciNodePool -ClusterName "cluster1" - Scale "nodepool1" to 2 worker nodes: `tSet-AksHciNodePool -ClusterName "cluster1" -Name "nodepool1" -Count 2 "@ if ($windowsNodeCount -gt 0) { $cluster = Get-KvaCluster -Name $Name -activity $activity Test-SupportedKubernetesVersion -imageType Windows -k8sVersion $cluster.KubernetesVersion } Set-KvaClusterNodeCount -Name $Name -linuxNodeCount $linuxNodeCount -windowsNodeCount $windowsNodeCount -activity $activity } Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function New-AksHciCluster { <# .SYNOPSIS Create a new managed Kubernetes cluster. .DESCRIPTION Create a new Azure Kubernetes Service on Azure Stack HCI cluster. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneNodeCount The number of control plane (master) nodes .PARAMETER linuxNodeCount The number of Linux worker nodes .PARAMETER windowsNodeCount The number of Windows worker nodes .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER nodePoolName The name of the node pool .PARAMETER nodeCount The number of worker nodes in the node pool .PARAMETER nodeMaxPodCount The maximum number of pods that can run on a worker node .PARAMETER taints A list of taints to put on each worker node .PARAMETER nodeVmSize The VM size to use for the worker nodes in the node pool .PARAMETER osType The OS type for the worker nodes in the node pool .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER enableMonitoring Enable deploying the monitoring once cluster creation is complete. .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .PARAMETER primaryNetworkPlugin Network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". .PARAMETER clusterStorageContainer Storage container that is associated to the Cluster. #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName = 'twonodepools')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [String] $kubernetesVersion = $global:defaultTargetK8Version, [Parameter()] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [int] $linuxNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [int] $windowsNodeCount = 0, [Parameter()] [String] $controlplaneVmSize = $global:defaultControlPlaneVmSize, [Parameter()] [String] $loadBalancerVmSize = $global:defaultLoadBalancerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $linuxNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $windowsNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepool')] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $nodePoolName = $global:defaultNodePoolName, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeCount = $global:defaultWorkerNodeCount, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeMaxPodCount = 0, [Parameter(ParameterSetName = 'onenodepool')] [String[]] $taints, [Parameter(ParameterSetName = 'onenodepool')] [VmSize] $nodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepool')] [OsType] $osType = $global:defaultWorkerNodeOS, [Parameter()] [Switch]$enableADAuth, [Parameter()] [Switch]$enableMonitoring, [Parameter()] [VirtualNetwork]$vnet, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity, [Parameter()] [ValidateScript({return $true})] #Note: ValidateScript automatically constructs the NetworkPlugin object, therefore validates the parameter [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter()] [String] $clusterStorageContainer = $global:cloudStorageContainer ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity if ($PSCmdlet.ParameterSetName -ieq "twonodepools") { Write-Output @" WARNING: In a future release, New-AksHciCluster will change how node pools are created. Currently, a user can only set a node count for two default node pools using the parameters linuxNodeCount and windowsNodeCount. This behavior will change giving users more control over the default node pool created after a cluster is deployed. Please consider using the new parameters to manage node pools below. Parameters to be deprecated: `t-linuxNodeCount `t-linuxNodeVmSize `t-windowsNodeCount `t-windowsNodeVmSize New parameters: `t-nodePoolName `t-nodeCount `t-nodeVmSize `t-osType Examples: - Create a cluster that has a node pool with 1 Windows worker node: `tNew-AksHciCluster -Name "cluster1" -osType Windows - Create a cluster that has a node pool with 2 Linux worker nodes and a custom name: `tNew-AksHciCluster -Name "cluster1" -nodePoolName "example-nodepool" -nodeCount 2 "@ Write-StatusWithProgress -activity $activity -status "Verifying Linux kubernetes version..." -moduleName $moduleName Test-SupportedKubernetesVersion -imageType Linux -k8sVersion $kubernetesVersion if ($windowsNodeCount -gt 0) { Write-StatusWithProgress -activity $activity -status "Verifying Windows kubernetes version..." -moduleName $moduleName Test-SupportedKubernetesVersion -imageType Windows -k8sVersion $kubernetesVersion } New-KvaCluster ` -Name $Name -kubernetesVersion $kubernetesVersion ` -controlPlaneNodeCount $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize ` -linuxNodeCount $linuxNodeCount -linuxNodeVmSize $linuxNodeVmSize ` -windowsNodeCount $windowsNodeCount -windowsNodeVmSize $windowsNodeVmSize ` -enableADAuth:$enableADAuth.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin.Name -vnet $vnet ` -activity $activity } elseif ($PSCmdlet.ParameterSetName -ieq "onenodepool") { Write-StatusWithProgress -activity $activity -status "Verifying $osType kubernetes version..." -moduleName $moduleName Test-SupportedKubernetesVersion -imageType $osType -k8sVersion $kubernetesVersion New-KvaCluster ` -Name $Name -kubernetesVersion $kubernetesVersion ` -controlPlaneNodeCount $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize ` -nodePoolName $nodePoolName -nodeCount $nodeCount -nodeMaxPodCount $nodeMaxPodCount -taints $taints ` -nodeVmSize $nodeVmSize -osType $osType ` -enableADAuth:$enableADAuth.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin.Name -vnet $vnet ` -activity $activity } Get-AksHciCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName ## If enableMonitoring is enabled then install the monitoring with default values. if ($enableMonitoring.IsPresent) { Install-AksHciMonitoring -Name $Name -storageSizeGB 100 -retentionTimeHours 240 } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Get-AksHciCluster { <# .SYNOPSIS List Kubernetes managed clusters including the Azure Kubernetes Service host. .DESCRIPTION List Kubernetes managed clusters including the Azure Kubernetes Service host. .PARAMETER Name Name of the cluster .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Gathering cluster information" -moduleName $moduleName Get-KvaCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Remove-AksHciCluster { <# .SYNOPSIS Delete a managed Kubernetes cluster. .DESCRIPTION Delete a managed Kubernetes cluster. .PARAMETER Name Name of the cluster .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($Name, "Delete the managed Kubernetes cluster")) { if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity Remove-KvaCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } } function Get-AksHciClusterUpdates { <# .SYNOPSIS Get the available Kubernetes upgrades for an Azure Kubernetes Service cluster. .DESCRIPTION Get the available Kubernetes upgrades for an Azure Kubernetes Service cluster. .PARAMETER Name Name of the cluster. .PARAMETER activity Activity name to use when updating progress. #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $upgrades = Get-KvaClusterUpgrades -Name $Name -activity $activity $upgrades.AvailableUpgrades Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Update-AksHciCluster { <# .SYNOPSIS Update a managed Kubernetes cluster to a newer Kubernetes or OS version. .DESCRIPTION Update a managed Kubernetes cluster to a newer Kubernetes or OS version. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to upgrade to .PARAMETER operatingSystem Perform an operating system upgrade instead of a kubernetes version upgrade .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'Low')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $kubernetesVersion, [Parameter()] [Switch] $operatingSystem, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null if ($operatingSystem.IsPresent -and $kubernetesVersion -ne "") { # operating system is updated when kubernetes version is upgraded. # if user specifies both, just turn the switch off, because we will internally # update the OS. $operatingSystem = $false } $nextVersion = $null if (-not $operatingSystem.IsPresent) { if ($kubernetesVersion -eq "") { # no version was requested. just try to make the highest jump. $nextVersion = Get-NextKubernetesVersionForUpgrade -Name $Name -activity $activity } else { $nextVersion = Get-CleanInputKubernetesVersion -KubernetesVersion $kubernetesVersion } } if ($PSCmdlet.ShouldProcess($Name, "Update the managed Kubernetes cluster")) { $confirmValue = $true if ($PSBoundParameters.ContainsKey('Confirm')) { $confirmValue = $PSBoundParameters['Confirm'] } try{ Update-KvaCluster -Name $Name -activity $activity -operatingSystem:$operatingSystem.IsPresent -nextVersion $nextVersion -Confirm:$confirmValue } catch [Exception] { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ChildStageName "UpdateKvaCluster" ` -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters } } Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function New-AksHciNodePool { <# .SYNOPSIS Create a new nodepool under a cluster. .DESCRIPTION Creates a new nodepool under a cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count The number of worker nodes in the nodepool .PARAMETER OSType OS type of the node pool. Defaults to Linux .PARAMETER VMSize The VM size to use for the worker nodes. Defaults to Standard_K8S3_v1 .PARAMETER MaxPodCount The maximum number of pods that can run on a worker node .PARAMETER Taints A list of taints to put on each worker node .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [int] $Count = $global:defaultNodeCount, [Parameter()] [VmSize] $VMSize = $global:defaultWorkerVmSize, [Parameter()] [OsType] $OSType = $global:defaultOSType, [Parameter()] [int] $MaxPodCount = 0, [Parameter()] [String[]] $Taints, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Creating node pool" -moduleName $moduleName New-KvaClusterNodePool ` -ClusterName $ClusterName -Name $Name ` -MaxPodCount $MaxPodCount -Taints $Taints -Count $Count ` -VMSize $VMSize -OSType $OSType ` -activity $activity Get-AksHciNodePool -ClusterName $ClusterName -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Get-AksHciNodePool { <# .SYNOPSIS List a Kubernetes managed cluster's nodepools. .DESCRIPTION List a Kubernetes managed cluster's nodepools. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Gathering cluster nodepool information" -moduleName $moduleName Get-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Set-AksHciNodePool { <# .SYNOPSIS Scale a Kubernetes managed cluster's nodepool. .DESCRIPTION Scale a Kubernetes managed cluster's nodepool. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count Node count to scale to .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter(Mandatory=$true)] [int] $Count, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Scaling nodepool" -moduleName $moduleName Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Count $Count -activity $activity Get-AksHciNodePool -ClusterName $ClusterName -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Remove-AksHciNodePool { <# .SYNOPSIS Delete a nodepool in a managed Kubernetes cluster. .DESCRIPTION Delete a nodepool in a managed Kubernetes cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($name, "Delete the node pool in the managed Kubernetes cluster")) { if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Deleting nodepool" -moduleName $moduleName Remove-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Confirm:$false -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Get-AksHciLogs { <# .SYNOPSIS Create a zipped folder with logs from all your pods. .DESCRIPTION Create a zipped folder with logs from all your pods. This command will create an output zipped folder called akshcilogs.zip in your AKS on Azure Stack HCI working directory. The full path to the akshcilogs.zip file will be the output after running Get-AksHciLogs (for example, C:\AksHci\0.9.6.3\akshcilogs.zip, where 0.9.6.3 is the AKS on Azure Stack HCI release number). .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .PARAMETER VirtualMachineLogs Switch to get only the logs from the vm's (LB vm if unstacked deployment and management-cluster vm) .PARAMETER AgentLogs Switch to get only logs of the wssdagent and wssdcloudagent on all nodes .PARAMETER EventLogs Switch to get only Windows Event Logson all nodes .PARAMETER KvaLogs Switch to get only the logs from KVA .PARAMETER DownloadSdkLogs Switch to get only the logs from DownloadSdk .PARAMETER BillingRecords Switch to get only the billing records #> param ( [Parameter()] [Switch]$AsJob, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$false)] [Switch]$VirtualMachineLogs, [Parameter(Mandatory=$false)] [Switch]$AgentLogs, [Parameter(Mandatory=$false)] [Switch]$EventLogs, [Parameter(Mandatory=$false)] [Switch]$KvaLogs, [Parameter(Mandatory=$false)] [Switch]$DownloadSdkLogs, [Parameter(Mandatory=$false)] [Switch]$BillingRecords ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } $allswitch = $true if ($VirtualMachineLogs.IsPresent -or $AgentLogs.IsPresent -or $EventLogs.IsPresent -or $KvaLogs.IsPresent -or $DownloadSdkLogs.IsPresent -or $BillingRecords.IsPresent) { $allswitch = $false } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck $logDir = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], "akshcilogs") if ($VirtualMachineLogs.IsPresent -or $AgentLogs.IsPresent -or $EventLogs.IsPresent -or $allswitch) { try { Get-MocLogs -path $logDir -activity $activity -VirtualMachineLogs:$VirtualMachineLogs.IsPresent -AgentLogs:$AgentLogs.IsPresent -EventLogs:$EventLogs.IsPresent | Out-Null } catch [Exception] { } } if ($allswitch -or $KvaLogs.IsPresent) { try{ Get-KvaLogs -path $logDir -activity $activity } catch [Exception] { } } if ($allswitch -or $DownloadSdkLogs.IsPresent) { try { Get-DownloadSdkLogs -Path $logDir } catch [Exception] { } } if ($allswitch -or $BillingRecords.IsPresent) { New-Item -ItemType Directory -Force -Path $logDir | Out-Null try { Get-KvaBillingRecords -activity $activity -outputformat "json" | ConvertFrom-Json | Format-List * > ($logDir + "\AksHciBillingRecords.log") } catch [Exception]{ Write-Status -moduleName $moduleName -msg "Warning: Billing records collection failed" Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() } } $akshcilogDir = [io.Path]::Combine($logDir, "akshci") New-Item -ItemType Directory -Force -Path $akshcilogDir | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Collecting $moduleName information...") $global:config[$moduleName] > $akshcilogDir"\AksHciConfig.txt" Get-AksHciEventLog | Format-List * > $akshcilogDir"\AksHciPS.log" Get-Command -Module AksHci | Sort-Object -Property Source > $($akshcilogDir+"\moduleinfo.txt") Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Compressing Logs...") $zipName = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], "akshcilogs.zip") try { Compress-Directory -ZipFilename $zipName -SourceDir $logDir } catch [Exception] { Write-Status -moduleName $moduleName -msg "Exception caught!!!" Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() Write-SubStatus -moduleName $moduleName -msg "Could not compress ""$zipName""" Write-Status -moduleName $moduleName -msg "Logs Directory is located at ""$logDir""" return $logDir } Remove-Item -Path $logDir -Force -Recurse -ErrorAction Continue Write-Status -moduleName $moduleName "Zip File is located at ""$zipName""" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $zipName } function Get-AksHciEventLog { <# .SYNOPSIS Gets all the event logs from the Azure Kubernetes Service on Azure Stack HCI PowerShell module. .DESCRIPTION Gets all the event logs from the Azure Kubernetes Service on Azure Stack HCI PowerShell module. #> trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$_" ` -CmdletParameters $PSBoundParameters throw $_ } $logs = Get-WinEvent -ProviderName $moduleName -ErrorAction Ignore $logs += Get-KvaEventLog $logs += Get-MocEventLog $logs += Get-DownloadSdkEventLog Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $logs } function Enable-AksHciArcConnection { <# .SYNOPSIS Connects an AKS on Azure Stack HCI workload cluster to Azure Arc for Kubernetes. .DESCRIPTION Connects an AKS on Azure Stack HCI workload cluster to Azure Arc for Kubernetes. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } # just to ensure the cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { New-KvaArcConnection -Name $Name -activity $activity } else { New-KvaArcConnection -Name $Name -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -activity $activity } Write-SubStatus -moduleName $moduleName "Arc has been installed to the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Disable-AksHciArcConnection { <# .DESCRIPTION Helper function to remove the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } # just to ensure the cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Remove-KvaArcConnection -Name $Name -activity $activity } else { Remove-KvaArcConnection -Name $Name -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -activity $activity } Write-SubStatus -moduleName $moduleName "Arc has been uninstalled from the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Install-AksHciAdAuth { <# .SYNOPSIS Install Active Directory authentication. .DESCRIPTION Install Active Directory authentication. .PARAMETER Name Cluster Name .PARAMETER keytab Path to the kerberos keytab corresponding to the current password on the local machine. Must be named current.keytab .PARAMETER previousKeytab Path to the kerberos keytab corresponding to the previous password on the local machine. Must be named previous.keytab .PARAMETER SPN SPN registered for the Active Directory account to be used with the api-server. .PARAMETER TTL Time to live (in hours) for previous keytab file if supplied. Default is 10 hours .PARAMETER adminUser The user name to be given cluster-admin permissions. Machine must be domain joined. .PARAMETER adminGroup The group name to be given cluster-admin permissions. Machine must be domain joined. .PARAMETER adminUserSID The user SID to be given cluster-admin permissions. .PARAMETER adminGroupSID The group SID to be given cluster-admin permissions. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='domainjoin')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [String] $keytab, [Parameter(Mandatory=$false)] [String] $previousKeytab, [Parameter(Mandatory=$true)] [String] $SPN, [Parameter(Mandatory=$false)] [int] $TTL, [Parameter(Mandatory=$false, ParameterSetName='domainjoin')] [String] $adminUser, [Parameter(Mandatory=$false, ParameterSetName='domainjoin')] [String] $adminGroup, [Parameter(Mandatory=$false, ParameterSetName='workplacejoin')] [String] $adminUserSID, [Parameter(Mandatory=$false, ParameterSetName='workplacejoin')] [String] $adminGroupSID, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $capiCluster = Get-KvaCapiCluster -Name $Name $canInstallWebhook = $false foreach($feature in $capiCluster.additionalfeatures ) { if ($feature.FeatureName -eq "ad-auth-webhook") { $canInstallWebhook = $true } } if (-not $canInstallWebhook) { Write-SubStatus -moduleName $moduleName "Cluster '$Name' doesn't have feature ADAuth enabled. Please Recreate AksHciCluster with the -enableADAuth switch" return } if(-not $adminUser -and -not $adminUserSID -and -not $adminGroup -and -not $adminGroupSID) { Write-SubStatus -moduleName $moduleName "One of -adminUser, -adminGroup, -adminGroupSID, or -adminUserSID is required to enable this add-on" return } try { if (![string]::IsNullOrEmpty($adminUser)) { $adminUserSIDYAML = (New-Object System.Security.Principal.NTAccount($adminUser)).Translate([System.Security.Principal.SecurityIdentifier]).value } if (![string]::IsNullOrEmpty($adminGroup)) { $adminGroupSIDYAML = (New-Object System.Security.Principal.NTAccount($adminGroup)).Translate([System.Security.Principal.SecurityIdentifier]).value } } catch { Write-SubStatus -moduleName $moduleName "Name to SID translation failed. If the machine is not domain joined, specify adminUserSID or adminGroupSID" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ChildStageName "NameToSIDTranslationFailed" ` -ErrorMessage "$activity - $_" return } if ($PSCmdlet.ParameterSetName -ieq "workplacejoin") { $adminUserSIDYAML = $adminUserSID $adminGroupSIDYAML = $adminGroupSID } if (![string]::IsNullOrEmpty($previousKeytab)) { $prevKtSt = "--from-file=`"$previousKeytab`"" } $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: ad-auth-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: ad-auth-webhook targetNamespace: kube-system templateType: yaml providerVariables: - key: AD_AUTH_SPN value: "$SPN" - key: ADMIN_USER value: "$adminUserSIDYAML" - key: ADMIN_GROUP value: "$adminGroupSIDYAML" - key: TICKET_LIFETIME value: "$TTL" - key: keytab valueFrom: secret: name: keytab-$Name "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-ad-auth-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KubeCtl -arguments $("create secret generic keytab-$Name --from-file=`"$keytab`" $prevKtSt") Invoke-Kubectl -arguments $("apply -f ""$yamlFile"" ") Write-SubStatus -moduleName $moduleName "Active Directory SSO has been installed to the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Uninstall-AksHciAdAuth { <# .SYNOPSIS Uninstall Active Directory authentication. .DESCRIPTION Uninstall Active Directory authentication. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: ad-auth-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: ad-auth-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-ad-auth-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("delete -f ""$yamlFile"" ") Remove-Item $yamlFile Write-SubStatus -moduleName $moduleName "Active Directory SSO webhook has been uninstalled" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Install-AksHciGMSAWebhook { <# .DESCRIPTION Installs gMSA webhook for an AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Set-KvaGMSAWebhook -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName "gMSA webhook has been installed to the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Uninstall-AksHciGMSAWebhook { <# .DESCRIPTION Uninstalls gmsa-webhook addon for an AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Reset-KvaGMSAWebhook -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName "GMSA webhook has been uninstalled" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Add-AksHciGMSACredentialSpec { <# .DESCRIPTION Helper function to add a credentials spec for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecFilePath File Path of the JSON cred spec file .PARAMETER credSpecName Name of the Kubernetes credential spec object the user would like to designate This will be the name the deployment yaml reference for the field gmsaCredentialSpec .PARAMETER secretName Name of the Kubernetes secret object storing the Active Directory user credentials and gMSA domain .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER serviceAccount Name of the Kubernetes service account assigned to read the Kubernetes gMSA credspec object .PARAMETER clusterRoleName Name of the Kubernetes clusterrole assigned to use the Kubernetes gMSA credspec object .PARAMETER overwrite Overwrites existing Kubernetes credential spec object .PARAMETER activity Activity name to use when updating progress .EXAMPLE Add-AksHciGMSACredentialSpec -Name test1 -credSpecFilePath .\credspectest.json -credSpecName credspec-test1 -secretName secret-test1 -clusterRoleName clusterrole-test1 Creates a GMSACredentialSpec object called credspec-test1 from the JSON credential spec file credspectest.json on a target cluster named test1. The object credspec-test1 references the default namespaced secret secret-test1 created by the user for Active Directory user credentials. The cmdlet also creates a cluster role named clusterrole-test1 that binds to the default service account along with a rolebinding that resides in the default namespace. .EXAMPLE Add-AksHciGMSACredentialSpec -Name test1 -credSpecFilePath .\credspectest.json -credSpecName credspec-test1 -secretName secret-test1 -secretNamespace secret-namespace -clusterRoleName clusterrole-test1 -serviceAccount svc1 -overwrite Creates a GMSACredentialSpec object called credspec-test1 from the JSON credential spec file credspectest.json on a target cluster named test1. The object credspec-test1 references the secret secret-test1 residing in the namespace secret-namespace. Both the secret and the namespace secret-namespace are created by the user. The also cmdlet creates a cluster role named clusterrole-test1 that binds to the user-created service account svc1 along with a rolebinding that resides in the secret-namespace namespace. The overwrite parameter checks for existing GMSACredentialSpec, clusterrole, and rolebinding objects with the same names as the ones specified by the cmdlet parameters and overwrites them with the new setup based on the new parameters. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecFilePath')] [String]$credSpecFilePath, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecName')] [String]$credSpecName, [Parameter(Mandatory=$true)] [String]$secretName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [switch]$overwrite, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = $MyInvocation.MyCommand.Name } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Set-KvaGMSACredentialSpec -Name $Name -credSpecFilePath $credSpecFilePath -credSpecName $credSpecName ` -secretName $secretName -secretNamespace $secretNamespace -serviceAccount $serviceAccount ` -clusterRoleName $clusterRoleName -overwrite:$overwrite.isPresent -activity $activity Write-SubStatus -moduleName $moduleName "GMSA credential spec has been installed" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Remove-AksHciGMSACredentialSpec { <# .DESCRIPTION Helper function to remove a credentials spec for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecName Name of the Kubernetes credential spec object the user would like to designate .PARAMETER serviceAccount Kubernetes service account assigned to read the Kubernetes gMSA credential spec object .PARAMETER clusterRoleName Name of the Kubernetes clusterrole assigned to use the Kubernetes gMSA credential spec object .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER activity Activity name to use when updating progress .EXAMPLE Remove-AksHciGMSACredentialSpec -Name test1 -credSpecName credspec-test1 -clusterRoleName clusterrole-test1 Removes the GMSACredentialSpec object credspec-test1 and the clusterrole object clusterrole-test1 along with the rolebinding object binding clusterrole-test1 to the default service account from a target cluster named test1 .EXAMPLE Remove-AksHciGMSACredentialSpec -Name test1 -credSpecName credspec-test1 -serviceAccount svc1 -secretNamespace secret-namespace -clusterRoleName clusterrole-test1 Removes a GMSACredentialSpec object credspec-test1 and the clusterrole object clusterrole-test1 from the target cluster test1. The rolebinding object binding clusterrole-test1 to the service account svc1 is also removed from the secret-namespace namespace in the target cluster named test1. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecName')] [String]$credSpecName, [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [String]$secretNamespace = "default", [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = $MyInvocation.MyCommand.Name } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Reset-KvaGMSACredentialSpec -Name $Name -credSpecName $credSpecName -serviceAccount $serviceAccount ` -clusterRoleName $clusterRoleName -secretNamespace $secretNamespace -activity $activity Write-SubStatus -moduleName $moduleName "GMSA credential spec has been uninstalled" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } function Get-AksHciCredential { <# .SYNOPSIS Access your cluster using kubectl. .DESCRIPTION Access your cluster using kubectl. This will use the specified cluster's kubeconfig file as the default kubeconfig file for kubectl. .PARAMETER Name Name of the cluster to obtain the credential/kubeconfig for. .PARAMETER configPath Location to output the credential/kubeconfig file to. .PARAMETER adAuth To get the Active Directory SSO version of the kubeconfig. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [string] $Name, [Parameter()] [string] $configPath = $($env:USERPROFILE+"\.kube\config"), [Parameter(Mandatory=$false)] [Switch] $adAuth, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($Name, $("Retrieve and write the cluster kubeconfig file to $configPath"))) { Initialize-AksHciEnvironment -activity $activity Get-KvaClusterCredential -Name $Name -outputLocation $configPath -adAuth:$adAuth.IsPresent -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters } } function Repair-AksHciClusterCerts { <# .DESCRIPTION Attempts to repair failed TLS on a cluster/cloudagent .PARAMETER Name Name of the node/cluster to fix .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER $fixCloudCredentials Fix cloud tls in a cluster .PARAMETER $fixKubeletCredentials Fix failed TLS on a cluster .PARAMETER force Force repair(without checks) .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(DefaultParameterSetName = 'cloud')] param ( [Parameter(Mandatory=$true)] [string] $Name, [Parameter()] [string] $sshPrivateKeyFile, [Parameter(Mandatory=$true, ParameterSetName='cloud')] [Switch] $fixCloudCredentials, [Parameter(Mandatory=$true, ParameterSetName='kubelet')] [Switch] $fixKubeletCredentials, [Parameter(ParameterSetName='cloud')] [Switch] $force, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity if (-not $sshPrivateKeyFile) { $m = Get-MocConfig $sshPrivateKeyFile = $m["sshPrivateKey"] } if ($PSCmdlet.ParameterSetName -ieq "cloud") { Repair-KvaCerts -Name $Name -sshPrivateKeyFile $sshPrivateKeyFile -force:$force.IsPresent -activity $activity } if($PSCmdlet.ParameterSetName -ieq "kubelet") { Repair-KvaCluster -Name $Name -sshPrivateKeyFile $sshPrivateKeyFile -fixCertificates -activity $activity } Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } function Repair-AksHciCerts { <# .DESCRIPTION Attempts to repair failed TLS on a cluster . .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [string] $sshPrivateKeyFile, [Parameter()] [Switch] $force, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name)" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" throw $_ } Initialize-AksHciEnvironment -activity $activity if (-not $sshPrivateKeyFile) { $m = Get-MocConfig $sshPrivateKeyFile = $m["sshPrivateKey"] } Repair-KvaCerts -sshPrivateKeyFile $sshPrivateKeyFile -force:$force.IsPresent -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } function Sync-AksHciBilling { <# .DESCRIPTION Sync Aks-Hci billing. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $syncResult = Sync-KvaBilling -activity $activity Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $syncResult } function Get-AksHciBillingStatus { <# .DESCRIPTION Get Aks-Hci billing status. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $statusResult = Get-KvaBillingStatus -activity $activity -outputformat "json" | ConvertFrom-Json Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $statusResult } function New-AksHciClusterNetwork { <# .DESCRIPTION Create network settings to be used for the target clusters. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNetDHCP = New-AksHciClusterNetwork -name e1 -vswitchName External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE $clusterVNetStatic = New-AksHciClusterNetwork -name e1 -vswitchName External -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" throw $_ } Initialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return New-KvaClusterNetwork -name $name -vswitchname $vswitchname -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vlanID $vlanID -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend -activity $activity } function Get-AksHciClusterNetwork { <# .DESCRIPTION Gets the VirtualNetwork object for a target cluster given either the vnet name or the cluster name. If no parameter is given, all vnet's are returned. .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] [string] $name, [Parameter(Mandatory=$false)] [string] $clusterName, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" throw $_ } Initialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return Get-KvaClusterNetwork -name $name -clusterName $clusterName -activity $activity } function Remove-AksHciClusterNetwork { <# .DESCRIPTION Remove a virtual network object for a target cluster .PARAMETER name The name of the vnet .NOTES The cmdlet will throw an exception if the network is still being used. The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE Remove-AksHciClusterNetwork -name e1 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter()] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters throw $_ } Initialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters return Remove-KvaClusterNetwork -name $name -activity $activity } function Install-AksHciMonitoring { <# .DESCRIPTION Installs monitoring infrastructure on AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER storageSizeGB Amount of storage for Prometheus in GB .PARAMETER retentionTimeHours metrics retention time in hours. (min 2 hours, max 876000 hours(100 years)) .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [int] $storageSizeGB, [Parameter(Mandatory=$true)] [int] $retentionTimeHours, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Installing monitoring" Set-KvaHciMonitoring -Name $Name -storageSizeGB $storageSizeGB -retentionTimeHours $retentionTimeHours -activity $activity Write-SubStatus -moduleName $moduleName "Monitoring has been installed to the cluster" Write-SubStatus -moduleName $moduleName "To watch progress for the monitoring Onboarding run: kubectl get pods -n monitoring" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Uninstall-AksHciMonitoring { <# .DESCRIPTION Uninstalls monitoring from an AKS-HCI cluster. .PARAMETER Name cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Uninstalling monitoring" Reset-KvaHciMonitoring -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName "Monitoring has been uninstalled from the cluster" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Add-AksHciNode { <# .DESCRIPTION Add new node to the Moc stack during a Failure Replacement Unit scenario .PARAMETER nodeName The name of the node in the Failover Cluster, the node is already expected to have been added to the failover cluster .PARAMETER activity Activity name to use when updating progress .EXAMPLE Add-AksHciNode -nodeName "node1" #> param ( [String]$nodeName, [String]$activity = $MyInvocation.MyCommand.Name ) trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters throw $_ } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Adding Node" New-MocPhysicalNode -nodeName $nodeName -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Remove-AksHciNode { <# .DESCRIPTION Remove a failed node from the Moc stack during a Failure Replacement Unit scenario .PARAMETER nodeName The name of the node in Failover Cluster .PARAMETER activity Activity name to use when updating progress .NOTES If the physical machine is shut down or removed or unreachable on the network prior to the cmdlet this guarntees that it is removed from the cloud-agent maps but not a complete cleaup of that node. .EXAMPLE Remove-AksHciNode -nodeName "node1" #> param ( [String]$nodeName, [String]$activity = $MyInvocation.MyCommand.Name ) trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters throw $_ } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Removing Node" Remove-MocPhysicalNode -nodeName $nodeName -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Done" -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function New-AksHciProxySetting { <# .DESCRIPTION Create proxy settings to be used for the Aks Hci deployment .PARAMETER name A name to associate with the proxy settings .PARAMETER http HTTP proxy server configuration .PARAMETER https HTTPS proxy server configuration .PARAMETER noProxy Proxy server exemption/bypass list .PARAMETER certFile Path to a CA certificate file used to establish trust with a HTTPS proxy server .PARAMETER credential Proxy server credentials (for basic authentication) .OUTPUTS Proxy Settings object .EXAMPLE $credential = Get-Credential $proxySetting = New-AksHciProxySetting -http http://contosoproxy:8080 -https https://contosoproxy:8080 -noProxy "localhost,127.0.0.1,.svc,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" -credential $credential -certFile c:\proxyca.crt #> param ( [Parameter()] [String] $name, [Parameter()] [String] $http, [Parameter()] [String] $https, [Parameter()] [String] $noProxy = $global:defaultProxyExemptions, [Parameter()] [String] $certFile, [Parameter()] [PSCredential] $credential = [PSCredential]::Empty ) trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$_" -CmdletParameters $PSBoundParameters throw $_ } Test-ProxyConfiguration -http $http -https $https -certFile $certFile Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters return [ProxySettings]::new($credential, $name, $http, $https, $noProxy, $certFile) } function Get-AksHciProxySetting { <# .DESCRIPTION Returns AksHci proxy settings .PARAMETER activity Activity name to use when updating progress .OUTPUTS Proxy Settings object #> param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $http = $global:config[$moduleName]["proxyServerHTTP"] $https = $global:config[$moduleName]["proxyServerHTTPS"] $noProxy = $global:config[$moduleName]["proxyServerNoProxy"] $certFile = $global:config[$moduleName]["proxyServerCertFile"] $credentials = [PSCredential]::Empty if ($($global:config[$moduleName]["proxyServerUsername"]) -and $($global:config[$moduleName]["ProxyServerPassword"])) { $securePass = $($global:config[$moduleName]["ProxyServerPassword"]) | ConvertTo-SecureString -Key $global:credentialKey $credentials = New-Object System.Management.Automation.PSCredential -ArgumentList $($global:config[$moduleName]["proxyServerUsername"]), $securePass } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters return [ProxySettings]::new($credentials, "", $http, $https, $noProxy, $certFile) } function New-AksHciContainerRegistry { <# .DESCRIPTION Create container registry settings to be used for the Aks Hci deployment .PARAMETER server The container registry server name .PARAMETER credential Credential to connect to the container registry (if required) .OUTPUTS Container Registry object .EXAMPLE $credential = Get-Credential $registry = New-AksHciContainerRegistry -server "ecpacr.azurecr.io" -credential $credential #> param ( [Parameter(Mandatory=$true)] [String] $server, [Parameter()] [PSCredential] $credential = [PSCredential]::Empty ) trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$_" -CmdletParameters $PSBoundParameters throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters return [ContainerRegistry]::new($credential, $server) } #endregion #region Installation and Provisioning functions function Install-AksHciInternal { <# .DESCRIPTION The main deployment method for AksHci. This function is responsible for installing MOC stack and the management appliance/cluster. .PARAMETER activity Activity name to use when updating progress #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installing) try { # Pre-requisite Install-Moc -activity $activity Get-AksHciPackage -Version (Get-AksHciVersion) Install-Kva -activity $activity } catch { Set-AksHciConfigValue -name "installState" -value ([InstallState]::InstallFailed) throw $_ } Write-Status -moduleName $moduleName "AksHci installation is complete!" Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installed) } function Initialize-AksHciEnvironment { <# .DESCRIPTION Executes steps to prepare the environment for AksHci operations. .PARAMETER createConfigIfNotPresent Whether the call should create a new AksHci deployment configuration if one is not already present. .PARAMETER skipMgmtKubeConfig Whether the call should skip a check to ensure that a appliance/management kubeconfig is present. .PARAMETER activity Activity name to use when updating progress #> param ( [Switch]$createConfigIfNotPresent, [Switch]$skipMgmtKubeConfig, [Switch]$skipInstallationCheck, [parameter(DontShow)] [String]$activity = "Preparing Environment" ) Write-StatusWithProgress -activity $activity -status "Initializing environment" -moduleName $moduleName Import-AksHciConfig -createIfNotPresent:($createConfigIfNotPresent.IsPresent) -activity $activity Initialize-Environment -checkForUpdates:$false -moduleName $script:moduleName if (-not $skipInstallationCheck.IsPresent) { if (-not (Test-IsProductInstalled -moduleName $moduleName -activity $activity)) { throw ("$moduleName is not installed. Please install and then retry this operation.") } } if (-not ($skipMgmtKubeConfig.IsPresent)) { Get-Kva -activity $activity | Out-Null } } function Get-AksHciVersion { <# .SYNOPSIS Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) trap { Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $global:config[$modulename]["version"] } function Update-AksHci { <# .SYNOPSIS Update the Azure Kubernetes Service host to the latest Kubernetes version. .DESCRIPTION Update the Azure Kubernetes Service host to the latest Kubernetes version. Performed as step updates, updating to the next available version until latest version is achieved. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) <# 1. Check if versions later than current version are available a. If yes, prompt for upgrade b. If no, return silenty, printing a message 2. If upgrade is requested, do the following check a. 3. In case of multiple available newer versions a. Run upgrade from current version to next, repeating until latest available version is achieved b. Upon failure, return to last successful version to be upgraded to 4. Handle No target cluster scenarios 5. Handle No Target and Mgmt cluster scenarios 6. Handle scenario when the product is not installed #> if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" throw $_ } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity Set-AksHciConfigValue -name "installState" -value ([InstallState]::Updating) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Updating AksHci" Write-SubStatus -moduleName $moduleName "Current Version :$(Get-AksHciVersion)" $updates = Get-AksHciUpdates if ($updates.Count -eq 0) { Write-SubStatus -moduleName $moduleName "You are on the LATEST version" return } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status "Finding suitable version to upgrade" $currentVersion = (Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName).Version $versionsToUpgrade = For ($v = $($updates.Keys).indexOf($currentVersion) - 1; $v -ge 0; $v-- ) { $tmpUpdate = $updates[$v] if (!$tmpUpdate.CanUpgradeTo) { break } $tmpUpdate.Version } # Check if we are able to find a version to upgrade to if (!$versionsToUpgrade) { throw "An update is available, but the current deployment is not able to update. Please run Get-AksHciUpdates and review the recommendations" } $versionsToUpgrade | ForEach-Object { Write-SubStatus -moduleName $moduleName "Found version to upgrade [$_]" $currentInstallationPath = $global:config[$modulename]["installationPackageDir"] # We found a version to Upgrade to # 1. Download the package Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Getting package version $_") Get-AksHciPackage -Version $_ $currentMocVersion = Get-MocVersion -activity $activity Get-KvaVersion -activity $activity | Out-Null try { $newInstallationPath = [io.Path]::Combine($global:config[$modulename]["workingDir"], $_) Set-AksHciConfigValue -name "installationPackageDir" -value $newInstallationPath New-Item -ItemType Directory -Force -Path $newInstallationPath | Out-Null # Trigger the platform update Update-Moc -activity $activity -version $_ # Trigger the appliance update - What happens when appliance update fails. Update-Kva -activity $activity -version $_ # Set the version, once successful Set-AksHciConfigValue -name "version" -value $_ } catch { Set-AksHciConfigValue -name "installState" -value ([InstallState]::UpdateFailed) Write-SubStatus -moduleName $moduleName $("Warning: Update failed with " + $_.Exception.Message) Write-SubStatus -moduleName $moduleName "Cleaning up updates" # Cleanup and Revert Set-AksHciConfigValue -name "installationPackageDir" -value $currentInstallationPath # Revert the platform Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Reverting platform to version $currentMocVersion") Update-Moc -activity $activity -version $currentMocVersion Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" # Do we need to cleanup the downloaded package - keep it, so we customers may attempt to update again throw "Update Failed [$_]" } # When upgrading from 1.0.1.10628 to 1.0.2.10719, there is a bug which can cause billing to go out of policy. # To prevent this from happening, in a chained upgrade case, force a sync inbetween upgrades. try { Sync-AksHciBilling } catch [Exception] { $errorMessageString = $_.Exception.Message.ToString() Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $("Failed to Sync Billing Status during upgrade: $errorMessageString") Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage "$activity - $_" } } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installed) Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName $currentVersion = (Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName).Version if ($currentVersion -ine $updates[0].Version) { throw "Update to $currentVersion completed, but a later update is still available. The current deployment is not able to update to latest. Please run Get-AksHciUpdates and review the recommendations" } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } #endregion #region Helper Functions function Get-AksHciPackage { <# .DESCRIPTION Downloads the package of the specified AksHCI Version .PARAMETER Version Version #> param ( [Parameter(Mandatory=$true)] [String]$Version ) # Validate the version Get-ProductRelease -Version $Version -moduleName $moduleName | Out-Null } function Get-AksHciLatestVersion { <# .DESCRIPTION Get the latest AksHci version #> $catalog = Get-LatestCatalog -moduleName $moduleName return $catalog.ProductStreamRefs[0].ProductReleases[0].Version } function Get-AksHciUpdates { <# .SYNOPSIS List the available Kubernetes updates for Azure Kubernetes Service on Azure Stack HCI in order. .DESCRIPTION List the available Kubernetes updates for Azure Kubernetes Service on Azure Stack HCI in order from latest to earliest. #> [CmdletBinding()] param () trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$_" throw $_ } Initialize-AksHciEnvironment $latestRelease = Get-LatestRelease -moduleName $moduleName $currentRelease = Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName $latestVersion = $latestRelease.Version $currentVersion = $currentRelease.Version $upgradePath = [ordered]@{} if ($latestVersion -ieq $currentVersion) { return } # There may be more updates that users might have not applied. # Show them the complete list, so they are aware of what will be updated # Assumption here is that product releases would be returned in order $updateReleases = Get-ProductReleasesUptoVersion -Version $currentVersion -moduleName $moduleName $targetKubernetesVersions = Get-TargetClusterKubernetesVersions $updateReleases | ForEach-Object { $tmp = $_ $tmpVersion = $tmp.Version $supportedK8sVersions = Get-AvailableKubernetesVersions -akshciVersion $tmpVersion $computedRelease = @{ Version = $tmpVersion; SupportedKubernetesVersions = $supportedK8sVersions; CanUpgradeTo = $false; } if ($tmpVersion -ieq $currentVersion) { $computedRelease += @{ Comments = "This is your CURRENT Version"; } } if ($tmpVersion -ieq $latestVersion) { $computedRelease += @{ Comments = "This is the LATEST Version"; } } if ($tmpVersion -gt $currentVersion) { $script:canupgrade = $true try { # Validate that this powershell module is compatible with the proposed upgrade Test-ModuleCompatibility -Version $tmpVersion | out-null } catch [Exception] { $computedRelease += @{ Recommendation = $_.Exception.Message.ToString(); } $script:canupgrade = $false Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$_" } if ($script:canupgrade) { # Validate that the current target cluster k8s versions are still supported by the proposed upgrade if ($targetKubernetesVersions -and $targetKubernetesVersions.Count -gt 0) { foreach($targetVersion in $targetKubernetesVersions) { if (-not ($supportedK8sVersions.OrchestratorVersion.Contains($targetVersion))) { $computedRelease += @{ Recommendation = "Target Cluster Kubernetes Version $targetVersion is not in the list of supported Kubernetes versions (" + $supportedK8sVersions.OrchestratorVersion + ") for $tmpVersion. Please upgrade your target clusters to one of the kubernetes versions supported by $tmpVersion to unblock"; } $script:canupgrade = $false break } } } } if ($script:canupgrade) { $computedRelease.CanUpgradeTo = $true $computedRelease += @{ Recommendation = "You can upgrade to AksHci Version [$tmpVersion]"; } } } $upgradePath[$tmpVersion] = $computedRelease; } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return $upgradePath } function Test-SupportedKubernetesVersion { <# .DESCRIPTION Test if the specified kubernetes version is supported by the current deployment .PARAMETER K8sVersion Kubernetes version to test .PARAMETER imageType Image type can be Windows or Linux #> param ( [Parameter(Mandatory=$true)] [String] $K8sVersion, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String] $imageType ) $availableVersions = Get-AvailableKubernetesVersions foreach($version in $availableVersions) { if (($version.OS -ieq $imageType) -and ($version.OrchestratorVersion -ieq $k8sVersion)) { return } } throw "$k8sVersion of type $imageType is not supported in this release.`nUse Get-AksHciKubernetesVersion to see a list of supported versions." } function Get-NextKubernetesVersionForUpgrade { <# .DESCRIPTION Get the next Kubernetes Version for Upgrade. .PARAMETER Name Cluster name .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $upgrades = Get-KvaClusterUpgrades -Name $Name -activity $activity if ($upgrades.AvailableUpgrades.Count -eq 0) { return $null } $kubernetesVersionArray = @() foreach($availableVersion in $upgrades.AvailableUpgrades) { $kubernetesVersionArray += Get-CleanInputKubernetesVersion -KubernetesVersion $availableVersion.kubernetesVersion -Semver } $sorted = $kubernetesVersionArray | ForEach-Object { new-object System.Version ($_) } | Sort-Object -Descending $highestUpgradeAvailable = $sorted[0].ToString() return "v$highestUpgradeAvailable" } function Get-CleanInputKubernetesVersion { <# .DESCRIPTION Cleans the input kubernetes verison .PARAMETER KubernetesVersion KubernetesVersion string to be cleaned. .PARAMETER Semver Semver switch to enforce semver valid output. #> param ( [Parameter(Mandatory=$true)] [String]$KubernetesVersion, [Parameter()] [Switch] $Semver ) $splitVersion = $KubernetesVersion.Split("-") if ($Semver.IsPresent) { $cleanVersion = $splitVersion[0] -replace '[v]','' } else { $cleanVersion = $splitVersion[0] } return $cleanVersion } function Get-AvailableKubernetesVersions { <# .DESCRIPTION Returns the kubernetes versions (by OS) that are supported by the specified AksHci release .PARAMETER akshciVersion AksHci Release version. Defaults to the version of the current deployment #> param ( [Parameter()] [String] $akshciVersion ) $result = @() if (-not $akshciVersion) { $akshciVersion = Get-AksHciVersion } # Get the Manifest for the specified Version $productRelease = Get-ProductRelease -version $akshciVersion -module $moduleName foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { foreach ($fileRelease in $subProductRelease.ProductFiles) { if (-not $fileRelease.CustomData.K8sPackages) { continue } $fileRelease.CustomData.K8SPackages | ForEach-Object { $version = [ordered]@{ 'OrchestratorType' = "Kubernetes"; 'OrchestratorVersion' = $("v"+$_.Version); 'OS' = $fileRelease.CustomData.BaseOSImage.OperatingSystem; 'IsPreview' = $false } $result += New-Object -TypeName PsObject -Property $version } } } } return $result } function Confirm-Configuration { <# .DESCRIPTION Validates the configuration .PARAMETER useStagingShare Requests a staging share to be used for downloading binaries and images (for private testing) .PARAMETER stagingShare The staging share endpoint to use when useStagingShare is requested .PARAMETER vnet Vnet to be validated, if provided #> param ( [Switch] $useStagingShare, [String] $stagingShare, [VirtualNetwork] $vnet ) if ($useStagingShare.IsPresent -and [string]::IsNullOrWhiteSpace($stagingShare)) { throw "-useStagingShare was requested, but no staging share was specified" } # Test networking on hosts if ($vnet) { Confirm-Vnet -vnet $vnet } } function Set-AksHciRegistration { <# .DESCRIPTION Register an AksHci with Azure. Calls Connect-AzAccount under the covers. .PARAMETER SubscriptionId SubscriptionId is an azure subscription id. .PARAMETER TenantId TenantId is an azure tenant id. .PARAMETER ArmAccessToken ArmAccessToken is the token for accessing arm. .PARAMETER GraphAccessToken GraphAccessToken is the token for accessing the graph. .PARAMETER AccountId AccountId is an azure account id. .PARAMETER EnvironmentName EnvironmentName is the intented public cloud. .PARAMETER Credential Credential is a PSCredential holding a user's Service Principal. .PARAMETER ResourceGroupName ResourceGroupName is the name of the azure resource group to place arc resources. .PARAMETER Region Region is the name of the azure resource group to place arc resources. .PARAMETER UseDeviceAuthentication UseDeviceAuthentication outputs a code to be used in the browser. .PARAMETER SkipLogin SkipLogin skips the Connect-AzAccount call. Useful in automation or when running from a connected shell. .PARAMETER activity Activity name to use when updating progress #> param( [Parameter(Mandatory = $true)] [string] $SubscriptionId, [Parameter(Mandatory = $false)] [string] $TenantId, [Parameter(Mandatory = $false)] [string] $ArmAccessToken, [Parameter(Mandatory = $false)] [string] $GraphAccessToken, [Parameter(Mandatory = $false)] [string] $AccountId, [Parameter(Mandatory = $false)] [string] $EnvironmentName = $global:azureCloud, [Parameter(Mandatory = $true)] [string] $ResourceGroupName, [Parameter(Mandatory = $false)] [string] $Region, [Parameter(Mandatory = $false)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [Switch] $UseDeviceAuthentication, [Parameter(Mandatory = $false)] [Switch] $SkipLogin, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage "$activity - $_" if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck if (-not $SkipLogin.IsPresent) { Set-AzureLogin -SubscriptionId $SubscriptionId -TenantId $TenantId -ArmAccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId -EnvironmentName $EnvironmentName -Credential $Credential -UseDeviceAuthentication:$UseDeviceAuthentication.IsPresent } $kubernetesProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes $kubernetesConfigProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration # The RPs should always exist but just in case arm is down, bail out. if (($null -eq $kubernetesProvider) -or ($null -eq $kubernetesProvider)) { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ChildStageName "KubernetesProvidersNotExists" ` -ErrorMessage "Unable to check registered Resource Providers" throw "Unable to check the registered Resource Providers. Please run Set-AksHciRegistration again." } if (($kubernetesProvider[0].RegistrationState -ne "Registered") -or ($kubernetesConfigProvider[0].RegistrationState -ne "Registered")) { Write-Status -moduleName $moduleName -Verbose -msg " Kubernetes Resource Providers are not registered for the current logged in tenant. Please run the following commands. With the azure cli: az provider register --namespace Microsoft.Kubernetes az provider register --namespace Microsoft.KubernetesConfiguration With Azure Powershell: Register-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes Register-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration Registration is an asynchronous process and may take approximately 10 minutes. You can monitor the registration process with the following commands: With the azure cli: az provider show -n Microsoft.Kubernetes -o table az provider show -n Microsoft.KubernetesConfiguration -o table With Azure Powershell: Get-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes Get-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration " throw "Kubernetes Resource Providers are not registered for the current logged in tenant." } if ($Region -eq "") { $rg = Get-AzResourceGroup -Name $ResourceGroupName $Region = $rg.Location.ToLower().replace(' ', '') } $isValidLocation = $false # in the case of an invalid location, build a string of all the locations to return to the user. $locationErrorString = "" foreach($location in $kubernetesProvider.Locations) { $cleanLocation = $location.ToLower().replace(' ', '') if ($Region -eq $cleanLocation) { $isValidLocation = $true } $locationErrorString += "$cleanLocation," } if (-not $isValidLocation) { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ChildStageName "RegionNotValid" ` -ErrorMessage "$Region is not a valid Region for AksHci" throw "$Region is not a valid Region for AksHci. Please use a resource group in one of the following locations. $locationErrorString" } Set-KvaRegistration -azureResourceGroup $ResourceGroupName -azureLocation $Region Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) } function Get-AksHciRegistration { <# .DESCRIPTION Gets the Registration for AksHci. #> trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$_" throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) return Get-KvaRegistration } function Set-AzureLogin { <# .DESCRIPTION Performs an Azure Login. Calls Connect-AzAccount under the covers. .PARAMETER SubscriptionId SubscriptionId .PARAMETER TenantId TenantId .PARAMETER ArmAccessToken ArmAccessToken .PARAMETER GraphAccessToken GraphAccessToken .PARAMETER AccountId AccountId .PARAMETER EnvironmentName EnvironmentName .PARAMETER Credential Credential .PARAMETER UseDeviceAuthentication UseDeviceAuthentication .PARAMETER activity Activity name to use when updating progress #> param( [Parameter(Mandatory = $true)] [string] $SubscriptionId, [Parameter(Mandatory = $false)] [string] $TenantId, [Parameter(Mandatory = $false)] [string] $ArmAccessToken, [Parameter(Mandatory = $false)] [string] $GraphAccessToken, [Parameter(Mandatory = $false)] [string] $AccountId, [Parameter(Mandatory = $false)] [string] $EnvironmentName, [Parameter(Mandatory = $false)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [Switch] $UseDeviceAuthentication, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) if($EnvironmentName -eq $AzurePPE) { Add-AzEnvironment -Name $AzurePPE -PublishSettingsFileUrl "https://windows.azure-test.net/publishsettings/index" -ServiceEndpoint "https://management-preview.core.windows-int.net/" -ManagementPortalUrl "https://windows.azure-test.net/" -ActiveDirectoryEndpoint "https://login.windows-ppe.net/" -ActiveDirectoryServiceEndpointResourceId "https://management.core.windows.net/" -ResourceManagerEndpoint "https://api-dogfood.resources.windows-int.net/" -GalleryEndpoint "https://df.gallery.azure-test.net/" -GraphEndpoint "https://graph.ppe.windows.net/" -GraphAudience "https://graph.ppe.windows.net/" | Out-Null } Disconnect-AzAccount | Out-Null if($null -ne $Credential) { if ([string]::IsNullOrEmpty($TenantId)) { throw "Parameter Credential was passed in, but TenantId is empty. Both are needed for Service Principal Login" } else { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -Credential $Credential -ServicePrincipal | Out-Null } } elseif([string]::IsNullOrEmpty($ArmAccessToken) -or [string]::IsNullOrEmpty($GraphAccessToken) -or [string]::IsNullOrEmpty($AccountId)) { # Interactive login $IsIEPresent = Test-Path "$env:SystemRoot\System32\ieframe.dll" if([string]::IsNullOrEmpty($TenantId)) { if($IsIEPresent -and (-not $UseDeviceAuthentication)) { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId | Out-Null } else # Use -UseDeviceAuthentication as IE Frame is not available to show Azure login popup { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId -UseDeviceAuthentication | Out-Null } } else { if($IsIEPresent -and (-not $UseDeviceAuthentication)) { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId | Out-Null } else # Use -UseDeviceAuthentication as IE Frame is not available to show Azure login popup { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -UseDeviceAuthentication | Out-Null } } } else { # Not an interactive login if([string]::IsNullOrEmpty($TenantId)) { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId -AccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId | Out-Null } else { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -AccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId | Out-Null } } } function New-AksHciStorageContainer { <# .DESCRIPTION Creates a new cloud storage container .PARAMETER activity Activity name to use when updating progress .PARAMETER Name The name of the new storage container .PARAMETER Path The path where the vhds will be stored #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$Path ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Creating a new storage container" -moduleName $moduleName $cloudLocation = (Get-MocConfig)["cloudLocation"] New-MocContainer -name $Name -path $Path -location $cloudLocation Write-SubStatus -moduleName $moduleName "Storage container has been created" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Get-AksHciStorageContainer { <# .DESCRIPTION Gets the storage containers .PARAMETER activity Activity name to use when updating progress .PARAMETER Name The name of the storage container, if not present returns all #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter()] [String]$Name ) trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage "$activity - $_" ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Gathering storage container information" -moduleName $moduleName $cloudLocation = (Get-MocConfig)["cloudLocation"] $result = Get-MocContainer -name $Name -location $cloudLocation Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters return $result } function Install-AksHciCsiSmb { <# .DESCRIPTION Installs csi smb plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Installing CSI SMB plugin" -moduleName $moduleName Set-KvaCsiSmb -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName "CSI SMB has been installed to the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Install-AksHciCsiNfs { <# .DESCRIPTION Installs csi nfs plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Installing CSI NFS plugin" -moduleName $moduleName Set-KvaCsiNfs -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName "CSI NFS has been installed to the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Uninstall-AksHciCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Uninstalling CSI SMB plugin" -moduleName $moduleName Reset-KvaCsiSmb -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName "CSI SMB has been uninstalled from the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Uninstall-AksHciCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage "$activity - $_" -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status "Uninstalling CSI NFS plugin" -moduleName $moduleName Reset-KvaCsiNfs -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName "CSI NFS has been uninstalled from the cluster" Write-StatusWithProgress -activity $activity -status "Done" -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters } function Confirm-Vnet { <# .DESCRIPTION Validate if the vnet configuration is valid #> param( [VirtualNetwork] $vnet ) $isMultinode = Test-MultiNodeDeployment $vSwitchName = $vnet.VswitchName $vipPoolStart = $vnet.VipPoolStart $vipPoolEnd = $vnet.VipPoolEnd if ($isMultinode) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Test-VipPoolAgainstVnicAddressPrefix -switchName $vSwitchName -multiNode -nodeName $_.Name -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd -vlanID $vnet.Vlanid } } else { Test-VipPoolAgainstVnicAddressPrefix -switchName $vSwitchName -nodeName ($env:computername) -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd -vlanID $vnet.Vlanid } } function Get-TraceConfigDetails { <# .DESCRIPTION Capture the config map details for the TraceCmdlet module. #> [TraceConfigDetails]$traceConfig = [TraceConfigDetails]::new() if (-not [string]::IsNullOrWhiteSpace($global:config[$modulename]["version"])) { $productRelease = Get-ProductRelease -version $global:config[$modulename]["version"] -moduleName $moduleName $traceConfig.Offer = "$($productRelease.ProductName)" } $traceConfig.DeploymentId = "$($global:config[$modulename]["deploymentId"])" $traceConfig.Catalog = "$($global:config[$modulename]["catalog"])" $traceConfig.Audience = "$($global:config[$modulename]["ring"])" $traceConfig.AksHciVersion = "$($global:config[$modulename]["version"])" $traceConfig.ModuleName = $moduleName $traceConfig.ModuleVersion = $moduleVersion return $traceConfig | ConvertTo-Json } #endregion # SIG # Begin signature block # MIIjnwYJKoZIhvcNAQcCoIIjkDCCI4wCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCqPAzJjbn6iLJ5 # eLbRWl9j6uIt9pnZjvlbZFxtW3OZyaCCDYEwggX/MIID56ADAgECAhMzAAAB32vw # LpKnSrTQAAAAAAHfMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjAxMjE1MjEzMTQ1WhcNMjExMjAyMjEzMTQ1WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQC2uxlZEACjqfHkuFyoCwfL25ofI9DZWKt4wEj3JBQ48GPt1UsDv834CcoUUPMn # s/6CtPoaQ4Thy/kbOOg/zJAnrJeiMQqRe2Lsdb/NSI2gXXX9lad1/yPUDOXo4GNw # PjXq1JZi+HZV91bUr6ZjzePj1g+bepsqd/HC1XScj0fT3aAxLRykJSzExEBmU9eS # yuOwUuq+CriudQtWGMdJU650v/KmzfM46Y6lo/MCnnpvz3zEL7PMdUdwqj/nYhGG # 3UVILxX7tAdMbz7LN+6WOIpT1A41rwaoOVnv+8Ua94HwhjZmu1S73yeV7RZZNxoh # EegJi9YYssXa7UZUUkCCA+KnAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUOPbML8IdkNGtCfMmVPtvI6VZ8+Mw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDYzMDA5MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAnnqH # tDyYUFaVAkvAK0eqq6nhoL95SZQu3RnpZ7tdQ89QR3++7A+4hrr7V4xxmkB5BObS # 0YK+MALE02atjwWgPdpYQ68WdLGroJZHkbZdgERG+7tETFl3aKF4KpoSaGOskZXp # TPnCaMo2PXoAMVMGpsQEQswimZq3IQ3nRQfBlJ0PoMMcN/+Pks8ZTL1BoPYsJpok # t6cql59q6CypZYIwgyJ892HpttybHKg1ZtQLUlSXccRMlugPgEcNZJagPEgPYni4 # b11snjRAgf0dyQ0zI9aLXqTxWUU5pCIFiPT0b2wsxzRqCtyGqpkGM8P9GazO8eao # mVItCYBcJSByBx/pS0cSYwBBHAZxJODUqxSXoSGDvmTfqUJXntnWkL4okok1FiCD # Z4jpyXOQunb6egIXvkgQ7jb2uO26Ow0m8RwleDvhOMrnHsupiOPbozKroSa6paFt # VSh89abUSooR8QdZciemmoFhcWkEwFg4spzvYNP4nIs193261WyTaRMZoceGun7G # CT2Rl653uUj+F+g94c63AhzSq4khdL4HlFIP2ePv29smfUnHtGq6yYFDLnT0q/Y+ # Di3jwloF8EWkkHRtSuXlFUbTmwr/lDDgbpZiKhLS7CBTDj32I0L5i532+uHczw82 # oZDmYmYmIUSMbZOgS65h797rj5JJ6OkeEUJoAVwwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVdDCCFXACAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAd9r8C6Sp0q00AAAAAAB3zAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQgnZeVsoTK # SUF4UJD+qgUj16PtZV+WmZP8squHMuasnOEwQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQACLUsL7WlFX9zf+dhPma+uov9nrn+L0v7GekTXCisi # 5bX/GW0L5CTe0AfG8xlLIBf5FSBWkKeeL95YBGBJXaxnunckOXhf63CJDfnUG2Me # vlUKkW3LGo/ZjurbTF2idaD0YKEBxYRaFFPlIm3lafYybah51cndBQge644ykeOh # /3jtaljUtm7q5VkhGRce1efk6QD4lRac42MVhWmb8dE5hssxzGeMznp33B59lGUx # ly4vaAg7jFFTX1i39LN2nwPYRMuOklm6ZGT61bTLBfRBPpqAhGYDYuk1AM09vkyt # NmhObszfJ0v2nGZf99z09DvQeP2IlPdSjlHFd524r/DLoYIS/jCCEvoGCisGAQQB # gjcDAwExghLqMIIS5gYJKoZIhvcNAQcCoIIS1zCCEtMCAQMxDzANBglghkgBZQME # AgEFADCCAVkGCyqGSIb3DQEJEAEEoIIBSASCAUQwggFAAgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEIHIjeO44dtq9/jT04jVP7KlB1rTjqFPpalhnC6tB # RgomAgZhSLiWI6gYEzIwMjExMDAxMTcyMDUzLjg2MlowBIACAfSggdikgdUwgdIx # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1p # Y3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEmMCQGA1UECxMdVGhh # bGVzIFRTUyBFU046MkFENC00QjkyLUZBMDExJTAjBgNVBAMTHE1pY3Jvc29mdCBU # aW1lLVN0YW1wIFNlcnZpY2Wggg5NMIIE+TCCA+GgAwIBAgITMwAAATjzlCHWYb7w # KQAAAAABODANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0Eg # MjAxMDAeFw0yMDEwMTUxNzI4MjBaFw0yMjAxMTIxNzI4MjBaMIHSMQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQg # SXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1Mg # RVNOOjJBRDQtNEI5Mi1GQTAxMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFt # cCBTZXJ2aWNlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxbn4p6le # cIMyAgKjbK7+A/vdqY+y1pmgXIjOwXGvDAjy7aKPsp978FH+VPHh+3Yk9tc0ET35 # 54bbS7TrObejbAH8bQ3UMqT7sU79nkALxKqykp2lYfI/ZvrDVBecswbz1v/9CZcT # drSailbNDIg1lTfSA0LDhebTt02J+R5eCZrhfXedCWJP1kt1jy0E5wJUryqYOhXs # rgewGdQsdH2bvp5JfRip+vg8rwVAJ78RHbbT5xTof+oFLOCAgmJG0e2yC7PAItEr # KPMWjM86pkVKR6atoVKuA5oG4d4NWktiUzKT2bynwlVkx74uu6rF7U+56udCwwk2 # mnNjD+OXDhyPGQIDAQABo4IBGzCCARcwHQYDVR0OBBYEFN/qrdzo76xTsTL7OYjF # YMGeccC1MB8GA1UdIwQYMBaAFNVjOlyKMZDzQ3t8RhvFM2hahW1VMFYGA1UdHwRP # ME0wS6BJoEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1 # Y3RzL01pY1RpbVN0YVBDQV8yMDEwLTA3LTAxLmNybDBaBggrBgEFBQcBAQROMEww # SgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMv # TWljVGltU3RhUENBXzIwMTAtMDctMDEuY3J0MAwGA1UdEwEB/wQCMAAwEwYDVR0l # BAwwCgYIKwYBBQUHAwgwDQYJKoZIhvcNAQELBQADggEBAKN4sEhcC7G/DRKQy9LI # to5OI5VZUgS3SBFZrNHsr/ZzR5MsaiY3eVxm9zZmbg+m4utdCi8FcUdPScD6r8Fo # tfHO3dF/I6rjXej/sGbNv7h8HsBPWmL3Se+mZ+//IQrFi5rktqxs6LSkCPirmMhY # Z6hUfJgN4PgtkG/mwqSqF04f74G8A2JHwhDzsLBi4MYBZDT4KLJ9KAOgWZAmS4O3 # MAYxzsfbsN0WsjMgDMq8B2XqQNzILESwkAPRJKWXfX73C6IJS5MHWarGIj+BygDs # 5p6M28w53sXPWAKNAqt7ZGBaspG+k/t3xfWAm8eD4CnvEKSVM8Vffr2HwG+CUgAC # ppowggZxMIIEWaADAgECAgphCYEqAAAAAAACMA0GCSqGSIb3DQEBCwUAMIGIMQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNy # b3NvZnQgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0xMDA3MDEy # MTM2NTVaFw0yNTA3MDEyMTQ2NTVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAy # MDEwMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqR0NvHcRijog7PwT # l/X6f2mUa3RUENWlCgCChfvtfGhLLF/Fw+Vhwna3PmYrW/AVUycEMR9BGxqVHc4J # E458YTBZsTBED/FgiIRUQwzXTbg4CLNC3ZOs1nMwVyaCo0UN0Or1R4HNvyRgMlhg # RvJYR4YyhB50YWeRX4FUsc+TTJLBxKZd0WETbijGGvmGgLvfYfxGwScdJGcSchoh # iq9LZIlQYrFd/XcfPfBXday9ikJNQFHRD5wGPmd/9WbAA5ZEfu/QS/1u5ZrKsajy # eioKMfDaTgaRtogINeh4HLDpmc085y9Euqf03GS9pAHBIAmTeM38vMDJRF1eFpwB # BU8iTQIDAQABo4IB5jCCAeIwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFNVj # OlyKMZDzQ3t8RhvFM2hahW1VMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsG # A1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2VsuP6KJc # YmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwubWljcm9z # b2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIz # LmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cubWlj # cm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3J0 # MIGgBgNVHSABAf8EgZUwgZIwgY8GCSsGAQQBgjcuAzCBgTA9BggrBgEFBQcCARYx # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL1BLSS9kb2NzL0NQUy9kZWZhdWx0Lmh0 # bTBABggrBgEFBQcCAjA0HjIgHQBMAGUAZwBhAGwAXwBQAG8AbABpAGMAeQBfAFMA # dABhAHQAZQBtAGUAbgB0AC4gHTANBgkqhkiG9w0BAQsFAAOCAgEAB+aIUQ3ixuCY # P4FxAz2do6Ehb7Prpsz1Mb7PBeKp/vpXbRkws8LFZslq3/Xn8Hi9x6ieJeP5vO1r # VFcIK1GCRBL7uVOMzPRgEop2zEBAQZvcXBf/XPleFzWYJFZLdO9CEMivv3/Gf/I3 # fVo/HPKZeUqRUgCvOA8X9S95gWXZqbVr5MfO9sp6AG9LMEQkIjzP7QOllo9ZKby2 # /QThcJ8ySif9Va8v/rbljjO7Yl+a21dA6fHOmWaQjP9qYn/dxUoLkSbiOewZSnFj # nXshbcOco6I8+n99lmqQeKZt0uGc+R38ONiU9MalCpaGpL2eGq4EQoO4tYCbIjgg # tSXlZOz39L9+Y1klD3ouOVd2onGqBooPiRa6YacRy5rYDkeagMXQzafQ732D8OE7 # cQnfXXSYIghh2rBQHm+98eEA3+cxB6STOvdlR3jo+KhIq/fecn5ha293qYHLpwms # ObvsxsvYgrRyzR30uIUBHoD7G4kqVDmyW9rIDVWZeodzOwjmmC3qjeAzLhIp9cAv # VCch98isTtoouLGp25ayp0Kiyc8ZQU3ghvkqmqMRZjDTu3QyS99je/WZii8bxyGv # WbWu3EQ8l1Bx16HSxVXjad5XwdHeMMD9zOZN+w2/XU/pnR4ZOC+8z1gFLu8NoFA1 # 2u8JJxzVs341Hgi62jbb01+P3nSISRKhggLXMIICQAIBATCCAQChgdikgdUwgdIx # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1p # Y3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEmMCQGA1UECxMdVGhh # bGVzIFRTUyBFU046MkFENC00QjkyLUZBMDExJTAjBgNVBAMTHE1pY3Jvc29mdCBU # aW1lLVN0YW1wIFNlcnZpY2WiIwoBATAHBgUrDgMCGgMVAEC86zs20AKnzakuISFL # JspZ5oH5oIGDMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwDQYJ # KoZIhvcNAQEFBQACBQDlAbcZMCIYDzIwMjExMDAyMDAzNDMzWhgPMjAyMTEwMDMw # MDM0MzNaMHcwPQYKKwYBBAGEWQoEATEvMC0wCgIFAOUBtxkCAQAwCgIBAAICCeYC # Af8wBwIBAAICEgIwCgIFAOUDCJkCAQAwNgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYB # BAGEWQoDAqAKMAgCAQACAwehIKEKMAgCAQACAwGGoDANBgkqhkiG9w0BAQUFAAOB # gQCILjlqjDG275G9/dCDmCgczAu5c+tGsZTh7DdEByzyTYG8LO+YbFVEb4NDQoqL # eVVfff3vP7YjD49q+PwptJd47y8PaaHbcDZEF6kfQPOT9oO87HMgUXmD6iP72LZ6 # vEtfNtCaXeMiX5407QZpz9RG+13Ctwex9q7fM+9W6n9QRTGCAw0wggMJAgEBMIGT # MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMT # HU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAABOPOUIdZhvvApAAAA # AAE4MA0GCWCGSAFlAwQCAQUAoIIBSjAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQ # AQQwLwYJKoZIhvcNAQkEMSIEIO3mjuuYWbWtPD7pqv2AhtT6O3HaeJ5IhGL+8ck+ # SBVDMIH6BgsqhkiG9w0BCRACLzGB6jCB5zCB5DCBvQQgQ0CTSvl/RCqSSGLiLKbu # 7sbS3mjmCOpO+ith7RaPhygwgZgwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UE # CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z # b2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQ # Q0EgMjAxMAITMwAAATjzlCHWYb7wKQAAAAABODAiBCDQQ3YFCsK3PkGCTggD/ZnD # 1C3pwroVa+NWytir7U/1UTANBgkqhkiG9w0BAQsFAASCAQBWcty7HU+u4eEaylbM # /S0zr3GmicS3P7lU/YupIzmJ09g5Oann+TlYx7tuk085S8k4CYmtCYnexFhE94Kw # cidj0XmKnSIWrYxKXPVxJwAAdJJWufqwB3yCUboCuH1uPmrELuIROiti/lpkMVBm # moUx4tAXeOeAiCZGAqa+ywVI2zWyD0a2cEGw4zWaZcbJFySj+u+J7kNcSCeS7QuS # SAMg75bXt7MI+f2XC/ibayUbhGvQQ/j/nSILQJulAMcl2IWSw23y7Y0XWfMzqBN6 # IoiezGoZh9vDZ3FSBp66rQFYYF6cTTB6G4F+t4T9cS3fFDVC0ZoGUI6XW9JgXptL # U+FG # SIG # End signature block |