AksHci.psm1
|
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # AksHci Day 0/2 Operations # ######################################################################################### #requires -runasadministrator using module .\Common.psm1 #region Module Constants $moduleName = "AksHci" $moduleVersion = "1.1.39" #endregion #region requires #Requires -Modules @{ModuleName="Az.Resources"; RequiredVersion="4.4.0"; GUID="48bb344d-4c24-441e-8ea0-589947784700"} #Requires -Modules @{ModuleName="Az.Accounts"; RequiredVersion="2.6.0"; GUID="17a2feff-488b-47f9-8729-e2cec094624c"} #Requires -Modules @{ModuleName="AzureAD"; RequiredVersion="2.0.2.137"; GUID="d60c0004-962d-4dfb-8d28-5707572ffd00"} #endregion #region Download catalog constants $catalogName = "aks-hci-stable-catalogs-ext" $ringName = "stable" # Major version is checked for compatibilty between PS and product $supportedProductVersion = "1.0.0" #endregion #region Aliases Set-Alias -Name Initialize-AksHciNode -Value Initialize-MocNode Set-Alias -Name Update-AksHciClusterCertificates -Value Repair-AksHciClusterCerts Set-Alias -Name Update-AksHciCertificates -Value Repair-AksHciCerts #endregion #region # Install Event Log New-ModuleEventLog -moduleName $moduleName #endregion #region to capture TraceCmdlet Configmap details class TraceConfigDetails{ [string] $DeploymentId [string] $Catalog [string] $Audience [string] $AksHciVersion [string] $ModuleName [string] $Offer [string] $ModuleVersion } #endregion #region Private Function function Initialize-AksHciConfiguration { <# .DESCRIPTION Initialize AksHci Configuration Wipes off any existing cached configuration #> if ($global:config.ContainsKey($moduleName)) { $global:config.Remove($moduleName) } $global:config += @{ $moduleName = @{ "installationPackageDir" = "" "installState" = [InstallState]::NotInstalled "manifestCache" = "" "moduleVersion" = $moduleVersion "skipUpdates" = $false "stagingShare" = "" "useStagingShare" = $false "version" = "" "workingDir" = "" "catalog" = "" "ring" = "" "proxyServerCertFile" = "" "proxyServerHTTP" = "" "proxyServerHTTPS" = "" "proxyServerNoProxy" = "" "proxyServerPassword" = "" "proxyServerUsername" = "" "deploymentId" = "" "caCertRotationThreshold" = 90 }; } } #endregion #region global config Initialize-AksHciConfiguration #endregion Import-LocalizedData -BindingVariable "GenericLocMessage" -FileName commonLocalizationMessages Import-LocalizedData -BindingVariable "AksHciLocMessage" -FileName AksHciLocalizationMessages #region Exported Functions function New-AksHciNetworkSetting { <# .SYNOPSIS Create an object for a new virtual network. .DESCRIPTION Create a virtual network to set the DHCP or static IP address for the control plane, load balancer, agent endpoints, and a static IP range for nodes in all Kubernetes clusters. This cmdlet will return a VirtualNetwork object, which can be used later in the configuration steps. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER MacPoolName The name of the mac pool .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .EXAMPLE New-AksHciNetworkSetting -name External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE New-AksHciNetworkSetting -name "Defualt Switch" -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [String] $MacPoolName = $global:cloudMacPool, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend ) $startCmdletTime = Get-Date $networkdetailsCmdletParams = @{name= $name; vswitchName= $vswitchName; MacPoolName= $MacPoolName; gateway= $gateway; dnsservers= $dnsservers; vippoolstart= $vippoolstart; vippoolend= $vippoolend; k8snodeippoolstart= $k8snodeippoolstart; k8snodeippoolend= $k8snodeippoolend} trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $networkdetailsCmdletParams throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $networkdetailsCmdletParams return New-VirtualNetwork -name $name -vswitchName $vswitchName -MacPoolName $MacPoolName -vlanID $vlanID -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend } function Test-ModuleCompatibility { <# .DESCRIPTION Tests if the requested product version is compatible with the version(s) understood by this Powershell module. .PARAMETER Version The AKS HCI product version to be tested for compatibility. #> param ( [String] $Version ) $result = Compare-Versions -Version $script:supportedProductVersion -ComparisonVersion $Version if ($result -eq 0) { return $true } $errorMsg = $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_incompatible_version, $Version, $moduleName, $moduleVersion)) if ($result -lt 0) { $errorMsg += $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_older_version, $moduleName)) } else { $errorMsg += $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_newer_version, $moduleName)) } throw $errorMsg } function Set-AksHciConfig { <# .SYNOPSIS Set or update the configurations settings for the Azure Kubernetes Service host. .DESCRIPTION Set the configuration settings for the Azure Kubernetes Service host. If you're deploying on a 2-4 node Azure Stack HCI cluster or a Windows Server 2019 Datacenter failover cluster, you must specify the imageDir and cloudConfigLocation parameters. For a single node Windows Server 2019 Datacenter, all parameters are optional and set to their default values. However, for optimal performance, we recommend using a 2-4 node Azure Stack HCI cluster deployment. .PARAMETER workingDir This is a working directory for the module to use for storing small files. Defaults to %systemdrive%\akshci for single node deployments. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as c:\ClusterStorage\Volume2\ImageStore or an SMB share such as \\FileShare\ImageStore. .PARAMETER imageDir The path to the directory where Azure Kubernetes Service on Azure Stack HCI will store its VHD images. Defaults to %systemdrive%\AksHciImageStore for single node deployments. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as C:\ClusterStorage\Volume2\ImageStore or a SMB share such as \\fileshare\ImageStore. .PARAMETER version The version of Azure Kubernetes Service on Azure Stack HCI that you want to deploy. The default is the latest version. We do not recommend changing the default. .PARAMETER cloudConfigLocation The location where the cloud agent will store its configuration. Defaults to %systemdrive%\wssdcloudagent for single node deployments. The location can be the same as the path of -imageDir. For multi-node deployments, this parameter must be specified. The path must point to a shared storage path such as C:\ClusterStorage\Volume2\ImageStore or an SMB share such as \\fileshare\ImageStore. The location needs to be on a highly available share so that the storage will always be accessible. .PARAMETER nodeConfigLocation The location where the node agents will store their configuration. Every node has a node agent, so its configuration is local to it. This location must be a local path. Defaults to %systemdrive%\programdata\wssdagent for all deployments. .PARAMETER cloudLocation This parameter provides a custom Microsoft Operated Cloud location name. The default name is "MocLocation". We do not recommend changing the default. .PARAMETER createAutoConfigContainers This parameter enables or disables generating auto-config-container- folders for Moc. The default value is $true. .PARAMETER vnet A VirtualNetwork object created using the New-AksHciNetworkSetting cmdlet. .PARAMETER controlplaneVmSize The size of the VM to create for the control plane. To get a list of available VM sizes, use Get-AksHciVmSize. .PARAMETER kvaName Kubernetes Virtual Appliance name. We do not recommend changing the default. .PARAMETER kvaPodCIDR Configures the Kubernetes POD CIDR. We do not recommend changing the default. .PARAMETER nodeAgentPort The TCP/IP port number that node agents should listen on. Defaults to 45000. We do not recommend changing the default. .PARAMETER nodeAgentAuthorizerPort The TCP/IP port number that node agents should use for their authorization port. Defaults to 45001. We do not recommend changing the default. .PARAMETER cloudAgentPort The TCP/IP port number that cloud agent should listen on. Defaults to 55000. We do not recommend changing the default. .PARAMETER cloudAgentAuthorizerPort The TCP/IP port number that cloud agent should use for its authorization port. Defaults to 65000. We do not recommend changing the default. .PARAMETER clusterRoleName This specifies the name to use when creating cloud agent as a generic service within the cluster. This defaults to a unique name with a prefix of ca- and a guid suffix (for example: "ca-9e6eb299-bc0b-4f00-9fd7-942843820c26"). We do not recommend changing the default. .PARAMETER cloudServiceCidr This can be used to provide a static IP/network prefix to be assigned to the MOC CloudAgent service. This value should be provided using the CIDR format. (Example: 192.168.1.2/16). You may want to specify this to ensure that anything important on the network is always accessible because the IP address will not change. Default is none. .PARAMETER proxySettings A ProxySettings object created using the New-AksHciProxySetting cmdlet. .PARAMETER sshPublicKey Path to an SSH public key file. Using this public key, you will be able to log in to any of the VMs created by the Azure Kubernetes Service on Azure Stack HCI deployment. If you have your own SSH public key, you will pass its location here. If no key is provided, we will look for one under %systemdrive%\akshci\.ssh\akshci_rsa.pub. If the file does not exist, an SSH key pair in the above location will be generated and used. .PARAMETER skipHostLimitChecks Requests the script to skip any checks it does to confirm memory and disk space is available before allowing the deployment to proceed. We do not recommend using this setting. .PARAMETER skipRemotingChecks Requests the script to skip any checks it does to confirm remoting capabilities to both local and remote nodes. We do not recommend using this setting. .PARAMETER insecure Deploys Azure Kubernetes Service on Azure Stack HCI components such as cloud agent and node agent(s) in insecure mode (no TLS secured connections). We do not recommend using insecure mode in production environments. .PARAMETER forceDnsReplication DNS replication can take up to an hour on some systems. This will cause the deployment to be slow. To bypass this issue, try to use this flag. The -forceDnsReplication flag is not a guaranteed fix. If the logic behind the flag fails, the error will be hidden, and the command will carry on as if the flag was not provided. .PARAMETER macPoolStart This is used to specify the start of the MAC address of the MAC pool that you wish to use for the Azure Kubernetes Service host VM. The syntax for the MAC address requires that the least significant bit of the first byte should always be 0, and the first byte should always be an even number (that is, 00, 02, 04, 06...). A typical MAC address can look like: 02:1E:2B:78:00:00. Use MAC pools for long-lived deployments so that MAC addresses assigned are consistent. This is useful if you have a requirement that the VMs have specific MAC addresses. Default is none. .PARAMETER macPoolEnd This is used to specify the end of the MAC address of the MAC pool that you wish to use for the Azure Kubernetes Service host VM. The syntax for the MAC address requires that the least significant bit of the first byte should always be 0, and the first byte should always be an even number (that is, 00, 02, 04, 06...). The first byte of the address passed as the -macPoolEnd should be the same as the first byte of the address passed as the -macPoolStart. Use MAC pools for long-lived deployments so that MAC addresses assigned are consistent. This is useful if you have a requirement that the VMs have specific MAC addresses. Default is none. .PARAMETER useStagingShare Reserved for internal use. We do not recommend using this parameter. .PARAMETER containerRegistry Reserved for internal use. We do not recommend using this parameter. .PARAMETER catalog Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Reserved for internal use. We do not recommend using this parameter. .PARAMETER deploymentId Reserved for internal use. We do not recommend using this parameter. .PARAMETER skipUpdates Reserved for internal use. We do not recommend using this parameter. .PARAMETER stagingShare Reserved for internal use. We do not recommend using this parameter. .PARAMETER kvaSkipWaitForBootstrap Reserved for internal use. We do not recommend using this parameter. .PARAMETER deploymentType Reserved for internal use. We do not recommend using this parameter. .PARAMETER activity Reserved for internal use. We do not recommend using this parameter. .PARAMETER enablePreview Enable AKS HCI Early Access Preview feature on Azure. .PARAMETER skipCleanOnFailure Skip auto cleanup on installation failure .PARAMETER caCertRotationThreshold Threshold days on certificate expiry on when cloudagent CA certificte should be rotated .PARAMETER concurrentDownloads How many parts to segment content downloads into (causes concurrent connections to the hosting server) for the big binaries. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter()] [String] $workingDir = $global:defaultWorkingDir, [parameter()] [String] $imageDir, [parameter()] [String] $version, [parameter(DontShow)] [String] $stagingShare = $global:defaultStagingShare, [parameter()] [String] $cloudConfigLocation = $global:defaultCloudConfigLocation, [parameter()] [String] $nodeConfigLocation = $global:defaultNodeConfigLocation, [parameter()] [String] $cloudLocation = $global:defaultCloudLocation, [parameter()] [bool] $createAutoConfigContainers = $global:defaultCreateAutoConfigContainers, [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet, [parameter()] [VmSize] $controlplaneVmSize = $global:defaultMgmtControlPlaneVmSize, [parameter(DontShow)] [String] $kvaName = (New-Guid).Guid, [parameter()] [String] $kvaPodCIDR = $global:defaultPodCidr, [parameter(DontShow)] [Switch] $kvaSkipWaitForBootstrap, [parameter()] [int] $nodeAgentPort = $global:defaultNodeAgentPort, [parameter()] [int] $nodeAgentAuthorizerPort = $global:defaultNodeAuthorizerPort, [parameter()] [int] $cloudAgentPort = $global:defaultCloudAgentPort, [parameter()] [int] $cloudAgentAuthorizerPort = $global:defaultCloudAuthorizerPort, [parameter()] [String] $clusterRoleName = $($global:cloudAgentAppName + "-" + [guid]::NewGuid()), [parameter()] [Alias("cloudServiceIP")] [String] $cloudServiceCidr = "", [parameter()] [ProxySettings] $proxySettings = $null, [parameter()] [String] $sshPublicKey, [parameter(DontShow)] [Switch] $skipUpdates, [parameter(DontShow)] [Switch] $skipHostLimitChecks, [parameter(DontShow)] [Switch] $skipRemotingChecks, [parameter(DontShow)] [Switch] $insecure, [parameter(DontShow)] [Switch] $forceDnsReplication, [parameter()] [String] $macPoolStart, [parameter()] [String] $macPoolEnd, [parameter(DontShow)] [switch] $useStagingShare, [parameter(DontShow)] [ContainerRegistry] $containerRegistry = $null, [parameter(DontShow)] [String] $catalog = $script:catalogName, [parameter(DontShow)] [String] $ring = $script:ringName, [parameter(DontShow)] [String] $deploymentId = [Guid]::NewGuid().ToString(), [parameter(DontShow)] [int] $operatorTokenValidity = $global:operatorTokenValidity, [parameter(DontShow)] [int] $addonTokenValidity = $global:addonTokenValidity, [parameter(DontShow)] [float] $certificateValidityFactor = $global:certificateValidityFactor, [parameter(DontShow)] [float] $caCertificateValidityFactor = $global:caCertificateValidityFactor, [parameter(DontShow)] [Switch] $enablePreview, [parameter(DontShow)] [Switch] $skipCleanOnFailure, [parameter(DontShow)] [float] $nodeCertificateValidityFactor = $global:nodeCertificateValidityFactor, [parameter(DontShow)] [int] $caCertRotationThreshold = $global:caCertRotationThreshold, [Parameter(Mandatory=$false)] [Int] $concurrentDownloads = 10 ) $startCmdletTime = Get-Date $configCmdletParams = @{version= $version; proxySettings= $proxySettings; catalog= $catalog; ring= $ring; moduleName= $moduleName; kvaName= $kvaName; enablePreview= $enablePreview} trap { Trace-CmdletError -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams throw $_ } Confirm-Configuration ` -useStagingShare:$useStagingShare.IsPresent -stagingShare $stagingShare -vnet $vnet Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName if ($enablePreview.IsPresent) { $catalog = "aks-hci-stable-catalogs-ext" $ring = "earlyaccesspreview" } Set-MocConfig -activity $activity -workingDir $workingDir -imageDir $imageDir -stagingShare $stagingShare ` -cloudConfigLocation $cloudConfigLocation -nodeConfigLocation $nodeConfigLocation ` -vnet $vnet -cloudLocation $cloudLocation ` -nodeAgentPort $nodeAgentPort -nodeAgentAuthorizerPort $nodeAgentAuthorizerPort ` -cloudAgentPort $cloudAgentPort -cloudAgentAuthorizerPort $cloudAgentAuthorizerPort -version $version ` -clusterRoleName $clusterRoleName -cloudServiceCidr $cloudServiceCidr -skipUpdates:$skipUpdates.IsPresent ` -skipHostLimitChecks:$skipHostLimitChecks.IsPresent -insecure:$insecure.IsPresent ` -forceDnsReplication:$forceDnsReplication.IsPresent ` -useStagingShare:$useStagingShare.IsPresent -macPoolStart $macPoolStart -macPoolEnd $macPoolEnd ` -sshPublicKey $sshPublicKey -skipRemotingChecks:$skipRemotingChecks.IsPresent ` -proxySettings $proxySettings -catalog $catalog -ring $ring -createAutoConfigContainers $createAutoConfigContainers ` -deploymentId $deploymentId -certificateValidityFactor $certificateValidityFactor ` -nodeCertificateValidityFactor $nodeCertificateValidityFactor -caCertificateValidityFactor $caCertificateValidityFactor Set-KvaConfig -activity $activity -workingDir $workingDir -imageDir $imageDir -stagingShare $stagingShare ` -kvaName $kvaName -kvaPodCIDR $kvaPodCIDR -kvaSkipWaitForBootstrap:$kvaSkipWaitForBootstrap.IsPresent ` -controlplaneVmSize $controlplaneVmSize ` -vnet $vnet -cloudLocation $cloudLocation ` -skipUpdates:$skipUpdates.IsPresent -insecure:$insecure.IsPresent ` -useStagingShare:$useStagingShare.IsPresent -version $version -macPoolStart $macPoolStart -macPoolEnd $macPoolEnd ` -proxySettings $proxySettings -containerRegistry:$containerRegistry ` -catalog $catalog -ring $ring ` -cloudAgentPort $cloudAgentPort -cloudAgentAuthorizerPort $cloudAgentAuthorizerPort ` -deploymentId $deploymentId -operatorTokenValidity $operatorTokenValidity -addonTokenValidity $addonTokenValidity ` -concurrentDownloads $concurrentDownloads Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_creating_config, $moduleName)) Set-AksHciConfigValue -name "workingDir" -value $workingDir Set-AksHciConfigValue -name "manifestCache" -value ([io.Path]::Combine($workingDir, $("$catalog.json"))) New-Item -ItemType Directory -Force -Path $workingDir | out-null Set-AksHciConfigValue -name "moduleVersion" -value $moduleVersion Set-AksHciConfigValue -name "installState" -value ([InstallState]::NotInstalled) Set-AksHciConfigValue -name "stagingShare" -value $stagingShare Set-AksHciConfigValue -name "skipUpdates" -value $skipUpdates.IsPresent Set-AksHciConfigValue -name "useStagingShare" -value $useStagingShare.IsPresent Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Set-AKsHciConfigValue -name "deploymentId" -value $deploymentId Set-AksHciConfigValue -name "skipCleanOnFailure" -value $skipCleanOnFailure.IsPresent Set-AKsHciConfigValue -name "caCertRotationThreshold" -value $caCertRotationThreshold if (-not $version) { $version = Get-ConfigurationValue -Name "version" -module $moduleName if (-not $version) { # If no version is specified, use the latest $version = Get-AksHciLatestVersion Set-AksHciConfigValue -name "version" -value $version } } else { Get-AksHciLatestVersion | out-null # This clears the cache Get-ProductRelease -Version $version -module $moduleName | Out-Null Set-AksHciConfigValue -name "version" -value $version } Test-ModuleCompatibility -Version $version | Out-Null $installationPackageDir = [io.Path]::Combine($workingDir, $version) Set-AksHciConfigValue -name "installationPackageDir" -value $installationPackageDir New-Item -ItemType Directory -Force -Path $installationPackageDir | Out-Null Save-ConfigurationDirectory -moduleName $moduleName -WorkingDir $workingDir Save-Configuration -moduleName $moduleName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_saved_config) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $configCmdletParams } function Set-AksHciConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER value Value to be persisted #> param ( [String] $name, [Object] $value ) Set-ConfigurationValue -name $name -value $value -module $moduleName } function Get-AksHciConfigValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value #> param ( [String] $name ) return Get-ConfigurationValue -name $name -module $moduleName } function Get-AksHciConfig { <# .SYNOPSIS List the current configuration settings for the Azure Kubernetes Service host. .DESCRIPTION List the current configuration settings for the Azure Kubernetes Service host. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ErrorMessage $_ ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime throw $_ } Import-AksHciConfig -activity $activity Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_get_config, $moduleName)) $global:config[$modulename]["installState"] = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $global:config } function Import-AksHciConfig { <# .DESCRIPTION Loads a configuration from persisted storage. If no configuration is present then a default configuration can be optionally generated and persisted. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter()] [Switch] $createIfNotPresent, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -module $moduleName -status $($AksHciLocMessage.akshci_import_config) # Check if configuration exists if (Test-Configuration -moduleName $moduleName) { # 1. Trigger an import of the dependent configurations Get-MocConfig | Out-Null Get-KvaConfig | Out-Null Import-Configuration -moduleName $moduleName } else { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_cannot_deploy, $moduleName))), $true) } Write-StatusWithProgress -activity $activity -module $moduleName -status $($AksHciLocMessage.akshci_import_config_complete) } function Install-AksHci { <# .SYNOPSIS Install the Azure Kubernetes Service on Azure Stack HCI agents/services and host. .DESCRIPTION Install the Azure Kubernetes Service on Azure Stack HCI agents/services and host. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $activity = $MyInvocation.MyCommand.Name $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -createConfigIfNotPresent -skipMgmtKubeConfig -skipInstallationCheck -activity $activity Test-KvaAzureConnection Install-AksHciInternal -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime } function Enable-AksHciPreview { <# .SYNOPSIS Enable AKSHCI catalog and ring configuration to expose early access preview builds. .DESCRIPTION Enable AKSHCI catalog and ring configuration to expose early access preview builds. .PARAMETER activity Activity name to use when updating progress .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter(DontShow)] [String] $ring = "earlyaccesspreview" ) $activity = $MyInvocation.MyCommand.Name $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity #Set MocConfig for early access preview Enable-MocPreview -catalog $catalog -ring $ring #Set KvaConfig for early access preview Enable-KvaPreview -catalog $catalog -ring $ring #Set AksHCiConfig for early access preview Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_enable_preview, $moduleName)) Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_config, $moduleName)) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Write-Warning $($AksHciLocMessage.akshci_preview_warning) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Disable-AksHciPreview { <# .SYNOPSIS Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .DESCRIPTION Disable AKSHCI catalog and ring configuration which exposes early access preview builds and revert to a stable build. .PARAMETER activity Activity name to use when updating progress. .PARAMETER catalog Release catalog for AKS HCI. Reserved for internal use. We do not recommend using this parameter. .PARAMETER ring Audience (aka ring) type of each catalog. Reserved for internal use. We do not recommend using this parameter. #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name, [parameter(DontShow)] [String] $catalog = "aks-hci-stable-catalogs-ext", [parameter(DontShow)] [String] $ring = "stable" ) $activity = $MyInvocation.MyCommand.Name $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity #Set MocConfig Disable-MocPreview -catalog $catalog -ring $ring #Set KvaConfig Disable-KvaPreview -catalog $catalog -ring $ring #Set AksHCiConfig Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_preview_disable, $moduleName)) Set-AksHciConfigValue -name "catalog" -value $catalog Set-AksHciConfigValue -name "ring" -value $ring Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_config_update, $moduleName)) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Restart-AksHci { <# .SYNOPSIS Restart Azure Kubernetes Service on Azure Stack HCI and remove all deployed Kubernetes clusters. .DESCRIPTION Restarting Azure Kubernetes Service on Azure Stack HCI will remove all of your Kubernetes clusters if any, and the Azure Kubernetes Service host. It will also uninstall the Azure Kubernetes Service on Azure Stack HCI agents and services from the nodes. It will then go back through the original install process steps until the host is recreated. The Azure Kubernetes Service on Azure Stack HCI configuration that you configured via Set-AksHciConfig and the downloaded VHDX images are preserved. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding (PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) Write-Warning $($AksHciLocMessage.akshci_restart_userprompt_warning) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if(-not $PSCmdlet.ShouldProcess("Azure Stack HCI deployment", "Restart-AksHci" )) { Write-Warning $($AksHciLocMessage.akshci_restart_abort_warning) return } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck Uninstall-AksHci -SkipConfigCleanup -activity $activity Install-AksHciInternal -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime } function Uninstall-AksHci { <# .SYNOPSIS Removes Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Removes Azure Kubernetes Service on Azure Stack HCI. If PowerShell commands are run on a cluster where Windows Admin Center was previously used to deploy, the PowerShell module checks the existence of the Windows Admin Center configuration file. Windows Admin Center places the Windows Admin Center configuration file across all nodes. .PARAMETER SkipConfigCleanup skips removal of the configurations after uninstall. After Uninstall, you have to Set-AksHciConfig to install again. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding (PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter()] [Switch] $SkipConfigCleanup, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) Write-Warning $($AksHciLocMessage.akshci_uninstall_userprompt_warning) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ throw $_ } if(-not $PSCmdlet.ShouldProcess("Azure Stack HCI deployment", "Uninstall-AksHci" )) { Write-Warning $($AksHciLocMessage.akshci_uninstall_abort_warning) return } try { Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity $configDetails = Get-TraceConfigDetails $aksHciRegistration = Get-AksHciRegistration if (-not [string]::IsNullOrWhiteSpace($aksHciRegistration.azureResourceGroup)) { try { Test-KvaAzureConnection } catch [Exception] { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_azure_connection_warning) } } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Uninstalling) try { $clusters = Get-AksHciCluster foreach($cluster in $clusters) { try { Remove-AksHciCluster -Name $cluster.Name -Confirm:$false } catch [Exception] { Write-Status -moduleName $moduleName -msg $($GenericLocMessage.generic_exception) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() Write-SubStatus -moduleName $moduleName -msg $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_cannot_delete_target_cluster, $moduleName)) } } } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } } catch [Exception] { # If AksHci is not installed, you would reach here Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } try { Uninstall-Kva -SkipConfigCleanup:$SkipConfigCleanup.IsPresent -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } #ConfigMap details are not available after kva is uninstalled try { Uninstall-Moc -SkipConfigCleanup:$SkipConfigCleanup.IsPresent -activity $activity } catch [Exception] { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" } Trace-Cmdlet -ConfigDetails $configDetails -StartCmdletTime $startCmdletTime Set-AksHciConfigValue -name "installState" -value ([InstallState]::NotInstalled) if (!$SkipConfigCleanup.IsPresent) { Reset-Configuration -moduleName $moduleName } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed } function Get-AksHciKubernetesVersion { <# .SYNOPSIS List the available versions for creating a managed Kubernetes cluster. .DESCRIPTION List the available versions for creating a managed Kubernetes cluster. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_kube_versions) Get-AvailableKubernetesVersions Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime } function Get-AksHciVmSize { <# .SYNOPSIS Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_vm_size) $result = @() foreach($definition in $global:vmSizeDefinitions) { $size = [ordered]@{'VmSize' = $definition[0]; 'CPU' = $definition[1]; 'MemoryGB' = $definition[2]} $result += New-Object -TypeName PsObject -Property $size } Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters return $result } function Set-AksHciCluster { <# .SYNOPSIS Scale the number of control plane nodes or worker nodes in a cluster. Also enable/disable AutoScaler for the cluster. .DESCRIPTION Scale the number of control plane nodes or worker nodes in a cluster. The control plane nodes and the worker nodes must be scaled independently. Also enable or disable AutoScaler for the cluster. .PARAMETER Name Name of the cluster .PARAMETER controlPlaneNodeCount The number of control plane nodes to scale to .PARAMETER controlPlaneVMSize The VM size for the control plane nodes .PARAMETER linuxNodeCount The number of Linux worker nodes to scale to .PARAMETER windowsNodeCount The number of Windows worker nodes to scale to .PARAMETER EnableAutoScaler Enable or disable the cluster autoscaler. .PARAMETER AutoScalerProfileName The name of the AutoScalerProfile used to configure the cluster autoscaler. To change the profile on a cluster that has autoscaler enabled, set this parameter but do not set the EnableAutoScaler parameter. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(ParameterSetName='controlplane')] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount, [Parameter(ParameterSetName='controlplane')] [VmSize] $controlPlaneVMSize, [Parameter(Mandatory=$true, ParameterSetName='worker')] [int] $linuxNodeCount, [Parameter(Mandatory=$true, ParameterSetName='worker')] [int] $windowsNodeCount, [Parameter(Mandatory=$true, ParameterSetName='toggleautoscaler')] [Boolean] $EnableAutoScaler, [Parameter(ParameterSetName='toggleautoscaler')] [Parameter(Mandatory=$true, ParameterSetName='changeautoscalerprofile')] [String] $AutoScalerProfileName, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } # Since the scale parameter set has no mandatory parameters, make sure that at least one was provided # If not, throw a generic error message if ($PSCmdlet.ParameterSetName -eq "controlplane" -and -not ($PSBoundParameters.ContainsKey("controlplanevmsize") -or $PSBoundParameters.ContainsKey("controlplanenodecount"))) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_set_cluster_missing_parameters, $Name))), $true) } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity $mgmtCluster = (Get-KvaConfig)["kvaName"] if ($Name -ieq $mgmtCluster) { throw [CustomException]::new(($($AksHciLocMessage.akshci_scaling_unsupported)), $true) } switch ($PSCmdlet.ParameterSetName) { ("controlplane") { # Based on the above check, at least one of the parameters should be present if ($PSBoundParameters.ContainsKey("controlplanevmsize") -and $PSBoundParameters.ContainsKey("controlplanenodecount")) { Set-KvaClusterNodeCount -Name $Name -controlPlaneNodeCount $controlPlaneNodeCount -controlPlaneVMSize $controlPlaneVMSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("controlplanevmsize")) { Set-KvaClusterNodeCount -Name $Name -controlPlaneVMSize $controlPlaneVMSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("controlplanenodecount")) { Set-KvaClusterNodeCount -Name $Name -controlPlaneNodeCount $controlPlaneNodeCount -activity $activity } break } ("worker") { Write-Output @" WARNING: Set-AksHciCluster can now only be used to scale either the control plane node count or the worker count of the default nodepools that were created as a part of the older cluster creation workflow. Please consider using Set-AksHciNodePool to manage your node pool worker count as it can be used for any nodepool. Examples: - Get a list of cluster1's nodepools: `tGet-AksHciNodePool -ClusterName "cluster1" - Scale "nodepool1" to 2 worker nodes: `tSet-AksHciNodePool -ClusterName "cluster1" -Name "nodepool1" -Count 2 "@ if ($windowsNodeCount -gt 0) { $cluster = Get-KvaCluster -Name $Name -activity $activity Test-SupportedKubernetesVersion -imageType Windows -k8sVersion $cluster.KubernetesVersion } Set-KvaClusterNodeCount -Name $Name -linuxNodeCount $linuxNodeCount -windowsNodeCount $windowsNodeCount -activity $activity break } ("toggleautoscaler") { if (![string]::IsNullOrEmpty($AutoScalerProfileName)) { Set-KvaClusterAutoScaler -Name $Name -Enable $EnableAutoScaler -ProfileName $AutoScalerProfileName -activity $activity } else { Set-KvaClusterAutoScaler -Name $Name -Enable $EnableAutoScaler -activity $activity } break } ("changeautoscalerprofile") { Set-KvaClusterAutoScaler -Name $Name -ProfileName $AutoScalerProfileName -activity $activity break } } Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function New-AksHciAutoScalerProfile { <# .SYNOPSIS Creates a new AutoScalerProfile. .DESCRIPTION Creates a new AutoScalerProfile. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AutoScalerProfileConfig Hashtable containing AutoScalerProfile config keys and their values .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .INPUTS An AutoScalerProfile config is comprised of the following keys. Note: keys that are not provided will fall back to its default value - min-node-count - max-node-count - max-nodes-total - scale-down-enabled - scan-interval - scale-down-delay-after-add - scale-down-delay-after-delete - scale-down-delay-after-failure - scale-down-unneeded-time - scale-down-unready-time - scale-down-utilization-threshold - max-graceful-termination-sec - balance-similar-node-groups - expander - skip-nodes-with-local-storage - skip-nodes-with-system-pods - max-empty-bulk-delete - new-pod-scale-up-delay - max-total-unready-percentage - max-node-provision-time - ok-total-unready-count #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidAutoScalerProfileConfig -AutoScalerProfileConfig $_ })] [hashtable] $AutoScalerProfileConfig, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_creating_autoscalerprofile, $Name)) New-KvaAutoScalerProfile -Name $Name -AutoScalerProfileConfig $AutoScalerProfileConfig -activity $activity Get-AksHciAutoScalerProfile -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Get-AksHciAutoScalerProfile { <# .SYNOPSIS Retrieve AutoScalerProfiles and their settings. .DESCRIPTION Retrieve AutoScalerProfiles and their settings. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_autoscalerprofile_info) Get-KvaAutoScalerProfile -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Set-AksHciAutoScalerProfile { <# .SYNOPSIS Updates an existing AutoScalerProfile. .DESCRIPTION Updates an existing AutoScalerProfile. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AutoScalerProfileConfig Hashtable containing AutoScalerProfile config keys and their values .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .INPUTS An AutoScalerProfile config is comprised of the following keys. Note: keys' values in the profile will not change if not specified in the argument to this cmdlet. - min-node-count - max-node-count - max-nodes-total - scale-down-enabled - scan-interval - scale-down-delay-after-add - scale-down-delay-after-delete - scale-down-delay-after-failure - scale-down-unneeded-time - scale-down-unready-time - scale-down-utilization-threshold - max-graceful-termination-sec - balance-similar-node-groups - expander - skip-nodes-with-local-storage - skip-nodes-with-system-pods - max-empty-bulk-delete - new-pod-scale-up-delay - max-total-unready-percentage - max-node-provision-time - ok-total-unready-count #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidAutoScalerProfileConfig -AutoScalerProfileConfig $_ })] [hashtable] $AutoScalerProfileConfig, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_updating_autoscalerprofile, $Name)) Set-KvaAutoScalerProfile -Name $Name -AutoScalerProfileConfig $AutoScalerProfileConfig -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Remove-AksHciAutoScalerProfile { <# .SYNOPSIS Removes the AutoScalerProfile. .DESCRIPTION Removes the AutoScalerProfile. .PARAMETER Name Name of the AutoScalerProfile .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_deleting_autoscalerprofile, $Name)) Remove-KvaAutoScalerProfile -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function New-AksHciCluster { <# .SYNOPSIS Create a new managed Kubernetes cluster. .DESCRIPTION Create a new Azure Kubernetes Service on Azure Stack HCI cluster. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to deploy .PARAMETER controlPlaneNodeCount The number of control plane (master) nodes .PARAMETER linuxNodeCount The number of Linux worker nodes .PARAMETER windowsNodeCount The number of Windows worker nodes .PARAMETER controlplaneVmSize The VM size to use for control plane nodes .PARAMETER loadBalancerVmSize The VM size to use for the cluster load balancer .PARAMETER linuxNodeVmSize The VM size to use for Linux worker nodes .PARAMETER windowsNodeVmSize The VM size to use for Windows worker nodes .PARAMETER nodePoolName The name of the node pool .PARAMETER nodeCount The number of worker nodes in the node pool .PARAMETER nodeMaxPodCount The maximum number of pods that can run on a worker node .PARAMETER taints A list of taints to put on each worker node .PARAMETER nodeVmSize The VM size to use for the worker nodes in the node pool .PARAMETER osType The OS type for the worker nodes in the node pool .PARAMETER enableAutoScaler Enable the horizontal Nodepool AutoScaler for this cluster .PARAMETER autoScalerProfileName The name of the AutoScalerProfile to use .PARAMETER enableADAuth Whether the call should or not setup Kubernetes for AD Auth .PARAMETER enableMonitoring Enable deploying the monitoring once cluster creation is complete. .PARAMETER vnet The virtual network to use for the cluster. If not specified, the virtual network of the management cluster will be used .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .PARAMETER primaryNetworkPlugin Network plugin (CNI) definition. Simple string values can be passed to this parameter such as "flannel", or "calico". Defaults to "calico". .PARAMETER clusterStorageContainer Storage container that is associated to the Cluster. .PARAMETER loadBalancerSettings LoadBalancer object specifying the type and other params of the loadbalancer #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName = 'twonodepools')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [String] $kubernetesVersion = $global:defaultTargetK8Version, [Parameter()] [ValidateSet(1,3,5)] [int] $controlPlaneNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [int] $linuxNodeCount = 1, [Parameter(ParameterSetName = 'twonodepools')] [int] $windowsNodeCount = 0, [Parameter()] [String] $controlplaneVmSize = $global:defaultControlPlaneVmSize, [Parameter()] [String] $loadBalancerVmSize = $global:defaultLoadBalancerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $linuxNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'twonodepools')] [String] $windowsNodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepool')] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $nodePoolName = $global:defaultNodePoolName, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeCount = $global:defaultWorkerNodeCount, [Parameter(ParameterSetName = 'onenodepool')] [int] $nodeMaxPodCount = 0, [Parameter(ParameterSetName = 'onenodepool')] [String[]] $taints, [Parameter(ParameterSetName = 'onenodepool')] [VmSize] $nodeVmSize = $global:defaultWorkerVmSize, [Parameter(ParameterSetName = 'onenodepool')] [OsType] $osType = $global:defaultWorkerNodeOS, [Parameter()] [Switch] $enableAutoScaler, [Parameter()] [String] $autoScalerProfileName, [Parameter()] [Switch]$enableADAuth, [Parameter()] [Switch]$enableMonitoring, [Parameter()] [VirtualNetwork]$vnet, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity, [Parameter()] [ValidateScript({return $true})] #Note: ValidateScript automatically constructs the NetworkPlugin object, therefore validates the parameter [NetworkPlugin] $primaryNetworkPlugin = [NetworkPlugin]::new(), [Parameter()] [String] $clusterStorageContainer = $global:cloudStorageContainer, [Parameter(Mandatory=$false)] [LoadBalancerSettings] $loadBalancerSettings ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } if (-not $loadBalancerSettings) { # Backward compatibility $loadBalancerSettings = New-AksHciLoadBalancerSetting -name "haProxyLB" -LoadBalancerSku HAProxy -vmSize Standard_K8S3_v1 } Initialize-AksHciEnvironment -activity $activity if ($PSCmdlet.ParameterSetName -ieq "twonodepools") { Write-Output @" WARNING: In a future release, New-AksHciCluster will change how node pools are created. Currently, a user can only set a node count for two default node pools using the parameters linuxNodeCount and windowsNodeCount. This behavior will change giving users more control over the default node pool created after a cluster is deployed. Please consider using the new parameters to manage node pools below. Parameters to be deprecated: `t-linuxNodeCount `t-linuxNodeVmSize `t-windowsNodeCount `t-windowsNodeVmSize New parameters: `t-nodePoolName `t-nodeCount `t-nodeVmSize `t-osType Examples: - Create a cluster that has a node pool with 1 Windows worker node: `tNew-AksHciCluster -Name "cluster1" -osType Windows - Create a cluster that has a node pool with 2 Linux worker nodes and a custom name: `tNew-AksHciCluster -Name "cluster1" -nodePoolName "example-nodepool" -nodeCount 2 "@ Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_linux_kube_version) Test-SupportedKubernetesVersion -imageType Linux -k8sVersion $kubernetesVersion if ($windowsNodeCount -gt 0) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_windows_kube_version) Test-SupportedKubernetesVersion -imageType Windows -k8sVersion $kubernetesVersion } New-KvaCluster ` -Name $Name -kubernetesVersion $kubernetesVersion ` -controlPlaneNodeCount $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize ` -linuxNodeCount $linuxNodeCount -linuxNodeVmSize $linuxNodeVmSize ` -windowsNodeCount $windowsNodeCount -windowsNodeVmSize $windowsNodeVmSize ` -enableAutoScaler:$enableAutoScaler.IsPresent -autoScalerProfileName $autoScalerProfileName ` -enableADAuth:$enableADAuth.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin.Name -vnet $vnet ` -activity $activity -loadBalancerSettings $loadBalancerSettings } elseif ($PSCmdlet.ParameterSetName -ieq "onenodepool") { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_other_os_kube_version, $osType)) Test-SupportedKubernetesVersion -imageType $osType -k8sVersion $kubernetesVersion New-KvaCluster ` -Name $Name -kubernetesVersion $kubernetesVersion ` -controlPlaneNodeCount $controlPlaneNodeCount -controlplaneVmSize $controlplaneVmSize ` -loadBalancerVmSize $loadBalancerVmSize ` -nodePoolName $nodePoolName -nodeCount $nodeCount -nodeMaxPodCount $nodeMaxPodCount -taints $taints ` -nodeVmSize $nodeVmSize -osType $osType ` -enableAutoScaler:$enableAutoScaler.IsPresent -autoScalerProfileName $autoScalerProfileName ` -enableADAuth:$enableADAuth.IsPresent ` -primaryNetworkPlugin $primaryNetworkPlugin.Name -vnet $vnet ` -activity $activity -loadBalancerSettings $loadBalancerSettings } Get-AksHciCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName ## If enableMonitoring is enabled then install the monitoring with default values. if ($enableMonitoring.IsPresent) { Install-AksHciMonitoring -Name $Name -storageSizeGB 100 -retentionTimeHours 240 } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Get-AksHciCluster { <# .SYNOPSIS List Kubernetes managed clusters including the Azure Kubernetes Service host. .DESCRIPTION List Kubernetes managed clusters including the Azure Kubernetes Service host. .PARAMETER Name Name of the cluster .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_cluster_info) Get-KvaCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Remove-AksHciCluster { <# .SYNOPSIS Delete a managed Kubernetes cluster. .DESCRIPTION Delete a managed Kubernetes cluster. .PARAMETER Name Name of the cluster .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($Name, "Delete the managed Kubernetes cluster")) { if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity Remove-KvaCluster -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } } function Get-AksHciClusterUpdates { <# .SYNOPSIS Get the available Kubernetes upgrades for an Azure Kubernetes Service cluster. .DESCRIPTION Get the available Kubernetes upgrades for an Azure Kubernetes Service cluster. .PARAMETER Name Name of the cluster. .PARAMETER activity Activity name to use when updating progress. #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $upgrades = Get-KvaClusterUpgrades -Name $Name -activity $activity $upgrades.AvailableUpgrades Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Update-AksHciCluster { <# .SYNOPSIS Update a managed Kubernetes cluster to a newer Kubernetes or OS version. .DESCRIPTION Update a managed Kubernetes cluster to a newer Kubernetes or OS version. .PARAMETER Name Name of the cluster .PARAMETER kubernetesVersion Version of kubernetes to upgrade to .PARAMETER operatingSystem Perform an operating system upgrade instead of a kubernetes version upgrade .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'Low')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter()] [String] $kubernetesVersion, [Parameter()] [Switch] $operatingSystem, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity Get-KvaCluster -Name $Name -activity $activity | Out-Null if ($operatingSystem.IsPresent -and $kubernetesVersion -ne "") { # operating system is updated when kubernetes version is upgraded. # if user specifies both, just turn the switch off, because we will internally # update the OS. $operatingSystem = $false } $nextVersion = $null if (-not $operatingSystem.IsPresent) { if ($kubernetesVersion -eq "") { # no version was requested. just try to make the highest jump. $nextVersion = Get-NextKubernetesVersionForUpgrade -Name $Name -activity $activity } else { $nextVersion = Get-CleanInputKubernetesVersion -KubernetesVersion $kubernetesVersion } } if ($PSCmdlet.ShouldProcess($Name, "Update the managed Kubernetes cluster")) { $confirmValue = $true if ($PSBoundParameters.ContainsKey('Confirm')) { $confirmValue = $PSBoundParameters['Confirm'] } Update-KvaCluster -Name $Name -activity $activity -operatingSystem:$operatingSystem.IsPresent -nextVersion $nextVersion -Confirm:$confirmValue } Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function New-AksHciNodePool { <# .SYNOPSIS Create a new nodepool under a cluster. .DESCRIPTION Creates a new nodepool under a cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count The number of worker nodes in the nodepool .PARAMETER OSType OS type of the node pool. Defaults to Linux .PARAMETER VMSize The VM size to use for the worker nodes. Defaults to Standard_K8S3_v1 .PARAMETER MaxPodCount The maximum number of pods that can run on a worker node .PARAMETER Taints A list of taints to put on each worker node .PARAMETER DisableAutoScaler Prevent the horizontal nodepool AutoScaler from managing this nodepool. If AutoScaler is enabled on the cluster, all new nodepools will be managed by it. This flag can be used to override that for the new nodepool. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [int] $Count = $global:defaultWorkerNodeCount, [Parameter()] [VmSize] $VMSize = $global:defaultWorkerVmSize, [Parameter()] [OsType] $OSType = $global:defaultWorkerNodeOS, [Parameter()] [int] $MaxPodCount = 0, [Parameter()] [String[]] $Taints, [Parameter()] [Switch] $DisableAutoScaler, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_create_node_pool) -moduleName $moduleName New-KvaClusterNodePool ` -ClusterName $ClusterName -Name $Name ` -MaxPodCount $MaxPodCount -Taints $Taints -Count $Count ` -VMSize $VMSize -OSType $OSType ` -DisableAutoScaler:$DisableAutoScaler.IsPresent ` -activity $activity Get-AksHciNodePool -ClusterName $ClusterName -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Get-AksHciNodePool { <# .SYNOPSIS List a Kubernetes managed cluster's nodepools. .DESCRIPTION List a Kubernetes managed cluster's nodepools. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter()] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_cluster_node_pool_info) Get-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Set-AksHciNodePool { <# .SYNOPSIS Scale a Kubernetes managed cluster's nodepool. .DESCRIPTION Scale a Kubernetes managed cluster's nodepool. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER Count Node count to scale to .PARAMETER VMSize VM size to for the nodepool nodes .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER AutoScaler If set to true, allow the cluster AutoScaler to horizontally scale this nodepool if it is enabled on the cluster. Else, if set to false, autoscaling will be disabled on this nodepool. .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter(ParameterSetName='scale')] [int] $Count, [Parameter(ParameterSetName='scale')] [VmSize] $VMSize, [Parameter(Mandatory=$true, ParameterSetName='autoscaler')] [Boolean] $AutoScaler, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } # Since the scale parameter set has no mandatory parameters, make sure that at least one was provided # If not, throw a generic error message if ($PSCmdlet.ParameterSetName -eq "scale" -and -not ($PSBoundParameters.ContainsKey("vmsize") -or $PSBoundParameters.ContainsKey("count"))) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_set_nodepool_missing_parameters, $Name, $ClusterName))), $true) } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity switch ($PSCmdlet.ParameterSetName) { ("scale") { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_scaling_node_pool) # Based on the above check, at least one of the parameters should be present if ($PSBoundParameters.ContainsKey("vmsize") -and $PSBoundParameters.ContainsKey("count")) { Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Count $Count -VmSize $VmSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("vmsize")) { Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -VmSize $VmSize -activity $activity } elseif ($PSBoundParameters.ContainsKey("count")) { Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Count $Count -activity $activity } break } ("autoscaler") { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_updating_node_pool_autoscaler) Set-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -AutoScaler $AutoScaler -activity $activity break } } Get-AksHciNodePool -ClusterName $ClusterName -Name $Name -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Remove-AksHciNodePool { <# .SYNOPSIS Delete a nodepool in a managed Kubernetes cluster. .DESCRIPTION Delete a nodepool in a managed Kubernetes cluster. .PARAMETER ClusterName Name of the cluster .PARAMETER Name Name of the nodepool .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidClusterName -Name $_ })] [String] $ClusterName, [Parameter(Mandatory=$true)] [ValidateScript({Test-ValidNodePoolName -Name $_ })] [String] $Name, [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($name, "Delete the node pool in the managed Kubernetes cluster")) { if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters -allowDuplicateJobs } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_delete_node_pool) Remove-KvaClusterNodePool -ClusterName $ClusterName -Name $Name -Confirm:$false -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Get-AksHciLogs { <# .SYNOPSIS Create a zipped folder with logs from all your pods. .DESCRIPTION Create a zipped folder with logs from all your pods. This command will create an output zipped folder called akshcilogs.zip in your AKS on Azure Stack HCI working directory. The full path to the akshcilogs.zip file will be the output after running Get-AksHciLogs (for example, C:\AksHci\0.9.6.3\akshcilogs.zip, where 0.9.6.3 is the AKS on Azure Stack HCI release number). .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress .PARAMETER zipName Zip path and name to use for storing logs .PARAMETER VirtualMachineLogs Switch to get only the logs from the vm's (LB vm if unstacked deployment and management-cluster vm) .PARAMETER AgentLogs Switch to get only logs of the wssdagent and wssdcloudagent on all nodes .PARAMETER EventLogs Switch to get only Windows Event Logson all nodes .PARAMETER KvaLogs Switch to get only the logs from KVA .PARAMETER DownloadSdkLogs Switch to get only the logs from DownloadSdk .PARAMETER BillingRecords Switch to get only the billing records #> param ( [Parameter()] [Switch]$AsJob, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$false)] [String]$zipName, [Parameter(Mandatory=$false)] [Switch]$VirtualMachineLogs, [Parameter(Mandatory=$false)] [Switch]$AgentLogs, [Parameter(Mandatory=$false)] [Switch]$EventLogs, [Parameter(Mandatory=$false)] [Switch]$KvaLogs, [Parameter(Mandatory=$false)] [Switch]$DownloadSdkLogs, [Parameter(Mandatory=$false)] [Switch]$BillingRecords ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } $allswitch = $true if ($VirtualMachineLogs.IsPresent -or $AgentLogs.IsPresent -or $EventLogs.IsPresent -or $KvaLogs.IsPresent -or $DownloadSdkLogs.IsPresent -or $BillingRecords.IsPresent) { $allswitch = $false } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck $logName = $("akshcilogs" + [io.Path]::GetRandomFileName()) $logDir = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], $logName) if ($VirtualMachineLogs.IsPresent -or $AgentLogs.IsPresent -or $EventLogs.IsPresent -or $allswitch) { try { Get-MocLogs -path $logDir -activity $activity -VirtualMachineLogs:$VirtualMachineLogs.IsPresent -AgentLogs:$AgentLogs.IsPresent -EventLogs:$EventLogs.IsPresent | Out-Null } catch [Exception] { } } if ($allswitch -or $KvaLogs.IsPresent) { try { Get-KvaLogs -path $logDir -activity $activity } catch [Exception] { } } if ($allswitch -or $DownloadSdkLogs.IsPresent) { try { Get-DownloadSdkLogs -Path $logDir } catch [Exception] { } } if ($allswitch -or $BillingRecords.IsPresent) { New-Item -ItemType Directory -Force -Path $logDir | Out-Null try { Get-KvaBillingRecords -activity $activity -outputformat "json" | ConvertFrom-Json | Format-List * > ($logDir + "\AksHciBillingRecords.log") } catch [Exception]{ Write-Status -moduleName $moduleName -msg $($AksHciLocMessage.akshci_billing_collection_failed) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() } } $akshcilogDir = [io.Path]::Combine($logDir, "akshci") New-Item -ItemType Directory -Force -Path $akshcilogDir | Out-Null Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_collecting_info, $moduleName)) $global:config[$moduleName] > $akshcilogDir"\AksHciConfig.txt" Get-AksHciEventLog | Format-List * > $akshcilogDir"\AksHciPS.log" $modulelist = @("AksHci", "Kva", "Moc", "DownloadSdk", "Az.Resources", "Az.Accounts", "AzureAD", "TraceProvider" ) foreach ($module in $modulelist) { Get-Command -Module $module | Sort-Object -Property Source >> $($akshcilogDir+"\moduleinfo.txt") } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_compressing_logs) if ([string]::IsNullOrEmpty($zipName)) { $zipName = [io.Path]::Combine($global:config[$moduleName]["installationPackageDir"], "$logName.zip") } try { Compress-Directory -ZipFilename $zipName -SourceDir $logDir } catch [Exception] { Write-Status -moduleName $moduleName -msg $($GenericLocMessage.generic_exception) Write-SubStatus -moduleName $moduleName -msg $_.Exception.Message.ToString() Write-SubStatus -moduleName $moduleName -msg $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_cannot_compress, $zipName)) Write-Status -moduleName $moduleName -msg $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_log_path, $logDir)) return $logDir } Remove-Item -Path $logDir -Force -Recurse -ErrorAction Continue Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_zip_path, $zipName)) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $zipName } function Get-AksHciEventLog { <# .SYNOPSIS Gets all the event logs from the Azure Kubernetes Service on Azure Stack HCI PowerShell module. .DESCRIPTION Gets all the event logs from the Azure Kubernetes Service on Azure Stack HCI PowerShell module. #> $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime throw $_ } $logs = Get-WinEvent -ProviderName $moduleName -ErrorAction Ignore $logs += Get-KvaEventLog $logs += Get-MocEventLog $logs += Get-DownloadSdkEventLog Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $logs } function Enable-AksHciArcConnection { <# .SYNOPSIS Connects an AKS on Azure Stack HCI workload cluster to Azure Arc for Kubernetes. .DESCRIPTION Connects an AKS on Azure Stack HCI workload cluster to Azure Arc for Kubernetes. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } # just to ensure the cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { New-KvaArcConnection -Name $Name -activity $activity } else { New-KvaArcConnection -Name $Name -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -activity $activity } Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_arc_installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Disable-AksHciArcConnection { <# .DESCRIPTION Helper function to remove the arc onboarding agent addon on a cluster. .PARAMETER Name cluster Name .PARAMETER tenantId tenant id for azure .PARAMETER subscriptionId subscription id for azure .PARAMETER resourceGroup azure resource group for connected cluster .PARAMETER credential credential for azure service principal .PARAMETER location azure location .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParametersetName='None')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $tenantId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $subscriptionId, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $resourceGroup, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [PSCredential] $credential, [Parameter(Mandatory=$true, ParameterSetName='azureoveride')] [String] $location, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Test-KvaAzureConnection } # just to ensure the cluster exists Get-KvaCluster -Name $Name -activity $activity | Out-Null # because of the parameter set we know that subid can represent the set. if ([string]::IsNullOrWhiteSpace($subscriptionId)) { Remove-KvaArcConnection -Name $Name -activity $activity } else { Remove-KvaArcConnection -Name $Name -tenantId $tenantId -subscriptionId $subscriptionId -resourceGroup $resourceGroup -credential $credential -location $location -activity $activity } Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_arc_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Install-AksHciAdAuth { <# .SYNOPSIS Install Active Directory authentication. .DESCRIPTION Install Active Directory authentication. .PARAMETER Name Cluster Name .PARAMETER keytab Path to the kerberos keytab corresponding to the current password on the local machine. Must be named current.keytab .PARAMETER previousKeytab Path to the kerberos keytab corresponding to the previous password on the local machine. Must be named previous.keytab .PARAMETER SPN SPN registered for the Active Directory account to be used with the api-server. .PARAMETER TTL Time to live (in hours) for previous keytab file if supplied. Default is 10 hours .PARAMETER adminUser The user name to be given cluster-admin permissions. Machine must be domain joined. .PARAMETER adminGroup The group name to be given cluster-admin permissions. Machine must be domain joined. .PARAMETER adminUserSID The user SID to be given cluster-admin permissions. .PARAMETER adminGroupSID The group SID to be given cluster-admin permissions. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, DefaultParameterSetName='domainjoin')] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [String] $keytab, [Parameter(Mandatory=$false)] [String] $previousKeytab, [Parameter(Mandatory=$true)] [String] $SPN, [Parameter(Mandatory=$false)] [int] $TTL, [Parameter(Mandatory=$false, ParameterSetName='domainjoin')] [String] $adminUser, [Parameter(Mandatory=$false, ParameterSetName='domainjoin')] [String] $adminGroup, [Parameter(Mandatory=$false, ParameterSetName='workplacejoin')] [String] $adminUserSID, [Parameter(Mandatory=$false, ParameterSetName='workplacejoin')] [String] $adminGroupSID, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $capiCluster = Get-KvaCapiCluster -Name $Name $canInstallWebhook = $false foreach($feature in $capiCluster.additionalfeatures ) { if ($feature.FeatureName -eq "ad-auth-webhook") { $canInstallWebhook = $true } } if (-not $canInstallWebhook) { Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_adauth_disabled, $Name)) return } if(-not $adminUser -and -not $adminUserSID -and -not $adminGroup -and -not $adminGroupSID) { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_addon_enable_req) return } try { if (![string]::IsNullOrEmpty($adminUser)) { $adminUserSIDYAML = (New-Object System.Security.Principal.NTAccount($adminUser)).Translate([System.Security.Principal.SecurityIdentifier]).value } if (![string]::IsNullOrEmpty($adminGroup)) { $adminGroupSIDYAML = (New-Object System.Security.Principal.NTAccount($adminGroup)).Translate([System.Security.Principal.SecurityIdentifier]).value } } catch { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_sid_translation_failed) Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ChildStageName "NameToSIDTranslationFailed" ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ return } if ($PSCmdlet.ParameterSetName -ieq "workplacejoin") { $adminUserSIDYAML = $adminUserSID $adminGroupSIDYAML = $adminGroupSID } if (![string]::IsNullOrEmpty($previousKeytab)) { $prevKtSt = "--from-file=`"$previousKeytab`"" } $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: ad-auth-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: ad-auth-webhook targetNamespace: kube-system templateType: yaml providerVariables: - key: AD_AUTH_SPN value: "$SPN" - key: ADMIN_USER value: "$adminUserSIDYAML" - key: ADMIN_GROUP value: "$adminGroupSIDYAML" - key: TICKET_LIFETIME value: "$TTL" - key: keytab valueFrom: secret: name: keytab-$Name "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-ad-auth-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-KubeCtl -arguments $("create secret generic keytab-$Name --from-file=`"$keytab`" $prevKtSt") Invoke-Kubectl -arguments $("apply -f ""$yamlFile"" ") Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_active_dir_sso) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Uninstall-AksHciAdAuth { <# .SYNOPSIS Uninstall Active Directory authentication. .DESCRIPTION Uninstall Active Directory authentication. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $yaml = @" apiVersion: msft.microsoft/v1 kind: AddOn metadata: name: ad-auth-webhook-$Name labels: msft.microsoft/capicluster-name: $Name spec: configuration: supportedAddOnName: ad-auth-webhook targetNamespace: kube-system templateType: yaml "@ $yamlFile = $($global:config[$moduleName]["installationPackageDir"]+"\"+$global:yamlDirectoryName+"\$Name-ad-auth-webhook.yaml") Set-Content -Path $yamlFile -Value $yaml -ErrorVariable err if ($null -ne $err -and $err.count -gt 0) { throw $err } Invoke-Kubectl -arguments $("delete -f ""$yamlFile"" ") Remove-Item $yamlFile Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_active_dir_sso_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Install-AksHciGMSAWebhook { <# .DESCRIPTION Installs gMSA webhook for an AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Set-KvaGMSAWebhook -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Uninstall-AksHciGMSAWebhook { <# .DESCRIPTION Uninstalls gmsa-webhook addon for an AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Reset-KvaGMSAWebhook -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Add-AksHciGMSACredentialSpec { <# .DESCRIPTION Helper function to add a credentials spec for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecFilePath File Path of the JSON cred spec file .PARAMETER credSpecName Name of the Kubernetes credential spec object the user would like to designate This will be the name the deployment yaml reference for the field gmsaCredentialSpec .PARAMETER secretName Name of the Kubernetes secret object storing the Active Directory user credentials and gMSA domain .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER serviceAccount Name of the Kubernetes service account assigned to read the Kubernetes gMSA credspec object .PARAMETER clusterRoleName Name of the Kubernetes clusterrole assigned to use the Kubernetes gMSA credspec object .PARAMETER overwrite Overwrites existing Kubernetes credential spec object .PARAMETER activity Activity name to use when updating progress .EXAMPLE Add-AksHciGMSACredentialSpec -Name test1 -credSpecFilePath .\credspectest.json -credSpecName credspec-test1 -secretName secret-test1 -clusterRoleName clusterrole-test1 Creates a GMSACredentialSpec object called credspec-test1 from the JSON credential spec file credspectest.json on a target cluster named test1. The object credspec-test1 references the default namespaced secret secret-test1 created by the user for Active Directory user credentials. The cmdlet also creates a cluster role named clusterrole-test1 that binds to the default service account along with a rolebinding that resides in the default namespace. .EXAMPLE Add-AksHciGMSACredentialSpec -Name test1 -credSpecFilePath .\credspectest.json -credSpecName credspec-test1 -secretName secret-test1 -secretNamespace secret-namespace -clusterRoleName clusterrole-test1 -serviceAccount svc1 -overwrite Creates a GMSACredentialSpec object called credspec-test1 from the JSON credential spec file credspectest.json on a target cluster named test1. The object credspec-test1 references the secret secret-test1 residing in the namespace secret-namespace. Both the secret and the namespace secret-namespace are created by the user. The also cmdlet creates a cluster role named clusterrole-test1 that binds to the user-created service account svc1 along with a rolebinding that resides in the secret-namespace namespace. The overwrite parameter checks for existing GMSACredentialSpec, clusterrole, and rolebinding objects with the same names as the ones specified by the cmdlet parameters and overwrites them with the new setup based on the new parameters. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecFilePath')] [String]$credSpecFilePath, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecName')] [String]$credSpecName, [Parameter(Mandatory=$true)] [String]$secretName, [Parameter()] [String]$secretNamespace = "default", [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [switch]$overwrite, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = $MyInvocation.MyCommand.Name } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Set-KvaGMSACredentialSpec -Name $Name -credSpecFilePath $credSpecFilePath -credSpecName $credSpecName ` -secretName $secretName -secretNamespace $secretNamespace -serviceAccount $serviceAccount ` -clusterRoleName $clusterRoleName -overwrite:$overwrite.isPresent -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_cred_spec_installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Remove-AksHciGMSACredentialSpec { <# .DESCRIPTION Helper function to remove a credentials spec for gmsa deployments on a cluster. .PARAMETER Name Cluster Name .PARAMETER credSpecName Name of the Kubernetes credential spec object the user would like to designate .PARAMETER serviceAccount Kubernetes service account assigned to read the Kubernetes gMSA credential spec object .PARAMETER clusterRoleName Name of the Kubernetes clusterrole assigned to use the Kubernetes gMSA credential spec object .PARAMETER secretNamespace Namespace where the Kubernetes secret object resides in .PARAMETER activity Activity name to use when updating progress .EXAMPLE Remove-AksHciGMSACredentialSpec -Name test1 -credSpecName credspec-test1 -clusterRoleName clusterrole-test1 Removes the GMSACredentialSpec object credspec-test1 and the clusterrole object clusterrole-test1 along with the rolebinding object binding clusterrole-test1 to the default service account from a target cluster named test1 .EXAMPLE Remove-AksHciGMSACredentialSpec -Name test1 -credSpecName credspec-test1 -serviceAccount svc1 -secretNamespace secret-namespace -clusterRoleName clusterrole-test1 Removes a GMSACredentialSpec object credspec-test1 and the clusterrole object clusterrole-test1 from the target cluster test1. The rolebinding object binding clusterrole-test1 to the service account svc1 is also removed from the secret-namespace namespace in the target cluster named test1. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification='Not a plaintext password')] [CmdletBinding(PositionalBinding=$False)] param ( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [Alias('gmsaCredentialSpecName')] [String]$credSpecName, [Parameter()] [String]$serviceAccount = "default", [Parameter(Mandatory=$true)] [String]$clusterRoleName, [Parameter()] [String]$secretNamespace = "default", [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = $MyInvocation.MyCommand.Name } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Reset-KvaGMSACredentialSpec -Name $Name -credSpecName $credSpecName -serviceAccount $serviceAccount ` -clusterRoleName $clusterRoleName -secretNamespace $secretNamespace -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_gmsa_cred_spec_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } function Get-AksHciCredential { <# .SYNOPSIS Access your cluster using kubectl. .DESCRIPTION Access your cluster using kubectl. This will use the specified cluster's kubeconfig file as the default kubeconfig file for kubectl. .PARAMETER Name Name of the cluster to obtain the credential/kubeconfig for. .PARAMETER configPath Location to output the credential/kubeconfig file to. .PARAMETER adAuth To get the Active Directory SSO version of the kubeconfig. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(PositionalBinding=$False, SupportsShouldProcess, ConfirmImpact = 'High')] param ( [Parameter(Mandatory=$true)] [string] $Name, [Parameter()] [string] $configPath = $($env:USERPROFILE+"\.kube\config"), [Parameter(Mandatory=$false)] [Switch] $adAuth, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } if ($PSCmdlet.ShouldProcess($Name, $("Retrieve and write the cluster kubeconfig file to $configPath"))) { Initialize-AksHciEnvironment -activity $activity Get-KvaClusterCredential -Name $Name -outputLocation $configPath -adAuth:$adAuth.IsPresent -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } } function Repair-AksHciClusterCerts { <# .DESCRIPTION Attempts to repair failed TLS on a cluster/cloudagent .PARAMETER Name Name of the node/cluster to fix .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER $fixCloudCredentials Fix cloud tls in a cluster .PARAMETER $fixKubeletCredentials Fix failed TLS on a cluster .PARAMETER $patchLoadBalancer Patch load balancer certificates .PARAMETER force Force repair(without checks) .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding(DefaultParameterSetName = 'cloud')] param ( [Parameter(Mandatory=$true)] [string] $Name, [Parameter()] [string] $sshPrivateKeyFile, [Parameter(Mandatory=$true, ParameterSetName='cloud')] [Switch] $fixCloudCredentials, [Parameter(ParameterSetName='cloud')] [Switch] $patchLoadBalancer, [Parameter(Mandatory=$true, ParameterSetName='kubelet')] [Switch] $fixKubeletCredentials, [Parameter(ParameterSetName='cloud')] [Switch] $force, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity if (-not $sshPrivateKeyFile) { $m = Get-MocConfig $sshPrivateKeyFile = $m["sshPrivateKey"] } if ($PSCmdlet.ParameterSetName -ieq "cloud") { Repair-KvaCerts -Name $Name -sshPrivateKeyFile $sshPrivateKeyFile -patchLoadBalancer:$patchLoadBalancer.IsPresent -force:$force.IsPresent -activity $activity } if($PSCmdlet.ParameterSetName -ieq "kubelet") { Repair-KvaCluster -Name $Name -sshPrivateKeyFile $sshPrivateKeyFile -fixCertificates -activity $activity } Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime } function Repair-AksHciCerts { <# .DESCRIPTION Attempts to repair failed TLS on a cluster . .PARAMETER sshPrivateKeyFile Kubeconfig for the cluster the node belongs to .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [string] $sshPrivateKeyFile, [Parameter()] [Switch] $force, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name)" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity if (-not $sshPrivateKeyFile) { $m = Get-MocConfig $sshPrivateKeyFile = $m["sshPrivateKey"] } # Rotate NodeAgent tokens Repair-Moc # Rotate KVA and Management cluster tokens Repair-KvaCerts -sshPrivateKeyFile $sshPrivateKeyFile -force:$force.IsPresent -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime } function Sync-AksHciBilling { <# .DESCRIPTION Sync Aks-Hci billing. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $syncResult = Sync-KvaBilling -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $syncResult } function Get-AksHciBillingStatus { <# .DESCRIPTION Get Aks-Hci billing status. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $statusResult = Get-KvaBillingStatus -activity $activity -outputformat "json" | ConvertFrom-Json Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $statusResult } function New-AksHciClusterNetwork { <# .DESCRIPTION Create network settings to be used for the target clusters. .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNetDHCP = New-AksHciClusterNetwork -name e1 -vswitchName External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE $clusterVNetStatic = New-AksHciClusterNetwork -name e1 -vswitchName External -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [string] $vswitchName, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$true)] [String] $vippoolstart, [Parameter(Mandatory=$true)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend, [Parameter()] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return New-KvaClusterNetwork -name $name -vswitchname $vswitchname -ipaddressprefix $ipaddressprefix -gateway $gateway -dnsservers $dnsservers -vlanID $vlanID -vippoolstart $vippoolstart -vippoolend $vippoolend -k8snodeippoolstart $k8snodeippoolstart -k8snodeippoolend $k8snodeippoolend -activity $activity } function Get-AksHciClusterNetwork { <# .DESCRIPTION Gets the VirtualNetwork object for a target cluster given either the vnet name or the cluster name. If no parameter is given, all vnet's are returned. .PARAMETER name The name of the vnet .PARAMETER clusterName The name of the cluster (NOTE: This is P2 -- but we really want to add this functionality for Ben) .OUTPUTS If name is specified, the VirtualNetwork object will be returned. If clusterName is specified, the VirtualNetwork object that the cluster is using will be returned. If no parameters are specified all VirtualNetwork objects will be returned. .NOTES The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -name e1 .EXAMPLE $clusterVNet = Get-AksHciClusterNetwork -clusterName myTargetCluster .EXAMPLE $allClusterVNets = Get-AksHciClusterNetwork #> param ( [Parameter(Mandatory=$false)] [string] $name, [Parameter(Mandatory=$false)] [string] $clusterName, [Parameter()] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ throw $_ } Initialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return Get-KvaClusterNetwork -name $name -clusterName $clusterName -activity $activity } function Remove-AksHciClusterNetwork { <# .DESCRIPTION Remove a virtual network object for a target cluster .PARAMETER name The name of the vnet .NOTES The cmdlet will throw an exception if the network is still being used. The cmdlet will throw an exception if the mgmt cluster is not up. .EXAMPLE Remove-AksHciClusterNetwork -name e1 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter()] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime return Remove-KvaClusterNetwork -name $name -activity $activity } function Install-AksHciMonitoring { <# .DESCRIPTION Installs monitoring infrastructure on AKS-HCI cluster. .PARAMETER Name Cluster Name .PARAMETER storageSizeGB Amount of storage for Prometheus in GB .PARAMETER retentionTimeHours metrics retention time in hours. (min 2 hours, max 876000 hours(100 years)) .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [int] $storageSizeGB, [Parameter(Mandatory=$true)] [int] $retentionTimeHours, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_installing_monitoring) Set-KvaHciMonitoring -Name $Name -storageSizeGB $storageSizeGB -retentionTimeHours $retentionTimeHours -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_installed_monitoring) Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_monitoring_progress) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Uninstall-AksHciMonitoring { <# .DESCRIPTION Uninstalls monitoring from an AKS-HCI cluster. .PARAMETER Name cluster Name .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter(Mandatory = $true)] [String] $Name, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_uninstalling_monitoring) Reset-KvaHciMonitoring -Name $Name -activity $activity Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_uninstalled_monitoring) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Add-AksHciNode { <# .DESCRIPTION Add new node to the Moc stack during a Failure Replacement Unit scenario .PARAMETER nodeName The name of the node in the Failover Cluster, the node is already expected to have been added to the failover cluster .PARAMETER activity Activity name to use when updating progress .EXAMPLE Add-AksHciNode -nodeName "node1" #> param ( [String]$nodeName, [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_adding_node) New-MocPhysicalNode -nodeName $nodeName -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Remove-AksHciNode { <# .DESCRIPTION Remove a failed node from the Moc stack during a Failure Replacement Unit scenario .PARAMETER nodeName The name of the node in Failover Cluster .PARAMETER activity Activity name to use when updating progress .NOTES If the physical machine is shut down or removed or unreachable on the network prior to the cmdlet this guarntees that it is removed from the cloud-agent maps but not a complete cleaup of that node. .EXAMPLE Remove-AksHciNode -nodeName "node1" #> param ( [String]$nodeName, [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_removing_node) Remove-MocPhysicalNode -nodeName $nodeName -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.generic_done) -completed Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function New-AksHciProxySetting { <# .DESCRIPTION Create proxy settings to be used for the Aks Hci deployment .PARAMETER name A name to associate with the proxy settings .PARAMETER http HTTP proxy server configuration .PARAMETER https HTTPS proxy server configuration .PARAMETER noProxy Proxy server exemption/bypass list .PARAMETER certFile Path to a CA certificate file used to establish trust with a HTTPS proxy server .PARAMETER credential Proxy server credentials (for basic authentication) .OUTPUTS Proxy Settings object .EXAMPLE $credential = Get-Credential $proxySetting = New-AksHciProxySetting -http http://contosoproxy:8080 -https https://contosoproxy:8080 -noProxy "localhost,127.0.0.1,.svc,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" -credential $credential -certFile c:\proxyca.crt #> param ( [Parameter()] [String] $name, [Parameter()] [String] $http, [Parameter()] [String] $https, [Parameter()] [String] $noProxy = $global:defaultProxyExemptions, [Parameter()] [String] $certFile, [Parameter()] [PSCredential] $credential = [PSCredential]::Empty ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Test-ProxyConfiguration -http $http -https $https -noProxy $noProxy -certFile $certFile Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime return [ProxySettings]::new($credential, $name, $http, $https, $noProxy, $certFile) } function Set-AksHciProxySetting { <# .DESCRIPTION Update proxy settings noProxy list .PARAMETER noProxy Proxy server exemption/bypass list .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter()] [ValidateNotNull()] [String] $noProxy, [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -activity $activity $proxySettings = Get-AksHciProxySetting if ($proxySettings.HTTP -eq "" -and $proxySettings.HTTPS -eq "") { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_proxy_update_failed) return } $proxySettings.noProxy = $noProxy Test-ProxyConfiguration -http $proxySettings.HTTP -https $proxySettings.https -noProxy $proxySettings.noProxy -certFile $proxySettings.CertFile Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $moduleName Set-ProxyConfiguration -proxySettings $proxySettings -moduleName $global:MocModule Set-KvaProxySetting -proxySettings $proxySettings Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Get-AksHciProxySetting { <# .DESCRIPTION Returns AksHci proxy settings .PARAMETER activity Activity name to use when updating progress .OUTPUTS Proxy Settings object #> param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity $http = $global:config[$moduleName]["proxyServerHTTP"] $https = $global:config[$moduleName]["proxyServerHTTPS"] $noProxy = $global:config[$moduleName]["proxyServerNoProxy"] $certFile = $global:config[$moduleName]["proxyServerCertFile"] $credentials = [PSCredential]::Empty if ($($global:config[$moduleName]["proxyServerUsername"]) -and $($global:config[$moduleName]["ProxyServerPassword"])) { $securePass = $($global:config[$moduleName]["ProxyServerPassword"]) | ConvertTo-SecureString -Key $global:credentialKey $credentials = New-Object System.Management.Automation.PSCredential -ArgumentList $($global:config[$moduleName]["proxyServerUsername"]), $securePass } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime return [ProxySettings]::new($credentials, "", $http, $https, $noProxy, $certFile) } function New-AksHciContainerRegistry { <# .DESCRIPTION Create container registry settings to be used for the Aks Hci deployment .PARAMETER server The container registry server name .PARAMETER credential Credential to connect to the container registry (if required) .OUTPUTS Container Registry object .EXAMPLE $credential = Get-Credential $registry = New-AksHciContainerRegistry -server "ecpacr.azurecr.io" -credential $credential #> param ( [Parameter(Mandatory=$true)] [String] $server, [Parameter()] [PSCredential] $credential = [PSCredential]::Empty ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime return [ContainerRegistry]::new($credential, $server) } function Invoke-AksHciRotateCACertificate { <# .DESCRIPTION Rotate cloudagent CA certificate .PARAMETER activity Activity name to use when updating progress #> param ( [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_rotate_moc_ca_certificate) Invoke-MocRotateCACertificate -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_cloud_certificates) Invoke-AksHciPatchCloudCertificates -activity $activity Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Invoke-AksHciPatchCloudCertificates { <# .DESCRIPTION Patch all the cloudagent certificate in the cluster .PARAMETER activity Activity name to use when updating progress #> param ( [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_nodeagent_certificates) Repair-AksHciCerts -force Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_mgmt_certificates) $clusters = Get-AksHciCluster foreach($cluster in $clusters) { Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_patching_cluster_certificates, $cluster.Name)) Repair-AksHciClusterCerts -Name $cluster.Name -fixCloudCredentials -patchLoadBalancer -force } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_patching_cloud_certificates_complete) Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } #endregion #region Installation and Provisioning functions function Install-AksHciInternal { <# .DESCRIPTION The main deployment method for AksHci. This function is responsible for installing MOC stack and the management appliance/cluster. .PARAMETER activity Activity name to use when updating progress #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installing) $startCmdletTime = Get-Date try { # Pre-requisite Install-Moc -activity $activity Get-AksHciPackage -Version (Get-AksHciVersion) Install-Kva -activity $activity } catch { try { $skipClean = Get-AksHciConfigValue -name "skipCleanOnFailure" if (-not $skipClean) { # Save the logs to temp $zipName = [io.Path]::GetTempFileName() + ".zip" Get-AksHciLogs -zipName $zipName | Out-Null $errorMessage += "`r`n Logs are available at $zipName" Uninstall-AksHci -SkipConfigCleanup:$True -activity $activity -Confirm:$false } } catch { Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message "$activity - $_" } Set-AksHciConfigValue -name "installState" -value ([InstallState]::InstallFailed) throw $_ } Write-Status -moduleName $moduleName $($AksHciLocMessage.akshci_installation_complete) Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installed) } function Initialize-AksHciEnvironment { <# .DESCRIPTION Executes steps to prepare the environment for AksHci operations. .PARAMETER createConfigIfNotPresent Whether the call should create a new AksHci deployment configuration if one is not already present. .PARAMETER skipMgmtKubeConfig Whether the call should skip a check to ensure that a appliance/management kubeconfig is present. .PARAMETER activity Activity name to use when updating progress #> param ( [Switch]$createConfigIfNotPresent, [Switch]$skipMgmtKubeConfig, [Switch]$skipInstallationCheck, [parameter(DontShow)] [String]$activity = "Preparing Environment" ) Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_initializing_environment) -moduleName $moduleName Import-AksHciConfig -createIfNotPresent:($createConfigIfNotPresent.IsPresent) -activity $activity Initialize-Environment -checkForUpdates:$false -moduleName $script:moduleName if (-not $skipInstallationCheck.IsPresent) { if (-not (Test-IsProductInstalled -moduleName $moduleName -activity $activity)) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_module_not_installed, $moduleName))), $true) } } if (-not ($skipMgmtKubeConfig.IsPresent)) { Get-Kva -activity $activity | Out-Null } } function Get-AksHciVersion { <# .SYNOPSIS Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .DESCRIPTION Get the current Kubernetes version of Azure Kubernetes Service on Azure Stack HCI. .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [parameter(DontShow)] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Trace-CmdletError -ErrorMessage $_ ` -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $global:config[$modulename]["version"] } function Update-AksHci { <# .SYNOPSIS Update the Azure Kubernetes Service host to the latest Kubernetes version. .DESCRIPTION Update the Azure Kubernetes Service host to the latest Kubernetes version. Performed as step updates, updating to the next available version until latest version is achieved. .PARAMETER AsJob Execute asynchronously as a background job .PARAMETER activity Activity name to use when updating progress #> [CmdletBinding()] param ( [Parameter()] [Switch] $AsJob, [parameter(DontShow)] [String] $activity ) <# 1. Check if versions later than current version are available a. If yes, prompt for upgrade b. If no, return silenty, printing a message 2. If upgrade is requested, do the following check a. 3. In case of multiple available newer versions a. Run upgrade from current version to next, repeating until latest available version is achieved b. Upon failure, return to last successful version to be upgraded to 4. Handle No target cluster scenarios 5. Handle No Target and Mgmt cluster scenarios 6. Handle scenario when the product is not installed #> $startCmdletTime = Get-Date $startingVersion = "" $versionsToUpgrade = @() $targetAksHciVersion = "" $updateAksHciCorrelationId = "" $stepUpdateAksHciCorrelationId = "" if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } trap { #This is original PowerShell Error event which is same for all cmdlets Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -AksHciTargetVersion $targetAksHciVersion #This is new error event specific to Update cmdlet #This is the 'PowerShell.UpdateAksHci.StepUpdate.Complete' Error Event Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -IsSuccess $false ` -IsStepUpdateAksHciComplete $true ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId ` -StepUpdateAksHciCorrelationId $stepUpdateAksHciCorrelationId #This is the 'PowerShell.UpdateAksHci.Complete' Error Event Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -IsSuccess $false ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId throw $_ } #Get the initial/starting Akshci version $startingVersion = Get-AksHciVersion if ($AsJob) { return New-BackgroundJob -name $activity -cmdletName $MyInvocation.MyCommand.Name -argDictionary $PSBoundParameters } Initialize-AksHciEnvironment -activity $activity $curState = Get-ConfigurationValue -module $moduleName -type ([Type][InstallState]) -name "installState" if ($null -ne $curState) { switch ($curState) { ([InstallState]::Updating) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_update_in_progress, $moduleName)) return } } } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_updating) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_current_version, $(Get-AksHciVersion))) $updates = Get-AksHciUpdates if ($updates.Count -eq 0) { Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_latest_version) return } ## Set caCertRotationThreshold value if not set $certThreshold = Get-ConfigurationValue -Name "caCertRotationThreshold" -module $moduleName if ((-not $certThreshold) -or ($certThreshold -eq 0)) { Set-ConfigurationValue -name "caCertRotationThreshold" -value $global:caCertRotationThreshold -module $moduleName } #Fetch deploymentID to pass to Update Moc during Upgrade. This can be removed once May2022 release is obselete $deploymentId = Get-ConfigurationValue -Name "deploymentId" -module $moduleName Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_suitable_version) $currentVersion = (Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName).Version $versionsToUpgrade = For ($v = $($updates.Keys).indexOf($currentVersion) - 1; $v -ge 0; $v-- ) { $tmpUpdate = $updates[$v] if (!$tmpUpdate.CanUpgradeTo) { break } $tmpUpdate.Version } #Get the value of the latest version for which the system will get updated to (Target version) $targetAksHciVersion ="" if ($versionsToUpgrade) { #For 1 element, it treats it as string and not array #Therefore, build the array and access its last element $versionsUpgradeValues = @() $versionsUpgradeValues += $versionsToUpgrade #Access the last element of the array $targetAksHciVersion = $versionsUpgradeValues[-1] } # Check if we are able to find a version to upgrade to if (!$versionsToUpgrade) { throw [CustomException]::new(($($AksHciLocMessage.akshci_unable_to_update)), $true) } #One event for each instance of [Update-AksHci] (i.e. before entering in update loop capture Start update) #This Update Start event should have a correlation id that is the same for all events in a single execution instance of [Update-AksHci] $updateAksHciCorrelationId = $(New-Guid) Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails) ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId ` -IsUpdateAksHciStarted $true Set-AksHciConfigValue -name "installState" -value ([InstallState]::Updating) $versionsToUpgrade | ForEach-Object { Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_upgrade_version, $_)) $currentInstallationPath = $global:config[$modulename]["installationPackageDir"] # We found a version to Upgrade to # 1. Download the package Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_package_version, $_)) Get-AksHciPackage -Version $_ $currentMocVersion = Get-MocVersion -activity $activity Get-KvaVersion -activity $activity | Out-Null #Before starting an update, Capture this event, in case update is hung; Get the update details for next possible update #For every new update, generate a new updateCorrelationId so that start and end events can be correlated $stepUpdateAksHciCorrelationId = $(New-Guid) $stepUpdateStartTime = Get-Date Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails) ` -AksHciInitialVersion $(Get-AksHciVersion) ` -AksHciTargetVersion $_ ` -StepUpdateAksHciCorrelationId $stepUpdateAksHciCorrelationId ` -IsStepUpdateAksHciStarted $true ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId try { $newInstallationPath = [io.Path]::Combine($global:config[$modulename]["workingDir"], $_) Set-AksHciConfigValue -name "installationPackageDir" -value $newInstallationPath New-Item -ItemType Directory -Force -Path $newInstallationPath | Out-Null # Trigger the platform update Update-Moc -activity $activity -version $_ -deploymentId $deploymentId # Trigger the appliance update - What happens when appliance update fails. Update-Kva -activity $activity -version $_ # Set the version, once successful Set-AksHciConfigValue -name "version" -value $_ #After every successful update, send a Success Event Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $stepUpdateStartTime ` -IsSuccess $true ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -StepUpdateAksHciCorrelationId $stepUpdateAksHciCorrelationId ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId ` -IsStepUpdateAksHciComplete $true } catch { Set-AksHciConfigValue -name "installState" -value ([InstallState]::UpdateFailed) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_update_failed, $_.Exception.Message)) Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_cleaning_up_updates) # Cleanup and Revert Set-AksHciConfigValue -name "installationPackageDir" -value $currentInstallationPath # Revert the platform Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_reverting_version, $currentMocVersion)) Update-Moc -activity $activity -version $currentMocVersion -deploymentId $deploymentId # Do we need to cleanup the downloaded package - keep it, so we customers may attempt to update again throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_update_failed, $_)) } # When upgrading from 1.0.1.10628 to 1.0.2.10719, there is a bug which can cause billing to go out of policy. # To prevent this from happening, in a chained upgrade case, force a sync inbetween upgrades. try { Sync-AksHciBilling | Out-Null } catch [Exception] { $errorMessageString = $_.Exception.Message.ToString() Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_failed_billing_sync, $errorMessageString)) } } Set-AksHciConfigValue -name "installState" -value ([InstallState]::Installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName $currentVersion = (Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName).Version if ($currentVersion -ine $updates[0].Version) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_update_not_latest, $currentVersion))), $true) } $isCertificateRotationRequired = Test-CloudCACertificateNearingExpiry -expiryThresholdDays $global:config[$moduleName]["caCertRotationThreshold"] if ($isCertificateRotationRequired) { Invoke-AksHciRotateCACertificate -activity $activity } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -AksHciTargetVersion $targetAksHciVersion Trace-CmdletUpdateAksHci -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -IsSuccess $true ` -AksHciInitialVersion $startingVersion ` -UpgradePath $versionsToUpgrade ` -AksHciTargetVersion $targetAksHciVersion ` -AksHciCurrentVersion $(Get-AksHciVersion) ` -UpdateAksHciCorrelationId $updateAksHciCorrelationId } #endregion #region Helper Functions function Get-AksHciPackage { <# .DESCRIPTION Downloads the package of the specified AksHCI Version .PARAMETER Version Version #> param ( [Parameter(Mandatory=$true)] [String]$Version ) # Validate the version Get-ProductRelease -Version $Version -moduleName $moduleName | Out-Null } function Get-AksHciLatestVersion { <# .DESCRIPTION Get the latest AksHci version #> $catalog = Get-LatestCatalog -moduleName $moduleName return $catalog.ProductStreamRefs[0].ProductReleases[0].Version } function Get-AksHciUpdates { <# .SYNOPSIS List the available Kubernetes updates for Azure Kubernetes Service on Azure Stack HCI in order. .DESCRIPTION List the available Kubernetes updates for Azure Kubernetes Service on Azure Stack HCI in order from latest to earliest. #> [CmdletBinding()] param () $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime throw $_ } Initialize-AksHciEnvironment $latestRelease = Get-LatestRelease -moduleName $moduleName $currentRelease = Get-ProductRelease -Version (Get-AksHciVersion) -moduleName $moduleName $latestVersion = $latestRelease.Version $currentVersion = $currentRelease.Version $upgradePath = [ordered]@{} if ($latestVersion -ieq $currentVersion) { return } # There may be more updates that users might have not applied. # Show them the complete list, so they are aware of what will be updated # Assumption here is that product releases would be returned in order $updateReleases = Get-ProductReleasesUptoVersion -Version $currentVersion -moduleName $moduleName $targetKubernetesVersions = Get-TargetClusterKubernetesVersions $updateReleases | ForEach-Object { $tmp = $_ $tmpVersion = $tmp.Version $supportedK8sVersions = Get-AvailableKubernetesVersions -akshciVersion $tmpVersion $computedRelease = @{ Version = $tmpVersion; SupportedKubernetesVersions = $supportedK8sVersions; CanUpgradeTo = $false; } if ($tmpVersion -ieq $currentVersion) { $computedRelease += @{ Comments = "This is your CURRENT Version"; } } if ($tmpVersion -ieq $latestVersion) { $computedRelease += @{ Comments = "This is the LATEST Version"; } } if ([System.Version]$tmpVersion -gt [System.Version]$currentVersion) { $script:canupgrade = $true try { # Validate that this powershell module is compatible with the proposed upgrade Test-ModuleCompatibility -Version $tmpVersion | out-null } catch [Exception] { $computedRelease += @{ Recommendation = $_.Exception.Message.ToString(); } $script:canupgrade = $false Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime } if ($script:canupgrade) { # Validate that the current target cluster k8s versions are still supported by the proposed upgrade if ($targetKubernetesVersions -and $targetKubernetesVersions.Count -gt 0) { foreach($targetVersion in $targetKubernetesVersions) { if (-not ($supportedK8sVersions.OrchestratorVersion.Contains($targetVersion))) { $computedRelease += @{ Recommendation = "Target Cluster Kubernetes Version $targetVersion is not in the list of supported Kubernetes versions (" + $supportedK8sVersions.OrchestratorVersion + ") for $tmpVersion. Please upgrade your target clusters to one of the kubernetes versions supported by $tmpVersion to unblock"; } $script:canupgrade = $false break } } } } if ($script:canupgrade) { $computedRelease.CanUpgradeTo = $true $computedRelease += @{ Recommendation = "You can upgrade to AksHci Version [$tmpVersion]"; } } } $upgradePath[$tmpVersion] = $computedRelease; } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return $upgradePath } function Test-SupportedKubernetesVersion { <# .DESCRIPTION Test if the specified kubernetes version is supported by the current deployment .PARAMETER K8sVersion Kubernetes version to test .PARAMETER imageType Image type can be Windows or Linux #> param ( [Parameter(Mandatory=$true)] [String] $K8sVersion, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String] $imageType ) $availableVersions = Get-AvailableKubernetesVersions foreach($version in $availableVersions) { if (($version.OS -ieq $imageType) -and ($version.OrchestratorVersion -ieq $k8sVersion)) { return } } throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_unsupported_k8s, $k8sVersion, $imageType))), $true) } function Get-NextKubernetesVersionForUpgrade { <# .DESCRIPTION Get the next Kubernetes Version for Upgrade. .PARAMETER Name Cluster name .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $Name, [parameter(DontShow)] [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $upgrades = Get-KvaClusterUpgrades -Name $Name -activity $activity if ($upgrades.AvailableUpgrades.Count -eq 0) { return $null } $kubernetesVersionArray = @() foreach($availableVersion in $upgrades.AvailableUpgrades) { $kubernetesVersionArray += Get-CleanInputKubernetesVersion -KubernetesVersion $availableVersion.kubernetesVersion -Semver } $sorted = $kubernetesVersionArray | ForEach-Object { new-object System.Version ($_) } | Sort-Object -Descending $highestUpgradeAvailable = $sorted[0].ToString() return "v$highestUpgradeAvailable" } function Get-CleanInputKubernetesVersion { <# .DESCRIPTION Cleans the input kubernetes verison .PARAMETER KubernetesVersion KubernetesVersion string to be cleaned. .PARAMETER Semver Semver switch to enforce semver valid output. #> param ( [Parameter(Mandatory=$true)] [String]$KubernetesVersion, [Parameter()] [Switch] $Semver ) $splitVersion = $KubernetesVersion.Split("-") if ($Semver.IsPresent) { $cleanVersion = $splitVersion[0] -replace '[v]','' } else { $cleanVersion = $splitVersion[0] } return $cleanVersion } function Get-AvailableKubernetesVersions { <# .DESCRIPTION Returns the kubernetes versions (by OS) that are supported by the specified AksHci release .PARAMETER akshciVersion AksHci Release version. Defaults to the version of the current deployment #> param ( [Parameter()] [String] $akshciVersion ) $result = @() if (-not $akshciVersion) { $akshciVersion = Get-AksHciVersion } # Get the Manifest for the specified Version $productRelease = Get-ProductRelease -version $akshciVersion -module $moduleName foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { foreach ($fileRelease in $subProductRelease.ProductFiles) { if (-not $fileRelease.CustomData.K8sPackages) { continue } $fileRelease.CustomData.K8SPackages | ForEach-Object { $version = [ordered]@{ 'OrchestratorType' = "Kubernetes"; 'OrchestratorVersion' = $("v"+$_.Version); 'OS' = $fileRelease.CustomData.BaseOSImage.OperatingSystem; 'IsPreview' = $false } $result += New-Object -TypeName PsObject -Property $version } } } } return $result } function Confirm-Configuration { <# .DESCRIPTION Validates the configuration .PARAMETER useStagingShare Requests a staging share to be used for downloading binaries and images (for private testing) .PARAMETER stagingShare The staging share endpoint to use when useStagingShare is requested .PARAMETER vnet Vnet to be validated, if provided #> param ( [Switch] $useStagingShare, [String] $stagingShare, [VirtualNetwork] $vnet ) if ($useStagingShare.IsPresent -and [string]::IsNullOrWhiteSpace($stagingShare)) { throw $($GenericLocMessage.generic_staging_share_unspecified) } # Test networking on hosts if ($vnet) { # Commenting this to unblock customers and proper fix be made for Nov # Confirm-Vnet -vnet $vnet } } function Set-AksHciRegistration { <# .DESCRIPTION Register an AksHci with Azure. Calls Connect-AzAccount under the covers. .PARAMETER SubscriptionId SubscriptionId is an azure subscription id. .PARAMETER TenantId TenantId is an azure tenant id. .PARAMETER ArmAccessToken ArmAccessToken is the token for accessing arm. .PARAMETER GraphAccessToken GraphAccessToken is the token for accessing the graph. .PARAMETER AccountId AccountId is an azure account id. .PARAMETER EnvironmentName EnvironmentName is the intented public cloud. .PARAMETER Credential Credential is a PSCredential holding a user's Service Principal. .PARAMETER ResourceGroupName ResourceGroupName is the name of the azure resource group to place arc resources. .PARAMETER Region Region is the name of the azure resource group to place arc resources. .PARAMETER UseDeviceAuthentication UseDeviceAuthentication outputs a code to be used in the browser. .PARAMETER SkipLogin SkipLogin skips the Connect-AzAccount call. Useful in automation or when running from a connected shell. .PARAMETER activity Activity name to use when updating progress #> param( [Parameter(Mandatory = $true)] [string] $SubscriptionId, [Parameter(Mandatory = $false)] [string] $TenantId, [Parameter(Mandatory = $false)] [string] $ArmAccessToken, [Parameter(Mandatory = $false)] [string] $GraphAccessToken, [Parameter(Mandatory = $false)] [string] $AccountId, [Parameter(Mandatory = $false)] [string] $EnvironmentName = $global:azureCloud, [Parameter(Mandatory = $true)] [string] $ResourceGroupName, [Parameter(Mandatory = $false)] [string] $Region, [Parameter(Mandatory = $false)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [Switch] $UseDeviceAuthentication, [Parameter(Mandatory = $false)] [Switch] $SkipLogin, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ErrorMessage $_ ` -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -skipMgmtKubeConfig -activity $activity -skipInstallationCheck if (-not $SkipLogin.IsPresent) { Write-Host "`n`n" #Needed for output formatting Set-AzureLogin -SubscriptionId $SubscriptionId -TenantId $TenantId -ArmAccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId -EnvironmentName $EnvironmentName -Credential $Credential -UseDeviceAuthentication:$UseDeviceAuthentication.IsPresent } $kubernetesProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes $kubernetesConfigProvider = Get-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration # The RPs should always exist but just in case arm is down, bail out. if (($null -eq $kubernetesProvider) -or ($null -eq $kubernetesProvider)) { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ChildStageName "KubernetesProvidersNotExists" ` -ErrorMessage "Unable to check registered Resource Providers" ` -StartCmdletTime $startCmdletTime throw [CustomException]::new(($($AksHciLocMessage.akshci_resource_provider_err)), $true) } if (($kubernetesProvider[0].RegistrationState -ne "Registered") -or ($kubernetesConfigProvider[0].RegistrationState -ne "Registered")) { Write-Status -moduleName $moduleName -Verbose -msg " Kubernetes Resource Providers are not registered for the current logged in tenant. Please run the following commands. With the azure cli: az provider register --namespace Microsoft.Kubernetes az provider register --namespace Microsoft.KubernetesConfiguration With Azure Powershell: Register-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes Register-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration Registration is an asynchronous process and may take approximately 10 minutes. You can monitor the registration process with the following commands: With the azure cli: az provider show -n Microsoft.Kubernetes -o table az provider show -n Microsoft.KubernetesConfiguration -o table With Azure Powershell: Get-AzResourceProvider -ProviderNamespace Microsoft.Kubernetes Get-AzResourceProvider -ProviderNamespace Microsoft.KubernetesConfiguration " throw [CustomException]::new(($($AksHciLocMessage.akshci_k8s_err)), $true) } if ($Region -eq "") { $rg = Get-AzResourceGroup -Name $ResourceGroupName $Region = $rg.Location.ToLower().replace(' ', '') } $isValidLocation = $false # in the case of an invalid location, build a string of all the locations to return to the user. $locationErrorString = "" foreach($location in $kubernetesProvider.Locations) { $cleanLocation = $location.ToLower().replace(' ', '') if ($Region -eq $cleanLocation) { $isValidLocation = $true } $locationErrorString += "$cleanLocation," } if (-not $isValidLocation) { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -ChildStageName "RegionNotValid" ` -ErrorMessage "$Region is not a valid Region for AksHci" ` -StartCmdletTime $startCmdletTime throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $AksHciLocMessage.akshci_invalid_region, $Region, $locationErrorString))), $true) } Set-KvaRegistration -azureResourceGroup $ResourceGroupName -azureLocation $Region Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime } function Get-AksHciRegistration { <# .DESCRIPTION Gets the Registration for AksHci. #> $startCmdletTime = Get-Date trap { Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -StartCmdletTime $startCmdletTime throw $_ } Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -StartCmdletTime $startCmdletTime return Get-KvaRegistration } function Set-AzureLogin { <# .DESCRIPTION Performs an Azure Login. Calls Connect-AzAccount under the covers. .PARAMETER SubscriptionId SubscriptionId .PARAMETER TenantId TenantId .PARAMETER ArmAccessToken ArmAccessToken .PARAMETER GraphAccessToken GraphAccessToken .PARAMETER AccountId AccountId .PARAMETER EnvironmentName EnvironmentName .PARAMETER Credential Credential .PARAMETER UseDeviceAuthentication UseDeviceAuthentication .PARAMETER activity Activity name to use when updating progress #> param( [Parameter(Mandatory = $true)] [string] $SubscriptionId, [Parameter(Mandatory = $false)] [string] $TenantId, [Parameter(Mandatory = $false)] [string] $ArmAccessToken, [Parameter(Mandatory = $false)] [string] $GraphAccessToken, [Parameter(Mandatory = $false)] [string] $AccountId, [Parameter(Mandatory = $false)] [string] $EnvironmentName, [Parameter(Mandatory = $false)] [PSCredential] $Credential, [Parameter(Mandatory = $false)] [Switch] $UseDeviceAuthentication, [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name ) if($EnvironmentName -eq $AzurePPE) { Add-AzEnvironment -Name $AzurePPE -PublishSettingsFileUrl "https://windows.azure-test.net/publishsettings/index" -ServiceEndpoint "https://management-preview.core.windows-int.net/" -ManagementPortalUrl "https://windows.azure-test.net/" -ActiveDirectoryEndpoint "https://login.windows-ppe.net/" -ActiveDirectoryServiceEndpointResourceId "https://management.core.windows.net/" -ResourceManagerEndpoint "https://api-dogfood.resources.windows-int.net/" -GalleryEndpoint "https://df.gallery.azure-test.net/" -GraphEndpoint "https://graph.ppe.windows.net/" -GraphAudience "https://graph.ppe.windows.net/" | Out-Null } Disconnect-AzAccount | Out-Null if($null -ne $Credential) { if ([string]::IsNullOrEmpty($TenantId)) { throw [CustomException]::new(($($AksHciLocMessage.akshci_empty_tenantid)), $true) } else { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -Credential $Credential -ServicePrincipal | Out-Null } } elseif([string]::IsNullOrEmpty($ArmAccessToken) -or [string]::IsNullOrEmpty($GraphAccessToken) -or [string]::IsNullOrEmpty($AccountId)) { # Interactive login $IsIEPresent = Test-Path "$env:SystemRoot\System32\ieframe.dll" if([string]::IsNullOrEmpty($TenantId)) { if($IsIEPresent -and (-not $UseDeviceAuthentication)) { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId | Out-Null } else # Use -UseDeviceAuthentication as IE Frame is not available to show Azure login popup { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId -UseDeviceAuthentication | Out-Null } } else { if($IsIEPresent -and (-not $UseDeviceAuthentication)) { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId | Out-Null } else # Use -UseDeviceAuthentication as IE Frame is not available to show Azure login popup { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -UseDeviceAuthentication | Out-Null } } } else { # Not an interactive login if([string]::IsNullOrEmpty($TenantId)) { Connect-AzAccount -Environment $EnvironmentName -SubscriptionId $SubscriptionId -AccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId | Out-Null } else { Connect-AzAccount -Environment $EnvironmentName -TenantId $TenantId -SubscriptionId $SubscriptionId -AccessToken $ArmAccessToken -GraphAccessToken $GraphAccessToken -AccountId $AccountId | Out-Null } } } function New-AksHciStorageContainer { <# .DESCRIPTION Creates a new cloud storage container .PARAMETER activity Activity name to use when updating progress .PARAMETER Name The name of the new storage container .PARAMETER Path The path where the vhds will be stored #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$Path ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage $_ ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -status $($AksHciLocMessage.akshci_new_storage_container) -moduleName $moduleName $cloudLocation = (Get-MocConfig)["cloudLocation"] New-MocContainer -name $Name -path $Path -location $cloudLocation Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_storage_container_created) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Get-AksHciStorageContainer { <# .DESCRIPTION Gets the storage containers .PARAMETER activity Activity name to use when updating progress .PARAMETER Name The name of the storage container, if not present returns all #> param ( [parameter(DontShow)] [String]$activity = $MyInvocation.MyCommand.Name, [Parameter()] [String]$Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ErrorMessage $_ ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_storage_container_info) $cloudLocation = (Get-MocConfig)["cloudLocation"] $result = Get-MocContainer -name $Name -location $cloudLocation Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime return $result } function Install-AksHciCsiSmb { <# .DESCRIPTION Installs csi smb plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_install_csi_smb_plugin) Set-KvaCsiSmb -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_smb_plugin_installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Install-AksHciCsiNfs { <# .DESCRIPTION Installs csi nfs plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_install_csi_nfs_plugin) Set-KvaCsiNfs -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_nfs_plugin_installed) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Uninstall-AksHciCsiSmb { <# .DESCRIPTION Uninstalls csi smb plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_uninstall_csi_smb_plugin) Reset-KvaCsiSmb -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_smb_plugin_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Uninstall-AksHciCsiNfs { <# .DESCRIPTION Uninstalls csi nfs plugin in an AKS-HCI cluster. .PARAMETER ClusterName clusterName .PARAMETER activity Activity name to use when updating progress #> param ( [Parameter(Mandatory=$true)] [String] $ClusterName, [parameter(DontShow)] [String] $activity ) $startCmdletTime = Get-Date if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $ClusterName" } trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } Initialize-AksHciEnvironment -activity $activity Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($AksHciLocMessage.akshci_uninstall_csi_nfs_plugin) Reset-KvaCsiNfs -ClusterName $ClusterName Write-SubStatus -moduleName $moduleName $($AksHciLocMessage.akshci_csi_nfs_plugin_uninstalled) Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime } function Confirm-Vnet { <# .DESCRIPTION Validate if the vnet configuration is valid #> param( [VirtualNetwork] $vnet ) $isMultinode = Test-MultiNodeDeployment $vSwitchName = $vnet.VswitchName $vipPoolStart = $vnet.VipPoolStart $vipPoolEnd = $vnet.VipPoolEnd if ($isMultinode) { Get-ClusterNode -ErrorAction Stop | ForEach-Object { Test-VipPoolAgainstVnicAddressPrefix -switchName $vSwitchName -multiNode -nodeName $_.Name -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd -vlanID $vnet.Vlanid } } else { Test-VipPoolAgainstVnicAddressPrefix -switchName $vSwitchName -nodeName ($env:computername) -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd -vlanID $vnet.Vlanid } } function Get-TraceConfigDetails { <# .DESCRIPTION Capture the config map details for the TraceCmdlet module. #> [TraceConfigDetails]$traceConfig = [TraceConfigDetails]::new() if (-not [string]::IsNullOrWhiteSpace($global:config[$modulename]["version"])) { $productRelease = Get-ProductRelease -version $global:config[$modulename]["version"] -moduleName $moduleName $traceConfig.Offer = "$($productRelease.ProductName)" } $traceConfig.DeploymentId = "$($global:config[$modulename]["deploymentId"])" $traceConfig.Catalog = "$($global:config[$modulename]["catalog"])" $traceConfig.Audience = "$($global:config[$modulename]["ring"])" $traceConfig.AksHciVersion = "$($global:config[$modulename]["version"])" $traceConfig.ModuleName = $moduleName $traceConfig.ModuleVersion = $moduleVersion return $traceConfig | ConvertTo-Json } function New-AksHciLoadBalancerSetting { <# .DESCRIPTION Create a object for load balancer. .PARAMETER name Name of the LoadBalancer .PARAMETER loadBalancerSku Choice of load balancer for kubernetes service none/haproxy/kubevip/metalb .PARAMETER loadBalancerVMSize Size of load balance VM .PARAMETER loadBalancerCount Number of load Balancer VMs .OUTPUTS LoadBalancer object .EXAMPLE New-AksHciLoadBalancerSetting -Name "lb1" -loadBalancerSku KubeVIP .EXAMPLE New-AksHciLoadBalancerSetting -Name "lb1" -loadBalancerSku HAProxy -loadBalancerVmSize Standard_A4_v2 #> param ( [Parameter(Mandatory=$true)] [string] $name, [Parameter(Mandatory=$true)] [LoadBalancerSku] $LoadBalancerSku, [Parameter(Mandatory=$false)] [Vmsize] $vmSize, [Parameter(Mandatory=$false)] [int] $loadBalancerCount ) if ([string]::IsNullOrWhiteSpace($VmSize)) { $VmSize = $global:defaultLoadBalancerVmSize } if ( ($loadBalancerCount) -and ($loadBalancerCount -ge 1) -and ($LoadBalancerSku -ne [LoadBalancerSku]::HAProxy) ) { Write-Host $($AksHciLocMessage.akshci_scaling_loadbalancer_unsupported_for_non_haproxy_type) throw [CustomException]::new(($($AksHciLocMessage.akshci_scaling_loadbalancer_unsupported_for_non_haproxy_type)), $true) } if ( (-not $loadBalancerCount) -or ($loadBalancerCount -eq 0) ) { $loadBalancerCount = 1 } switch ($LoadBalancerSku) { ([LoadBalancerSku]::HAProxy) { #Today, if HAProxy is chosen both APi server and K8s service will use HAProxy $serviceLoadBalancerSku = [LoadBalancerSku]::HAProxy break } ([LoadBalancerSku]::None) { # Bring your LB $serviceLoadBalancerSku = $LoadBalancerSku $LoadBalancerSku = [LoadBalancerSku]::KubeVIP break } Default { $serviceLoadBalancerSku = $LoadBalancerSku $LoadBalancerSku = [LoadBalancerSku]::KubeVIP break } } if ($serviceLoadBalancerSku -eq [LoadBalancerSku]::None) { Write-Host "NOTE: None for load balancer implies the user will use their own Load Balancer, it will be user's responsibility to configure External Load Balancer correctly." } return [LoadBalancerSettings]::new($Name, $LoadBalancerSku, $serviceLoadBalancerSku, $VmSize, $loadBalancerCount) } function Test-CloudCACertificateNearingExpiry { <# .DESCRIPTION Validate that powershell remoting to a node is working. #> param ( [Parameter(Mandatory=$true)] [int] $expiryThresholdDays = $global:caCertRotationThreshold ) $result = Get-MocCertificate -name $global:cloudAgentCACertName -expiryDays $expiryThresholdDays | ConvertFrom-Json if ($result.tags[0].NearingExpiry -eq "true") { return $true } return $false } function Set-AksHciLoadBalancer { <# .DESCRIPTION Updates a load balancer object specifically, the number of replicas .PARAMETER clusterName Name of the cluster hosting the loadblancer .PARAMETER loadBalancerVMSize Size of load balance VM .PARAMETER loadBalancerCount Number of load balancer replicas .EXAMPLE Set-AksHciLoadBalancer -clusterName "cluster1" -loadBalancerCount 2 #> param ( [Parameter(Mandatory=$true)] [string] $clusterName, [Parameter(Mandatory=$false)] [int] $loadBalancerCount, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name ) $startCmdletTime = Get-Date trap { Write-ModuleEventLog -moduleName $moduleName -entryType Error -eventId 100 -message "$activity - $_" Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) ` -CmdletParameters $PSBoundParameters ` -StartCmdletTime $startCmdletTime ` -ErrorMessage $_ Trace-CmdletError -ConfigDetails $(Get-TraceConfigDetails) -ErrorMessage $_ -CmdletParameters $PSBoundParameters -StartCmdletTime $startCmdletTime if ($ErrorActionPreference -ne [System.Management.Automation.ActionPreference]::SilentlyContinue) { throw $_ } } $mgmtCluster = (Get-KvaConfig)["kvaName"] if ($clusterName -ieq $mgmtCluster) { throw $($AksHciLocMessage.akshci_scaling_unsupported) } Set-KvaLoadBalancer -clusterName $clusterName -loadBalancerCount $loadBalancerCount -activity $activity Write-StatusWithProgress -activity $activity -status $($GenericLocMessage.generic_done) -completed -moduleName $moduleName Trace-Cmdlet -ConfigDetails $(Get-TraceConfigDetails) ` -StartCmdletTime $startCmdletTime ` -CmdletParameters $PSBoundParameters } #endregion # SIG # Begin signature block # MIInoQYJKoZIhvcNAQcCoIInkjCCJ44CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCCFGXRE/wh8OXwE # G22vD1h8XHZOouR/sS8azvujAT3whaCCDYEwggX/MIID56ADAgECAhMzAAACzI61 # lqa90clOAAAAAALMMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjIwNTEyMjA0NjAxWhcNMjMwNTExMjA0NjAxWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQCiTbHs68bADvNud97NzcdP0zh0mRr4VpDv68KobjQFybVAuVgiINf9aG2zQtWK # No6+2X2Ix65KGcBXuZyEi0oBUAAGnIe5O5q/Y0Ij0WwDyMWaVad2Te4r1Eic3HWH # UfiiNjF0ETHKg3qa7DCyUqwsR9q5SaXuHlYCwM+m59Nl3jKnYnKLLfzhl13wImV9 # DF8N76ANkRyK6BYoc9I6hHF2MCTQYWbQ4fXgzKhgzj4zeabWgfu+ZJCiFLkogvc0 # RVb0x3DtyxMbl/3e45Eu+sn/x6EVwbJZVvtQYcmdGF1yAYht+JnNmWwAxL8MgHMz # xEcoY1Q1JtstiY3+u3ulGMvhAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUiLhHjTKWzIqVIp+sM2rOHH11rfQw # UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1 # ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDcwNTI5MB8GA1UdIwQYMBaAFEhu # ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu # bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w # Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx # MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAeA8D # sOAHS53MTIHYu8bbXrO6yQtRD6JfyMWeXaLu3Nc8PDnFc1efYq/F3MGx/aiwNbcs # J2MU7BKNWTP5JQVBA2GNIeR3mScXqnOsv1XqXPvZeISDVWLaBQzceItdIwgo6B13 # vxlkkSYMvB0Dr3Yw7/W9U4Wk5K/RDOnIGvmKqKi3AwyxlV1mpefy729FKaWT7edB # d3I4+hldMY8sdfDPjWRtJzjMjXZs41OUOwtHccPazjjC7KndzvZHx/0VWL8n0NT/ # 404vftnXKifMZkS4p2sB3oK+6kCcsyWsgS/3eYGw1Fe4MOnin1RhgrW1rHPODJTG # AUOmW4wc3Q6KKr2zve7sMDZe9tfylonPwhk971rX8qGw6LkrGFv31IJeJSe/aUbG # dUDPkbrABbVvPElgoj5eP3REqx5jdfkQw7tOdWkhn0jDUh2uQen9Atj3RkJyHuR0 # GUsJVMWFJdkIO/gFwzoOGlHNsmxvpANV86/1qgb1oZXdrURpzJp53MsDaBY/pxOc # J0Cvg6uWs3kQWgKk5aBzvsX95BzdItHTpVMtVPW4q41XEvbFmUP1n6oL5rdNdrTM # j/HXMRk1KCksax1Vxo3qv+13cCsZAaQNaIAvt5LvkshZkDZIP//0Hnq7NnWeYR3z # 4oFiw9N2n3bb9baQWuWPswG0Dq9YT9kb+Cs4qIIwggd6MIIFYqADAgECAgphDpDS # AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK # V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0 # IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0 # ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla # MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS # ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT # H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB # AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG # OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S # 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz # y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7 # 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u # M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33 # X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl # XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP # 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB # l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF # RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM # CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ # BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud # DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO # 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0 # LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p # Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y # Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB # FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw # cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA # XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY # 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj # 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd # d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ # Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf # wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ # aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j # NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B # xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96 # eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7 # r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I # RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIZdjCCGXICAQEwgZUwfjELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z # b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAsyOtZamvdHJTgAAAAACzDAN # BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor # BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQg9333JCic # f1/VCN9xTr9+W1BTrsP9JTNTZXgfspQMB+wwQgYKKwYBBAGCNwIBDDE0MDKgFIAS # AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN # BgkqhkiG9w0BAQEFAASCAQAjQEUNAmwg7XXlXsA6hQLu2FB8l5I6upDIvu/S1wTZ # hrIIJBKE7Kos556Iz/nJcmyOflyKeZovsZ+A1VwJJV5S2k3aT+rP73EhVlnCpXcz # hQrT1hqMNDft/pLRjNH3RsAkk0+DzvD5zZnDPjDRlTLqLByV8fR4j8ZQtVH9Hrki # XhwxzisfAryhSapV5xhjWa+6xrjZ7VFC33q+Npy82sPog2laYqlSxWDNDhzwW0ny # 5Hlv9GV2bdnHtwxS7fYoxguHECcYROvdGEW5MEL7UNBxoD6mDT3reOnnGciG3wYB # QdkvwhicMtDqmvUQvKtZNBatyLeQF02P7kQ8mRRg09nMoYIXADCCFvwGCisGAQQB # gjcDAwExghbsMIIW6AYJKoZIhvcNAQcCoIIW2TCCFtUCAQMxDzANBglghkgBZQME # AgEFADCCAVEGCyqGSIb3DQEJEAEEoIIBQASCATwwggE4AgEBBgorBgEEAYRZCgMB # MDEwDQYJYIZIAWUDBAIBBQAEILmTgWllMzo0VcCuHyIlOKg6tC6C4iWS1moIWnf6 # ttkWAgZi/ZPuEo4YEzIwMjIwODI1MTkzNDQ3LjQ3OVowBIACAfSggdCkgc0wgcox # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1p # Y3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJjAkBgNVBAsTHVRoYWxlcyBUU1Mg # RVNOOkVBQ0UtRTMxNi1DOTFEMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFt # cCBTZXJ2aWNloIIRVzCCBwwwggT0oAMCAQICEzMAAAGawHWixCFtPoUAAQAAAZow # DQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0 # b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3Jh # dGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcN # MjExMjAyMTkwNTE3WhcNMjMwMjI4MTkwNTE3WjCByjELMAkGA1UEBhMCVVMxEzAR # BgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1p # Y3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2Eg # T3BlcmF0aW9uczEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046RUFDRS1FMzE2LUM5 # MUQxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2UwggIiMA0G # CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDacgasKiu3ZGEU/mr6A5t9oXAgbsCJ # q0NnOu+54zZPt9Y/trEHSTlpE2n4jua4VnadE4sf2Ng8xfUxDQPO4Vb/3UHhhdHi # CnLoUIsW3wtE2OPzHFhAcUNzxuSpk667om4o/GcaPlwiIN4ZdDxSOz6ojSNT9azs # KXwQFAcu4c9tsvXiul99sifC3s2dEEJ0/BhyHiJAwscU4N2nm1UDf4uMAfC1B7SB # QZL30ssPyiUjU7gIijr1IRlBAdBYmiyR0F7RJvzy+diwjm0Isj3f8bsVIq9gZkUW # xxFkKZLfByleEo4BMmRMZE9+AfTprQne6mcjtVAdBLRKXvXjLSXPR6h54pttsShK # aV3IP6Dp6bXRf2Gb2CfdVSxty3HHAUyZXuFwguIV2OW3gF3kFQK3uL6QZvN8a6KB # 0hto06V98Otey1OTOvn1mRnAvVu4Wj8f1dc+9cOPdPgtFz4cd37mRRPEkAdX2Yae # TgpcNExa+jCbOSN++VtNScxwu4AjPoTfQjuQ+L1p8SMZfggT8khaXaWWZ9vLvO7P # IwIZ4b2SK3/XmWpk0AmaTha5QG0fu5uvd4YZ/xLuI/kiwHWcTykviAZOlwkrnsoY # ZJJ03RsIAWv6UHnYjAI8G3UgCFFlAm0nguQ3rIX54pmujS83lgrm1YqbL2Lrlhmi # 98Mk2ktCHCXKRwIDAQABo4IBNjCCATIwHQYDVR0OBBYEFF+2nlnwnNtR6aVZvQqV # yK02K9FwMB8GA1UdIwQYMBaAFJ+nFV0AXmJdg/Tl0mWnG1M1GelyMF8GA1UdHwRY # MFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01p # Y3Jvc29mdCUyMFRpbWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNybDBsBggrBgEF # BQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9w # a2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAo # MSkuY3J0MAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYBBQUHAwgwDQYJKoZI # hvcNAQELBQADggIBAAATu4fMRtRH20+nNzGAXFxdXEpRPTfbM0LJDeNe4QCxj0FM # +wrJdu6UKrM2wQuO31UDcQ4nrUJBe81N6W2RvEa8xNXjbO0qzNitwUfOVLeZp6HV # GcNTtYEMAvK9k//0daBFxbp04BzMaIyaHRy7y/K/zZ9ckEw7jF9VsJqlrwqkx9Hq # I/IBsCpJdlTtKBl/+LRbD8tWvw6FDrSkv/IDiKcarPE0BU6//bFXvZ5/h7diE13d # qv5DPU5Kn499HvUOAcHG31gr/TJPEftqqK40dfpB+1bBPSzAef58rJxRJXNJ661G # bOZ5e64EuyIQv0Vo5ZptaWZiftQ5pgmztaZCuNIIvxPHCyvIAjmSfRuX7Uyke0k2 # 9rSTruRsBVIsifG39gldsbyjOvkDN7S3pJtTwJV0ToC4VWg00kpunk72PORup31a # hW99fU3jxBh2fHjiefjZUa08d/nQQdLWCzadttpkZvCgH/dc8Mts2CwrcxCPZ5p9 # VuGcqyFhK2I6PS0POnMuf70R3lrl5Y87dO8f4Kv83bkhq5g+IrY5KvLcIEER5kt5 # uuorpWzJmBNGB+62OVNMz92YJFl/Lt+NvkGFTuGZy96TLMPdLcrNSpPGV5qHqnHl # r/wUz9UAViTKJArvSbvk/siU7mi29oqRxb0ahB4oYVPNuv7ccHTBGqNNGol4MIIH # cTCCBVmgAwIBAgITMwAAABXF52ueAptJmQAAAAAAFTANBgkqhkiG9w0BAQsFADCB # iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMp # TWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEw # OTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1WjB8MQswCQYDVQQGEwJVUzETMBEGA1UE # CBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9z # b2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQ # Q0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOThpkzntHIh # C3miy9ckeb0O1YLT/e6cBwfSqWxOdcjKNVf2AX9sSuDivbk+F2Az/1xPx2b3lVNx # WuJ+Slr+uDZnhUYjDLWNE893MsAQGOhgfWpSg0S3po5GawcU88V29YZQ3MFEyHFc # UTE3oAo4bo3t1w/YJlN8OWECesSq/XJprx2rrPY2vjUmZNqYO7oaezOtgFt+jBAc # nVL+tuhiJdxqD89d9P6OU8/W7IVWTe/dvI2k45GPsjksUZzpcGkNyjYtcI4xyDUo # veO0hyTD4MmPfrVUj9z6BVWYbWg7mka97aSueik3rMvrg0XnRm7KMtXAhjBcTyzi # YrLNueKNiOSWrAFKu75xqRdbZ2De+JKRHh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9 # fvzZnkXftnIv231fgLrbqn427DZM9ituqBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdH # GO2n6Jl8P0zbr17C89XYcz1DTsEzOUyOArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7X # KHYC4jMYctenIPDC+hIK12NvDMk2ZItboKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiE # R9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/ # eKtFtvUeh17aj54WcmnGrnu3tz5q4i6tAgMBAAGjggHdMIIB2TASBgkrBgEEAYI3 # FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQWBBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAd # BgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEE # AYI3TIN9AQEwQTA/BggrBgEFBQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraW9wcy9Eb2NzL1JlcG9zaXRvcnkuaHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMI # MBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMB # Af8EBTADAQH/MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1Ud # HwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3By # b2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggrBgEFBQcBAQRO # MEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2Vy # dHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3J0MA0GCSqGSIb3DQEBCwUAA4IC # AQCdVX38Kq3hLB9nATEkW+Geckv8qW/qXBS2Pk5HZHixBpOXPTEztTnXwnE2P9pk # bHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gng # ugnue99qb74py27YP0h1AdkY3m2CDPVtI1TkeFN1JFe53Z/zjj3G82jfZfakVqr3 # lbYoVSfQJL1AoL8ZthISEV09J+BAljis9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHC # gRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTpkbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6 # MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEU # BHG/ZPkkvnNtyo4JvbMBV0lUZNlz138eW0QBjloZkWsNn6Qo3GcZKCS6OEuabvsh # VGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJsWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+ # fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrp # NPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0dFtq0Z4+7X6gMTN9vMvpe784cETRkPHI # qzqKOghif9lwY1NNje6CbaUFEMFxBmoQtB1VM1izoXBm8qGCAs4wggI3AgEBMIH4 # oYHQpIHNMIHKMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUw # IwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMSYwJAYDVQQLEx1U # aGFsZXMgVFNTIEVTTjpFQUNFLUUzMTYtQzkxRDElMCMGA1UEAxMcTWljcm9zb2Z0 # IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcGBSsOAwIaAxUAAbquMnUCam/m7Ox1 # Uv/GNs1jmu+ggYMwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAN # BgkqhkiG9w0BAQUFAAIFAOax9X4wIhgPMjAyMjA4MjUyMTE5NThaGA8yMDIyMDgy # NjIxMTk1OFowdzA9BgorBgEEAYRZCgQBMS8wLTAKAgUA5rH1fgIBADAKAgEAAgIc # bgIB/zAHAgEAAgIRsDAKAgUA5rNG/gIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgor # BgEEAYRZCgMCoAowCAIBAAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUA # A4GBAKO7FxzYUP7GXlWfGK1xCUvJN1OREZJD8ivrwEu5Pce4G9EBq5/FoXdqb5EG # 5Zy885/RDYshsPx05Zs/2cE1DwMNTljKdRq/dm6O7xPzk0xcckNMVYY62z0X3fDC # QlGJp0+Ek15BqAryoMgI5Gi2FAECjqPIY5+vN1korbOX8Y+BMYIEDTCCBAkCAQEw # gZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAGawHWixCFtPoUA # AQAAAZowDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0B # CRABBDAvBgkqhkiG9w0BCQQxIgQgb3knWrplo+gH8j85p1ghdMDAv0twUylFIdPh # oDZCdX4wgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHkMIG9BCABTkDjOBEUfZnligJi # L539Lx+nsr/NFVTnKFX030iNYDCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwAhMzAAABmsB1osQhbT6FAAEAAAGaMCIEIBeRT3KqDwHWxAzwAR1S # ya9rSCgPuSCRuHgvdhdFGZZHMA0GCSqGSIb3DQEBCwUABIICAK+Z1ekpx0hiuIsK # U13ZFEcP1Ql2DWn0HzNbpU+4roeGZ9YoEBkKrMKFzBUpuL1VHWpBKyR8s5TiBIg/ # rLqN4WbMG6+2gIg6WpPlNehyuVEha2EKNvB33NLgXmVdKKwXPzHuGg0/smheLxRN # c2bs2DIv9+4ru4uVvfPIyqFc9bitS8JHYbBtRi9wnK5C4IXLfMzK7Xwgkj9Pvjaj # T7oFm06t28lgteC0MCtu1SqDKBDOPqrVKGfqvh+92FZ32Glh/z8Xl1Z/lPBIpJEg # S+hpD/4mz52IIdhRV7rzq0quRLaQ64v11hu8hk7C/VlhxOu39e6TRwg9hJUb4hfW # IDULPmotkyyctDCeDpbo39o0qt4SVMALvrbvpNsvaruES8R1D3Ug+SGliV3HzDIh # 62UsOztC1g1Lns5Thpu8Ays8xBDsXr746S+act005lAB44enY8c/jug4fAp9v6ON # eLmn1gK/3nzuYTkUJuQJ/nBFnTDnoivewt+1Fu3kU1ounnSm0tGBjKVR41zfSgTZ # uynmhScGRmo1zdakr5RFlDuQC4diJD0F8GkdO9HqP2i/3hAxOTYondZf6qm7hugZ # Moec2MPCb9SQPNbhiFoD2ejf3kKLGDuenOh9rxP1SnETTixBulW7f8qc9TRAhGnK # BKn0Ue03EhMeBvOFCF1n2yRN06ut # SIG # End signature block |