AksGMSA.psm1
$ErrorActionPreference = "Stop" function Install-ToolingRequirements { <# .SYNOPSIS Install the tooling requirements. .PARAMETER KubectlVersion The release version of the kubectl tool. Valid values for this parameter are release tags from: https://github.com/kubernetes/kubernetes/releases. #> [CmdletBinding()] Param( [String]$KubectlVersion="latest" ) PROCESS { Write-Output "Installing the tooling requirements." # Install Azure CLI, if it's not found in PATH. try { Assert-FoundInPath "az" Write-Verbose "The tool 'az' already found in PATH. Skipping installation." } catch [System.Exception] { Write-Verbose "Installing Azure CLI." $installerPath = Join-Path $env:TEMP "AzureCLI.msi" Start-FileDownload -URL "https://aka.ms/installazurecliwindows" -Destination $installerPath $p = Start-Process -FilePath "msiexec.exe" -ArgumentList @("/I", $installerPath, "/quiet") -Wait -PassThru if($p.ExitCode) { Throw "Failed to install Azure CLI. Exit code: $($p.ExitCode)" } Remove-Item -Force -Path $installerPath Add-ToPathEnvVar -Path "${env:ProgramFiles(x86)}\Microsoft SDKs\Azure\CLI2\wbin" -Target ([System.EnvironmentVariableTarget]::Machine) } # Install required PowerShell modules, if not installed already. $powerShellModules = @( @{ "Name" = "Az.Accounts" "MinimumVersion" = [System.Version]::new(2, 2, 8) }, @{ "Name" = "Az.Resources" "MinimumVersion" = [System.Version]::new(4, 1, 0) } @{ "Name" = "Az.Aks" "MinimumVersion" = [System.Version]::new(2, 1, 0) }, @{ "Name" = "Az.Compute" "MinimumVersion" = [System.Version]::new(4, 12, 0) }, @{ "Name" = "Az.ManagedServiceIdentity" "MinimumVersion" = [System.Version]::new(0, 7, 3) }, @{ "Name" = "Az.KeyVault" "MinimumVersion" = [System.Version]::new(3, 4, 3) } ) Install-PackageProvider -Name "NuGet" -Force -Confirm:$false | Out-Null foreach ($m in $powerShellModules) { try { $module = Start-Job ` -ScriptBlock { Param($Name) Import-Module $Name -PassThru } ` -ArgumentList $m["Name"] | Receive-Job -Wait if($module.Version -lt $m["MinimumVersion"]) { Throw "PowerShell module '$($m["Name"])' needs upgrade to version '$($m["MinimumVersion"])'." } Write-Verbose "PowerShell module '$($m["Name"])' minimum version '$($m["MinimumVersion"])' already installed." } catch [System.Exception] { Write-Verbose "Installing PowerShell module '$($m["Name"])' minimum version '$($m["MinimumVersion"])'." Install-Module -Name $m["Name"] -MinimumVersion $m["MinimumVersion"] -Force -AllowClobber -Confirm:$false } } # Install kubectl, if it's not found in PATH. try { Assert-FoundInPath "kubectl" Write-Verbose "The tool 'kubectl' already found in PATH. Skipping installation." } catch [System.Exception] { $dest = Join-Path $HOME ".azure-kubectl" Write-Verbose "Installing ${KubectlVersion} kubectl to ${dest}." Install-AzAksKubectl -Destination $dest -Version $KubectlVersion -Force -Confirm:$false Add-ToPathEnvVar -Path $dest -Target ([System.EnvironmentVariableTarget]::User) } # Install Active Directory RSAT (Remote Server Administration Tools), if not installed already. try { Start-Job -ScriptBlock { Import-Module "ActiveDirectory" } | Receive-Job -Wait Write-Verbose "PowerShell module 'ActiveDirectory' already installed." } catch [System.Exception] { Write-Verbose "Installing PowerShell RSAT (Remote Server Administration Tools)." $osInfo = Get-CimInstance -ClassName Win32_OperatingSystem if($osInfo.ProductType -eq 1) { # The current platform is a Windows client. Add-WindowsCapability -Name "Rsat.ActiveDirectory.DS-LDS.Tools~~~~0.0.1.0" -Online | Out-Null } else { # The current platform is a Windows server. $feature = Install-WindowsFeature -Name "RSAT-AD-PowerShell" -Confirm:$false if(!$feature.Success) { Throw "Failed to install RSAT-AD-PowerShell Windows feature." } } } Write-Output "Successfully installed the tooling requirements." } } function Get-AksGMSAParameters { <# .SYNOPSIS Get the input parameters for the AKS gMSA setup. #> [CmdletBinding()] Param() BEGIN { Import-Module "Az.Resources" } PROCESS { $params = [Ordered]@{ "aks-cluster-name" = @{ "Prompt" = "AKS cluster name" "Description" = "The name given to the AKS cluster." "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "aks-cluster-rg-name" = @{ "Prompt" = "AKS cluster resource group name" "Description" = "The Azure resource group where the AKS cluster is deployed." "ValidationScriptBlock" = { Assert-ValidAzureResourceGroupName $_ } } "aks-win-node-pools-names" = @{ "Prompt" = "Comma-separated list with the AKS Windows node pools' names" "Description" = ( "The AKS node pools' names with the Kubernetes Windows agents. " + "These pools are going to be authorized to fetch the configured gMSA.") "Type" = "String[]" "ValidationScriptBlock" = { Assert-ValidAksNodePoolNames $_ } } "domain-dns-server" = @{ "Prompt" = "AD Domain DNS server" "Description" = ( "Active Directory domain DNS server. This is going to be set as the main " + "DNS server for the AKS Windows hosts. The AD DNS server should be able to " + "forward requests, if they are not within the domain, otherwise the Windows " + "hosts' upstream DNS will be broken.") "ValidationScriptBlock" = { [System.Net.IPAddress]$_ } } "domain-fqdn" = @{ "Prompt" = "AD Domain FQDN" "Description" = "The fully qualified domain name for the Active Directory domain." "ValidationScriptBlock" = { Assert-ValidDnsName $_ } } "gmsa-name" = @{ "Prompt" = "gMSA name" "Description" = "The Active Directory group Managed Service Account (gMSA) name." "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "gmsa-domain-user-name" = @{ "Prompt" = "gMSA domain user name" "Description" = "The name of the domain user authorized to fetch the gMSA." "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "gmsa-domain-user-password" = @{ "Prompt" = "gMSA domain user password" "Description" = "The password of the domain user authorized to fetch the gMSA." "AsSecureString" = $true } "azure-location" = @{ "Prompt" = "Azure location" "Description" = "The Azure location used for the new resources (Azure key vault and Azure user-assigned managed identity)." "ValidationScriptBlock" = { Write-Host -ForegroundColor Cyan "Validating Azure location" Assert-ValidAzureLocation $_ } } "akv-name" = @{ "Prompt" = "AKV name" "Description" = "The name of the Azure key vault where the gMSA domain user credential will be stored." "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "akv-secret-name" = @{ "Prompt" = "AKV secret name" "Description" = "The name of the AKV secret that will contain the gMSA domain user credential." "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "ami-name" = @{ "Prompt" = "Azure MI name" "Description" = "The name of the Azure MI (managed identity) that will be used to fetch the AKV gMSA secret." "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "gmsa-spec-name" = @{ "Prompt" = "The gMSA spec name" "Description" = "The name given to the Kubernetes gMSA credential spec resource." "ValidationScriptBlock" = { Assert-ValidGMSACredentialSpecName $_ } } "logs-directory" = @{ "Prompt" = "Local logs directory" "Description" = ( "The local directory path where the AKS Windows hosts' logs will be stored. " + "If the directory doesn't exist, it will be created when the logs are collected.`n" + "Defaults to 'C:\gmsa-logs' if empty value is given.") "Default" = "C:\gmsa-logs" "Optional" = $true } "domain-controller-address" = @{ "Prompt" = "AD domain controller address" "Description" = ( "The address of the AD domain controller. This is going to be used " + "for remote commands execution, when creating the gMSA and the domain user. " + "This is not needed, if the AD domain controller doesn't allow remote commands execution. " + "More details in the docs.") "Optional" = $true } "domain-admin-user-name" = @{ "Prompt" = "Domain admin user name" "Description" = ( "The domain admin user name. This is going to be used when creating the gMSA " + "and the gMSA domain user. Give empty value if you don't have domain admin credential. " + "More details in the docs.") "Optional" = $true "ValidationScriptBlock" = { Assert-ValidResourceName $_ } } "domain-admin-user-password" = @{ "Prompt" = "Domain admin user password" "Description" = ( "The domain admin user password. This is going to be used when creating " + "the gMSA and the gMSA domain user. Give empty value if you don't have domain admin credential. " + "More details in the docs.") "AsSecureString" = $true "Optional" = $true } } $inputParams = @{} foreach ($p in $params.GetEnumerator()) { $kwargs = $p.Value $inputParams[$p.Name] = Get-InputParameter @kwargs } return $inputParams } } function Confirm-AksADDCConnectivity { <# .SYNOPSIS Validates that the AKS Windows hosts have proper connectivity to the Active Directory Domain Controller (AD DC) machine. .PARAMETER AksResourceGroupName The AKS cluster resource group name. .PARAMETER AksClusterName The AKS cluster name. .PARAMETER AksWindowsNodePoolsNames The AKS Windows node pools' names. .PARAMETER DomainDnsServer The AD DC DNS server. .PARAMETER RootDomainName The AD DC root domain name. .PARAMETER ContainerImage The container image used for validation. A Windows server image with PowerShell is required. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$AksResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$AksClusterName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAksNodePoolNames $_ })] [String[]]$AksWindowsNodePoolsNames, [Parameter(Mandatory=$true)] [ValidateScript({ [System.Net.IPAddress]$_ $true })] [String]$DomainDnsServer, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidDnsName $_ })] [String]$RootDomainName, [String]$ContainerImage="mcr.microsoft.com/windows/servercore:ltsc2019" ) BEGIN { Import-Module "Az.Aks", "Az.Compute" Assert-FoundInPath "kubectl" } PROCESS { Write-Output "Validating connectivity with AD DC for AKS Windows node pools: $($AksWindowsNodePoolsNames -join ', ')." $aksCluster = Get-AzAksCluster -ResourceGroupName $AksResourceGroupName -Name $AksClusterName foreach($nodePoolName in $AksWindowsNodePoolsNames) { $nodePool = Get-AzAksNodePool -ClusterObject $aksCluster -Name $nodePoolName if($nodePool.OsType -ne [Microsoft.Azure.Management.Compute.Models.OperatingSystemTypes]::Windows) { Throw "The agent pool '${nodePoolName}' is not a Windows agent pool." } if($nodePool.AgentPoolType -ne "VirtualMachineScaleSets") { Throw "Unsupported agent pool type. The pool '${nodePoolName}' uses '$($nodePool.AgentPoolType)' type. Only 'VirtualMachineScaleSets' type is supported." } } $manifests = @( @{ "kind" = "ConfigMap" "apiVersion" = "v1" "metadata" = @{ "name" = "aks-addc-validation" "labels" = @{ "app" = "aks-addc-validation" } } "data" = @{ "validate-aks-addc.ps1" = Get-ADDCValidationPSScript -DomainDnsServer $DomainDnsServer -RootDomainName $RootDomainName } }, @{ "kind" = "DaemonSet" "apiVersion" = "apps/v1" "metadata" = @{ "name" = "aks-addc-validation" "labels" = @{ "app" = "aks-addc-validation" } } "spec" = @{ "selector" = @{ "matchLabels" = @{ "app" = "aks-addc-validation" } } "template" = @{ "metadata" = @{ "labels" = @{ "app" = "aks-addc-validation" } } "spec" = @{ "containers" = @( @{ "name" = "aks-addc-validation" "image" = $ContainerImage "command" = @("ping", "-t", "localhost") "volumeMounts" = @( @{ "name" = "aks-addc-validation" "mountPath" = "/aks-addc-validation" } ) } ) "volumes" = @( @{ "name" = "aks-addc-validation" "configMap" = @{ "defaultMode" = 420 "name" = "aks-addc-validation" } } ) "affinity" = @{ "nodeAffinity" = @{ "requiredDuringSchedulingIgnoredDuringExecution" = @{ "nodeSelectorTerms" = @( @{ "matchExpressions" = @( @{ "key" = "agentpool" "operator" = "In" "values" = $AksWindowsNodePoolsNames } ) } ) } } } } } } } ) $jsonManifests = $manifests | ForEach-Object { ConvertTo-Json -Depth 100 $_ } Write-Verbose "Setting up the validation DaemonSet." Install-KubernetesManifests -Manifests $jsonManifests try { Write-Verbose "Waiting until the validation DaemonSet is ready." # Wait until all the DaemonSet pods are ready. It times-out after 5 minutes of waiting. Start-ExecuteWithRetry ` -ScriptBlock ${function:Assert-ReadyDaemonSet} ` -ArgumentList @("aks-addc-validation") ` -RetryMessage "The validation DaemonSet 'aks-addc-validation' is not ready yet." ` -MaxRetryCount 30 -RetryInterval 10 $pods = Invoke-Kubectl "get pods -l app=aks-addc-validation -o json" | ConvertFrom-Json if($pods.items.Length -eq 0) { Throw "There are no AKS AD DC validation pods spawned. Probably something is wrong with the pods' scheduling, or the given AKS Windows nodes are down." } foreach($pod in $pods.items) { Write-Output "Validating Windows host '$($pod.spec.nodeName)'." Invoke-Kubectl "exec $($pod.metadata.name) -- powershell.exe /aks-addc-validation/validate-aks-addc.ps1" } } finally { Write-Verbose "Cleaning up the validation DaemonSet." Uninstall-KubernetesManifests -Manifests $jsonManifests } Write-Output "Successfully validated connectivity with AD DC for AKS Windows node pools: $($AksWindowsNodePoolsNames -join ', ')." } } function Confirm-AksGMSAConfiguration { <# .SYNOPSIS Validates that the AKS cluster has the proper gMSA configuration. .PARAMETER AksResourceGroupName The AKS cluster resource group name. .PARAMETER AksClusterName The AKS cluster name. .PARAMETER AksGMSADomainDnsServer The AKS gMSA DNS server. .PARAMETER AksGMSARootDomainName The AKS gMSA root domain name. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$AksResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$AksClusterName, [Parameter(Mandatory=$true)] [ValidateScript({ [System.Net.IPAddress]$_ $true })] [String]$AksGMSADomainDnsServer, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidDnsName $_ })] [String]$AksGMSARootDomainName ) BEGIN { Assert-FoundInPath "az" } PROCESS { $aksCluster = Invoke-CommandLine "az" "aks show -g $AksResourceGroupName -n $AksClusterName -o json" | ConvertFrom-Json $gmsaProfile = $aksCluster.windowsProfile.gmsaProfile $isEnabled = ($gmsaProfile.enabled -and $gmsaProfile.dnsServer -eq $AksGMSADomainDnsServer -and $gmsaProfile.rootDomainName -eq $AksGMSARootDomainName) if($isEnabled) { Write-Output "The AKS gMSA feature is properly configured." return } if(!$gmsaProfile.enabled) { $promptTitle = "The AKS cluster doesn't have the gMSA feature enabled." $promptQuestion = "Do you want to enable the gMSA feature now?" $yesOption = New-Object System.Management.Automation.Host.ChoiceDescription "&Yes", "Enable AKS gMSA now." $noOption = New-Object System.Management.Automation.Host.ChoiceDescription "&No", "Enable AKS gMSA later." } else { if($gmsaProfile.dnsServer -ne $AksGMSADomainDnsServer) { Write-Warning "The AKS gMSA DNS server is not configured correctly. Current value is '$($gmsaProfile.domainDnsServer)', and it needs to be '$AksGMSADomainDnsServer'." } if($gmsaProfile.rootDomainName -ne $AksGMSARootDomainName) { Write-Warning "The AKS gMSA root domain name is not configured correctly. Current value is '$($gmsaProfile.rootDomainName)', and it needs to be '$AksGMSARootDomainName'." } $promptTitle = "The AKS cluster doesn't have the gMSA feature properly configured." $promptQuestion = "Do you want to reconfigure the gMSA feature now?" $yesOption = New-Object System.Management.Automation.Host.ChoiceDescription "&Yes", "Reconfigure AKS gMSA now." $noOption = New-Object System.Management.Automation.Host.ChoiceDescription "&No", "Reconfigure AKS gMSA later." } $promptChoices = [System.Management.Automation.Host.ChoiceDescription[]]($yesOption, $noOption) $decision = $Host.UI.PromptForChoice($promptTitle, $promptQuestion, $promptChoices, -1) if($decision -eq 0) { Write-Output "Updating AKS gMSA configuration." Invoke-CommandLine "az" "aks update -g $AksResourceGroupName -n $AksClusterName --enable-windows-gmsa --gmsa-dns-server $AksGMSADomainDnsServer --gmsa-root-domain-name $AksGMSARootDomainName -o none" } } } function New-GMSADomainUser { <# .SYNOPSIS Create the gMSA standard domain user. This is going to be authorized to fetch the gMSA credentials. .PARAMETER Name The gMSA standard domain user name. .PARAMETER Password The gMSA standard domain user password. .PARAMETER DomainControllerAddress The AD domain controller address used for remote command execution. This parameter is optional. .PARAMETER DomainAdmin The domain admin used for remote command execution. This parameter is optional. .PARAMETER DomainAdminPassword The domain admin password used for remote command execution. This parameter is optional. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name, [Parameter(Mandatory=$true)] [SecureString]$Password, [String]$DomainControllerAddress, [ValidateScript({ Assert-ValidDomainUserName $_ })] [String]$DomainAdmin, [SecureString]$DomainAdminPassword ) BEGIN { Import-Module "ActiveDirectory" } PROCESS { $remoteParams = Get-ADRemoteParams $DomainControllerAddress $DomainAdmin $DomainAdminPassword $commonParams = @{ "Enabled" = $true "PasswordNeverExpires" = $true "CannotChangePassword" = $true "PassThru" = $true "Confirm" = $false } $adUser = Get-ADUser @remoteParams -Filter "Name -eq '$Name'" if(!$adUser) { Write-Verbose "Creating gMSA domain user." New-ADUser @remoteParams @commonParams -Name $Name -AccountPassword $Password } else { Write-Verbose "gMSA domain user already exists." Write-Verbose "Resetting gMSA domain user password." $adUser | Set-ADAccountPassword @remoteParams -Reset -NewPassword $Password -Confirm:$false Write-Verbose "Updating gMSA domain user info." $adUser | Set-ADUser @remoteParams @commonParams } } } function Remove-GMSADomainUser { <# .SYNOPSIS Remove the gMSA standard domain user. .PARAMETER Name The gMSA standard domain user name. .PARAMETER DomainControllerAddress The AD domain controller address used for remote command execution. This parameter is optional. .PARAMETER DomainAdmin The domain admin used for remote command execution. This parameter is optional. .PARAMETER DomainAdminPassword The domain admin password used for remote command execution. This parameter is optional. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name, [String]$DomainControllerAddress, [ValidateScript({ Assert-ValidDomainUserName $_ })] [String]$DomainAdmin, [SecureString]$DomainAdminPassword ) BEGIN { Import-Module "ActiveDirectory" } PROCESS { $remoteParams = Get-ADRemoteParams $DomainControllerAddress $DomainAdmin $DomainAdminPassword $adUser = Get-ADUser @remoteParams -Filter "Name -eq '$Name'" if(!$adUser) { Write-Output "The AD user doesn't exist." return } Write-Verbose "Removing the AD user." $adUser | Remove-ADUser @remoteParams -Confirm:$false } } function New-GMSA { <# .SYNOPSIS Create the gMSA. .PARAMETER Name The gMSA name. .PARAMETER AuthorizedUser The domain user that will be authorized to fetch the gMSA credentials. .PARAMETER DomainControllerAddress The AD domain controller address used for remote command execution. This parameter is optional. .PARAMETER DomainAdmin The domain admin used for remote command execution. This parameter is optional. .PARAMETER DomainAdminPassword The domain admin password used for remote command execution. This parameter is optional. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$AuthorizedUser, [String]$DomainControllerAddress, [ValidateScript({ Assert-ValidDomainUserName $_ })] [String]$DomainAdmin, [SecureString]$DomainAdminPassword ) BEGIN { Import-Module "ActiveDirectory" } PROCESS { $remoteParams = Get-ADRemoteParams $DomainControllerAddress $DomainAdmin $DomainAdminPassword $gmsa = Get-ADServiceAccount @remoteParams -Filter "Name -eq '${Name}'" $domain = Get-ADDomain @remoteParams $spn = "host/${Name}", "host/${Name}.$($domain.DNSRoot)" $commonParams = @{ "DNSHostName" = "${Name}.$($domain.DNSRoot)" "PrincipalsAllowedToRetrieveManagedPassword" = $AuthorizedUser "Enabled" = $true "PassThru" = $true "Confirm" = $false } if(!$gmsa) { Write-Verbose "Creating gMSA." New-ADServiceAccount @remoteParams @commonParams -Name $Name -ServicePrincipalNames $spn } else { Write-Verbose "gMSA already exists." Write-Verbose "Updating gMSA info." $gmsa | Set-ADServiceAccount @remoteParams -ServicePrincipalNames $null $gmsa | Set-ADServiceAccount @remoteParams @commonParams -ServicePrincipalNames @{Add = $spn} } } } function Remove-GMSA { <# .SYNOPSIS Remove the gMSA. .PARAMETER Name The gMSA name. .PARAMETER DomainControllerAddress The AD domain controller address used for remote command execution. This parameter is optional. .PARAMETER DomainAdmin The domain admin used for remote command execution. This parameter is optional. .PARAMETER DomainAdminPassword The domain admin password used for remote command execution. This parameter is optional. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name, [String]$DomainControllerAddress, [ValidateScript({ Assert-ValidDomainUserName $_ })] [String]$DomainAdmin, [SecureString]$DomainAdminPassword ) BEGIN { Import-Module "ActiveDirectory" } PROCESS { $remoteParams = Get-ADRemoteParams $DomainControllerAddress $DomainAdmin $DomainAdminPassword $gmsa = Get-ADServiceAccount @remoteParams -Filter "Name -eq '${Name}'" if(!$gmsa) { Write-Output "The gMSA doesn't exist." return } Write-Verbose "Removing the gMSA." $gmsa | Remove-ADServiceAccount @remoteParams -Confirm:$false } } function New-GMSAAzureKeyVault { <# .SYNOPSIS Create the Azure key vault with the gMSA standard domain account credential. .PARAMETER ResourceGroupName The Azure key vault resource group name. .PARAMETER Location The Azure key vault location. .PARAMETER Name The Azure key vault name. .PARAMETER SecretName The name of the Azure key vault secret that holds the credential of the gMSA standard domain user. .PARAMETER GMSADomainUser The gMSA standard domain user name. .PARAMETER GMSADomainUserPassword The gMSA standard domain user password. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$ResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureLocation $_ })] [String]$Location, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$SecretName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidFQDNDomainUserName $_ })] [String]$GMSADomainUser, [Parameter(Mandatory=$true)] [SecureString]$GMSADomainUserPassword ) BEGIN { Import-Module "Az.KeyVault" } PROCESS { $akv = Get-AzKeyVault -ResourceGroupName $ResourceGroupName -Name $Name -ErrorAction SilentlyContinue if(!$akv) { Write-Verbose "Creating the Azure key vault." $akv = New-AzKeyVault -ResourceGroupName $ResourceGroupName -Location $Location -Name $Name } Write-Verbose "Setting the Azure key vault secret with the gMSA domain user credentials." $secretValue = ConvertTo-SecureString -String "${GMSADomainUser}:$([System.Net.NetworkCredential]::new('', $GMSADomainUserPassword).Password)" -AsPlainText -Force Set-AzKeyVaultSecret -VaultName $Name -Name $SecretName -SecretValue $secretValue | Out-Null return $akv } } function Remove-GMSAAzureKeyVault { <# .SYNOPSIS Remove the Azure key vault with the gMSA standard domain account credential. .PARAMETER ResourceGroupName The Azure key vault resource group name. .PARAMETER Name The Azure key vault name. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$ResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name ) BEGIN { Import-Module "Az.KeyVault" } PROCESS { $akv = Get-AzKeyVault -ResourceGroupName $ResourceGroupName -Name $Name -ErrorAction SilentlyContinue if(!$akv) { Write-Output "The Azure key vault doesn't exist." return } Write-Verbose "Removing the Azure key vault." Remove-AzKeyVault -InputObject $akv -Force -Confirm:$false Write-Verbose "Purging the Azure key vault." Remove-AzKeyVault -InputObject $akv -InRemovedState -Force -Confirm:$false } } function New-GMSAManagedIdentity { <# .SYNOPSIS Create the user-assigned managed identity. This is going to be authorized to fetch the Azure key vault secret, that holds the gMSA standard domain user credential. .PARAMETER ResourceGroupName The user-assigned managed identity resource group name. .PARAMETER Location The user-assigned managed identity location. .PARAMETER Name The user-assigned managed identity name. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$ResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureLocation $_ })] [String]$Location, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name ) BEGIN { Import-Module "Az.ManagedServiceIdentity" } PROCESS { $mi = Get-AzUserAssignedIdentity -ResourceGroupName $ResourceGroupName -Name $Name -ErrorAction SilentlyContinue if(!$mi) { Write-Verbose "Creating the gMSA user-assigned managed identity." $mi = New-AzUserAssignedIdentity -ResourceGroupName $ResourceGroupName -Location $Location -Name $Name } return $mi } } function Remove-GMSAManagedIdentity { <# .SYNOPSIS Remove the user-assigned managed identity, used to fetch the gMSA standard domain user credential from AKV. .PARAMETER ResourceGroupName The user-assigned managed identity resource group name. .PARAMETER Name The user-assigned managed identity name. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$ResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name ) BEGIN { Import-Module "Az.ManagedServiceIdentity" } PROCESS { $mi = Get-AzUserAssignedIdentity -ResourceGroupName $ResourceGroupName -Name $Name -ErrorAction SilentlyContinue if(!$mi) { Write-Output "The user-assigned managed identity doesn't exist." return } Write-Verbose "Removing the user-assigned managed identity." Remove-AzUserAssignedIdentity -InputObject $mi -Confirm:$false } } function Grant-AkvAccessToAksWindowsHosts { <# .SYNOPSIS This function will: * Append the user-assigned managed identity to the AKS Windows agent pools. * Configure the Azure key vault read access policy for the user-assigned managed identity. .PARAMETER AksResourceGroupName The AKS cluster resource group name. .PARAMETER AksClusterName The AKS cluster name. .PARAMETER AksWindowsNodePoolsNames The AKS Windows node pools' names. .PARAMETER VaultResourceGroupName The Azure key vault resource group name. .PARAMETER VaultName The Azure key vault name. .PARAMETER ManagedIdentityResourceGroupName The user-assigned managed identity resource group name. .PARAMETER ManagedIdentityName The user-assigned managed identity name. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$AksResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$AksClusterName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAksNodePoolNames $_ })] [String[]]$AksWindowsNodePoolsNames, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$VaultResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$VaultName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$ManagedIdentityResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$ManagedIdentityName ) BEGIN { Import-Module "Az.ManagedServiceIdentity", "Az.KeyVault", "Az.Aks", "Az.Compute" } PROCESS { Write-Output "Granting Azure key vault access to the user-assigned managed identity." $mi = Get-AzUserAssignedIdentity -ResourceGroupName $ManagedIdentityResourceGroupName -Name $ManagedIdentityName $akv = Get-AzKeyVault -ResourceGroupName $VaultResourceGroupName -Name $VaultName if(!$akv) { Throw "The Azure key vault '${VaultName}' was not found under resource group '${VaultResourceGroupName}'." } $aksCluster = Get-AzAksCluster -ResourceGroupName $AksResourceGroupName -Name $AksClusterName foreach($nodePoolName in $AksWindowsNodePoolsNames) { $nodePool = Get-AzAksNodePool -ClusterObject $aksCluster -Name $nodePoolName if($nodePool.OsType -ne [Microsoft.Azure.Management.Compute.Models.OperatingSystemTypes]::Windows) { Throw "The agent pool '${nodePoolName}' is not a Windows agent pool." } if($nodePool.AgentPoolType -ne "VirtualMachineScaleSets") { Throw "Unsupported agent pool type. The pool '${nodePoolName}' uses '$($nodePool.AgentPoolType)' type. Only 'VirtualMachineScaleSets' type is supported." } } # Assign the managed identity to the Windows agent pools. $windowsVMSSList = Get-AzVmss -ResourceGroupName $aksCluster.NodeResourceGroup | Where-Object { $_.Tags["aks-managed-poolName"] -in $AksWindowsNodePoolsNames } if(!$windowsVMSSList) { Throw "No VMSS resources, for the Windows agent pools, were found under resource group '$($aksCluster.NodeResourceGroup)'." } $validIdentityTypes = @( [Microsoft.Azure.Management.Compute.Models.ResourceIdentityType]::UserAssigned, [Microsoft.Azure.Management.Compute.Models.ResourceIdentityType]::SystemAssignedUserAssigned ) foreach($vmss in $windowsVMSSList) { if($vmss.Identity.Type -notin $validIdentityTypes) { Throw "The VMSS '$($vmss.Name)' doesn't allow user-assigned managed identities. The current VMSS identity type is '$($vmss.Identity.Type)'." } $exists = $vmss.Identity.UserAssignedIdentities.Keys | Where-Object { $_ -eq $mi.Id } if($exists) { Write-Verbose "The user-assigned managed identity is already assigned to the Windows VMSS '$($vmss.Name)'." continue } $identityIDs = New-Object Collections.Generic.List[String] $vmss.Identity.UserAssignedIdentities.Keys | ForEach-Object { $identityIDs.Add($_) } $identityIDs.Add($mi.Id) Write-Verbose "Adding the user-assigned managed identity to the Windows VMSS '$($vmss.Name)'." Update-AzVmss ` -ResourceGroupName $aksCluster.NodeResourceGroup ` -Name $vmss.Name ` -IdentityType $vmss.Identity.Type ` -IdentityId $identityIDs | Out-Null } # Set the proper AKV access policies for the managed identity. Write-Verbose "Setting read Azure key vault access policy for the user-assigned managed identity." Start-ExecuteWithRetry ` -ScriptBlock { Param( [Parameter(Mandatory=$true)] [String]$VaultResourceGroupName, [Parameter(Mandatory=$true)] [String]$VaultName, [Parameter(Mandatory=$true)] [String]$ObjectID ) Set-AzKeyVaultAccessPolicy ` -ResourceGroupName $VaultResourceGroupName ` -VaultName $VaultName ` -ObjectId $ObjectID ` -PermissionsToSecrets "get" } ` -ArgumentList @($VaultResourceGroupName, $VaultName, $mi.PrincipalId) ` -MaxRetryCount 15 -RetryInterval 10 ` -RetryMessage "Failed to set Azure key vault access policy. Perhaps the managed identity is not yet visible to the Azure key vault. Retrying." Write-Output "Successfully granted access to the user-assigned managed identity." } } function Get-AksAgentPoolsAkvAccess { <# .SYNOPSIS This function will output the AKS agent pools' access to all the Azure key vaults found in the given resource groups. .PARAMETER AksResourceGroupName The AKS cluster resource group name. .PARAMETER AksClusterName The AKS cluster name. .PARAMETER VaultResourceGroupNames The resource group names where the Azure key vaults are found. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$AksResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$AksClusterName, [Parameter(Mandatory=$true)] [ValidateScript({ foreach($name in $_) { Assert-ValidAzureResourceGroupName $name | Out-Null } $true })] [String[]]$VaultResourceGroupNames ) BEGIN { Import-Module "Az.KeyVault", "Az.Aks", "Az.Compute" $env:SuppressAzurePowerShellBreakingChangeWarnings = $true } PROCESS { Write-Verbose "Getting all the Azure key vaults." $vaults = New-Object Collections.Generic.List[Microsoft.Azure.Commands.KeyVault.Models.PSKeyVault] foreach($rgName in $VaultResourceGroupNames) { Get-AzKeyVault -ResourceGroupName $rgName | ForEach-Object { $vaults.Add((Get-AzKeyVault -ResourceGroupName $_.ResourceGroupName -Name $_.VaultName)) } } if(!$vaults) { Throw "No Azure key vaults were found under resource groups '${VaultResourceGroupNames}'." } Write-Verbose "Checking if there are agent pools in the AKS cluster." $aksCluster = Get-AzAksCluster -ResourceGroupName $AksResourceGroupName -Name $AksClusterName if(!$aksCluster.AgentPoolProfiles) { Throw "No agent pools were found in the AKS cluster '${AksClusterName}'." } Write-Verbose "Checking AKS agent pools' secret accesses for all the Azure key vaults." $table = New-Object System.Data.DataTable $table.Columns.AddRange(@("AksAgentPool", "AzureKeyVault", "IsAuthorized")) foreach($pool in $aksCluster.AgentPoolProfiles) { $isAuthorized = $true foreach($vault in $vaults) { $vmss = Get-AzVmss -ResourceGroupName $aksCluster.NodeResourceGroup | Where-Object { $_.Tags["aks-managed-poolName"] -eq $pool.Name } $vaultPrincipals = $vault.AccessPolicies | Where-Object { ($_.PermissionsToSecrets -contains 'get') } $principalsWithAccess = $vmss.Identity.UserAssignedIdentities.Values.PrincipalId | Where-Object { $_ -in $vaultPrincipals.ObjectId } if(!$principalsWithAccess) { $isAuthorized = $false } $table.Rows.Add($pool.Name, $vault.VaultName, $isAuthorized) | Out-Null } } return $table } END { $env:SuppressAzurePowerShellBreakingChangeWarnings = $null } } function New-GMSACredentialSpec { <# .SYNOPSIS Create a new gMSA credential spec Kubernetes resource. .PARAMETER Name The spec name. .PARAMETER GMSAName The gMSA name. .PARAMETER ManagedIdentityResourceGroupName The gMSA managed identity resource group name. .PARAMETER ManagedIdentityName The gMSA managed identity name. .PARAMETER VaultName The gMSA Azure key vault name. .PARAMETER VaultGMSASecretName The gMSA Azure key vault secret name. .PARAMETER DomainControllerAddress The AD domain controller address used for remote command execution. This parameter is optional. .PARAMETER DomainUser The domain user used for remote command execution. This parameter is optional. .PARAMETER DomainUserPassword The domain user password used for remote command execution. This parameter is optional. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidGMSACredentialSpecName $_ })] [String]$Name, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$GMSAName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAzureResourceGroupName $_ })] [String]$ManagedIdentityResourceGroupName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$ManagedIdentityName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$VaultName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidResourceName $_ })] [String]$VaultGMSASecretName, [String]$DomainControllerAddress, [ValidateScript({ Assert-ValidDomainUserName $_ })] [String]$DomainUser, [SecureString]$DomainUserPassword ) BEGIN { Assert-FoundInPath "kubectl" } PROCESS { Write-Output "Creating gMSA credential spec." # Setup the GMSA credential spec. $manifest = Get-GMSACredentialSpecManifest ` -Name $Name ` -GMSAName $GMSAName ` -ManagedIdentityResourceGroupName $ManagedIdentityResourceGroupName ` -ManagedIdentityName $ManagedIdentityName ` -VaultName $VaultName ` -VaultGMSASecretName $VaultGMSASecretName ` -DomainControllerAddress $DomainControllerAddress ` -DomainUser $DomainUser ` -DomainUserPassword $DomainUserPassword Install-KubernetesManifests -Manifests (ConvertTo-Json -InputObject $manifest -Depth 100) # Setup the GMSACredentialSpec RBAC ClusterRole. Install-KubernetesManifests -Manifests (Get-GMSACredentialSpecClusterRoleManifest -SpecName $Name | ConvertTo-Json -Depth 100) # Setup the GMSACredentialSpec RBAC RoleBinding. Install-KubernetesManifests -Manifests (Get-GMSACredentialSpecRoleBindingManifest -SpecName $Name | ConvertTo-Json -Depth 100) } } function Remove-GMSACredentialSpec { <# .SYNOPSIS Remove a gMSA credential spec Kubernetes resource. .PARAMETER Name The spec name. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidGMSACredentialSpecName $_ })] [String]$Name ) BEGIN { Assert-FoundInPath "kubectl" } PROCESS { Write-Verbose "Deleting the gMSA credential spec." Invoke-Kubectl "delete gmsacredentialspec --ignore-not-found=true $Name" Write-Verbose "Deleting the gMSA credential spec RBAC ClusterRole." Uninstall-KubernetesManifests -Manifests (Get-GMSACredentialSpecClusterRoleManifest -SpecName $Name | ConvertTo-Json -Depth 100) Write-Verbose "Deleting the gMSA credential spec RBAC RoleBinding." Uninstall-KubernetesManifests -Manifests (Get-GMSACredentialSpecRoleBindingManifest -SpecName $Name | ConvertTo-Json -Depth 100) } } function Get-GMSASampleApplicationYAML { <# .SYNOPSIS Get a sample YAML manifest with a Kubernetes deployment using gMSA credential spec. .PARAMETER SpecName The gMSA credential spec name. .PARAMETER AksWindowsNodePoolsNames The AKS Windows node pools' names. .PARAMETER Name The application name. .PARAMETER Namespace The application namespace. .PARAMETER Labels Hashtable with the application labels. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidGMSACredentialSpecName $_ })] [String]$SpecName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAksNodePoolNames $_ })] [String[]]$AksWindowsNodePoolsNames, [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Name, [ValidateScript({ Assert-ValidResourceName $_ })] [String]$Namespace="default", [Hashtable]$Labels=@{} ) PROCESS { if(!$Name) { $Name = "${SpecName}-demo" } if(!$Labels["app"]) { $Labels["app"] = $Name } $labelsStr = New-Object Collections.Generic.List[String] foreach($item in $Labels.GetEnumerator()) { $labelsStr.Add("$($item.Name): $($item.Value)") } return @" --- kind: ConfigMap apiVersion: v1 metadata: labels: $($labelsStr -join "`n ") name: ${Name} namespace: ${Namespace} data: run.ps1: | `$ErrorActionPreference = "Stop" Write-Output "Configuring IIS with authentication." # Add required Windows features, since they are not installed by default. Install-WindowsFeature "Web-Windows-Auth", "Web-Asp-Net45" # Create simple ASP.Net page. New-Item -Force -ItemType Directory -Path 'C:\inetpub\wwwroot\app' Set-Content -Path 'C:\inetpub\wwwroot\app\default.aspx' -Value 'Authenticated as <B><%=User.Identity.Name%></B>, Type of Authentication: <B><%=User.Identity.AuthenticationType%></B>' # Configure IIS with authentication. Import-Module IISAdministration Start-IISCommitDelay (Get-IISConfigSection -SectionPath 'system.webServer/security/authentication/windowsAuthentication').Attributes['enabled'].value = `$true (Get-IISConfigSection -SectionPath 'system.webServer/security/authentication/anonymousAuthentication').Attributes['enabled'].value = `$false (Get-IISServerManager).Sites[0].Applications[0].VirtualDirectories[0].PhysicalPath = 'C:\inetpub\wwwroot\app' Stop-IISCommitDelay Write-Output "IIS with authentication is ready." C:\ServiceMonitor.exe w3svc --- apiVersion: apps/v1 kind: Deployment metadata: labels: $($labelsStr -join "`n ") name: ${Name} namespace: ${Namespace} spec: replicas: 1 selector: matchLabels: $($labelsStr -join "`n ") template: metadata: labels: $($labelsStr -join "`n ") spec: securityContext: windowsOptions: gmsaCredentialSpecName: ${SpecName} containers: - name: iis image: mcr.microsoft.com/windows/servercore/iis:windowsservercore-ltsc2019 imagePullPolicy: IfNotPresent command: - powershell args: - -File - /gmsa-demo/run.ps1 volumeMounts: - name: gmsa-demo mountPath: /gmsa-demo volumes: - configMap: defaultMode: 420 name: ${Name} name: gmsa-demo affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: agentpool operator: In values: - $($AksWindowsNodePoolsNames -join "`n - ") --- apiVersion: v1 kind: Service metadata: labels: $($labelsStr -join "`n ") name: ${Name} namespace: ${Namespace} spec: ports: - port: 80 targetPort: 80 selector: $($labelsStr -join "`n ") type: LoadBalancer "@ } } function Start-GMSACredentialSpecValidation { <# .SYNOPSIS Validate that the gMSA credential spec is functional. A pod using gMSA credential spec is started on each Windows host for validation purposes. .PARAMETER SpecName The gMSA credential spec name. .PARAMETER AksWindowsNodePoolsNames The AKS Windows node pools' names. .PARAMETER ContainerImage The container image used for validation. A Windows server core image with PowerShell is required. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidGMSACredentialSpecName $_ })] [String]$SpecName, [Parameter(Mandatory=$true)] [ValidateScript({ Assert-ValidAksNodePoolNames $_ })] [String[]]$AksWindowsNodePoolsNames, [String]$ContainerImage="mcr.microsoft.com/windows/servercore:ltsc2019" ) BEGIN { Assert-FoundInPath "kubectl" } PROCESS { Write-Output "Validating gMSA credential spec '${SpecName}'." Write-Verbose "Getting the gMSA credential spec details." $spec = Invoke-Kubectl "get gmsacredentialspec $SpecName -o json" | ConvertFrom-Json $domainName = $spec.credspec.DomainJoinConfig.DnsName $manifests = @( @{ "kind" = "ConfigMap" "apiVersion" = "v1" "metadata" = @{ "name" = "gmsa-spec-${SpecName}-validation" "labels" = @{ "app" = "gmsa-spec-${SpecName}-validation" } } "data" = @{ "validate-spec.ps1" = Get-GMSACredSpecValidationPSScript $domainName } }, @{ "kind" = "DaemonSet" "apiVersion" = "apps/v1" "metadata" = @{ "name" = "gmsa-spec-${SpecName}-validation" "labels" = @{ "app" = "gmsa-spec-${SpecName}-validation" } } "spec" = @{ "selector" = @{ "matchLabels" = @{ "app" = "gmsa-spec-${SpecName}-validation" } } "template" = @{ "metadata" = @{ "labels" = @{ "app" = "gmsa-spec-${SpecName}-validation" } } "spec" = @{ "securityContext" = @{ "windowsOptions" = @{ "gmsaCredentialSpecName" = $SpecName } } "containers" = @( @{ "name" = "gmsa-spec-${SpecName}-validation" "image" = $ContainerImage "command" = @("ping", "-t", "localhost") "volumeMounts" = @( @{ "name" = "gmsa-validation" "mountPath" = "/gmsa-validation" } ) } ) "volumes" = @( @{ "name" = "gmsa-validation" "configMap" = @{ "defaultMode" = 420 "name" = "gmsa-spec-${SpecName}-validation" } } ) "affinity" = @{ "nodeAffinity" = @{ "requiredDuringSchedulingIgnoredDuringExecution" = @{ "nodeSelectorTerms" = @( @{ "matchExpressions" = @( @{ "key" = "agentpool" "operator" = "In" "values" = $AksWindowsNodePoolsNames } ) } ) } } } } } } } ) $jsonManifests = $manifests | ForEach-Object { ConvertTo-Json -Depth 100 $_ } Write-Verbose "Setting up the validation DaemonSet." Install-KubernetesManifests -Manifests $jsonManifests try { Write-Verbose "Waiting until the validation DaemonSet is ready." # Wait until all the DaemonSet pods are ready. It times-out after 5 minutes of waiting. Start-ExecuteWithRetry ` -ScriptBlock ${function:Assert-ReadyDaemonSet} ` -ArgumentList @("gmsa-spec-${SpecName}-validation") ` -RetryMessage "The validation DaemonSet 'gmsa-spec-${SpecName}-validation' is not ready yet." ` -MaxRetryCount 30 -RetryInterval 10 $pods = Invoke-Kubectl "get pods -l app=gmsa-spec-${SpecName}-validation -o json" | ConvertFrom-Json if($pods.items.Length -eq 0) { Throw "There are no validation pods spawned. Probably something is wrong with the pods' scheduling or the gMSA admission webhook." } foreach($pod in $pods.items) { Write-Output "Validating Windows host '$($pod.spec.nodeName)'." Invoke-Kubectl "exec $($pod.metadata.name) -- powershell.exe /gmsa-validation/validate-spec.ps1" } } finally { Write-Verbose "Cleaning up the validation DaemonSet." Uninstall-KubernetesManifests -Manifests $jsonManifests } Write-Output "Successfully validated the gMSA credential spec '${SpecName}'." } } function Copy-WindowsHostsLogs { <# .SYNOPSIS Extract logs from each Windows host. .PARAMETER LogsDirectory The local directory where the Windows hosts' logs are copied. .PARAMETER WindowsHosts List of the Kubernetes Windows nodes to collect logs from. If this is not explicitly given, the logs are collected from all the Windows hosts. .PARAMETER ContainerImage The container image used for logs collection. A Windows server core image with PowerShell is required. #> [CmdletBinding()] Param( [Parameter(Mandatory=$true)] [String]$LogsDirectory, [ValidateScript({ foreach($name in $_) { Assert-ValidResourceName $name | Out-Null } $true })] [String[]]$WindowsHosts=@(), [String]$ContainerImage="mcr.microsoft.com/windows/servercore:ltsc2019" ) BEGIN { Assert-FoundInPath "kubectl" } PROCESS { Write-Output "Extracting the Windows hosts logs to local directory '${LogsDirectory}'." Write-Verbose "Creating local logs directory '${LogsDirectory}'." New-Item -ItemType Directory -Path $LogsDirectory -Force | Out-Null $name = "windows-logs-$(Get-UniqueId)" $manifest = @{ "kind" = "DaemonSet" "apiVersion" = "apps/v1" "metadata" = @{ "name" = $name "labels" = @{ "app" = $name } } "spec" = @{ "selector" = @{ "matchLabels" = @{ "app" = $name } } "template" = @{ "metadata" = @{ "labels" = @{ "app" = $name } } "spec" = @{ "containers" = @( @{ "name" = $name "image" = $ContainerImage "command" = @("ping", "-t", "localhost") "volumeMounts" = @( @{ "name" = "host" "mountPath" = "/host" } ) } ) "volumes" = @( @{ "name" = "host" "hostPath" = @{ "path" = "/" } } ) "nodeSelector" = @{ "kubernetes.io/os" = "windows" } } } } } $jsonManifest = ConvertTo-Json -Depth 100 $manifest Write-Verbose "Setting up the logs collection DaemonSet." Install-KubernetesManifests -Manifests $jsonManifest $tmpDirPath = "/tmp-logs-$(Get-UniqueId)" try { Write-Verbose "Creating temporary logs directory." New-Item -ItemType Directory -Path $tmpDirPath | Out-Null Write-Verbose "Waiting until the logs collection DaemonSet is ready." Start-ExecuteWithRetry ` -ScriptBlock ${function:Assert-ReadyDaemonSet} ` -ArgumentList @($name) ` -RetryMessage "The logs collection DaemonSet '${name}' is not ready yet." ` -MaxRetryCount 30 -RetryInterval 10 $logsFiles = @( "/k/kubelet.log", "/k/kubelet.err.log", "/windows/system32/winevt/Logs/Microsoft-AKSGMSAPlugin%4Admin.evtx", "/windows/system32/winevt/Logs/Microsoft-Windows-Containers-CCG%4Admin.evtx" ) $pods = Invoke-Kubectl "get pods -l app=${name} -o json" | ConvertFrom-Json foreach($pod in $pods.items) { $podName = $pod.metadata.name $nodeName = $pod.spec.nodeName if($WindowsHosts.Length -gt 0 -and $nodeName -notin $WindowsHosts) { continue } Write-Verbose "Creating Windows host logs directory '${LogsDirectory}/${nodeName}'." New-Item -ItemType Directory -Path "${LogsDirectory}/${nodeName}" -Force | Out-Null Write-Output "Extracting Windows host '${nodeName}' logs." foreach($logFile in $logsFiles) { Write-Verbose "Extracting '${logFile}'." $fileName = Split-Path $logFile -Leaf Invoke-Kubectl "exec ${podName} -- powershell.exe cp /host/${logFile} /" Invoke-Kubectl "cp ${podName}:${fileName} ${tmpDirPath}/${fileName}" Move-Item -Force -Path "${tmpDirPath}/${fileName}" -Destination "${LogsDirectory}/${nodeName}" } } } finally { Write-Verbose "Cleaning up the temporary logs directory." Remove-Item -Recurse -Force -Path $tmpDirPath -ErrorAction SilentlyContinue Write-Verbose "Cleaning up the logs collection DaemonSet." Uninstall-KubernetesManifests -Manifests $jsonManifest } Write-Output "Finished copying logs from all the Windows hosts." } } ### Private functions below function Assert-FoundInPath { <# .SYNOPSIS Assert that a binary is found in PATH. Useful to verify that the requirements are installed. .PARAMETER Name The binary name(s). #> Param( [Parameter(Mandatory=$true)] [String[]]$Name ) foreach($binName in $Name) { $bin = Get-Command -Name $binName -ErrorAction SilentlyContinue if(!$bin) { Throw "The binary '${binName}' was not found in PATH. Please run 'Install-ToolingRequirements' first." } } } function Get-ADRemoteParams { <# .SYNOPSIS Get the parameters used for remote execution by the functions from the ActiveDirectory RSAT PowerShell module. .PARAMETER DomainControllerAddress The AD domain controller address. .PARAMETER DomainUser The domain user. If this is set, the parameter 'DomainUserPassword' is mandatory. .PARAMETER DomainUserPassword The domain user password. If this is set, the parameter 'DomainUser' is mandatory. #> Param( [String]$DomainControllerAddress, [ValidateScript({ if($_.Split('\').Count -ne 2) { Throw "The domain user must be given as 'DOMAIN\USERNAME'." } $true })] [String]$DomainUser, [SecureString]$DomainUserPassword ) $params = @{} if($DomainControllerAddress) { $params["Server"] = $DomainControllerAddress } if($DomainUser -and !$DomainUserPassword) { Throw "The domain user parameter is specified, but the password parameter is missing." } if(!$DomainUser -and $DomainUserPassword) { Throw "The domain user parameter is missing, but the password parameter is specified." } if($DomainUser -and $DomainUserPassword) { $params["Credential"] = New-Object PSCredential($DomainUser, $DomainUserPassword) } return $params } function Get-GMSACredentialSpecManifest { <# .SYNOPSIS Get the gMSA credential spec Kubernetes manifest. .PARAMETER Name The spec name. .PARAMETER GMSAName The gMSA name. .PARAMETER ManagedIdentityResourceGroupName The managed identity resource group name. .PARAMETER ManagedIdentityName The managed identity name. .PARAMETER VaultName The Azure key vault name. .PARAMETER VaultGMSASecretName The gMSA Azure key vault secret name. .PARAMETER DomainControllerAddress The AD domain controller address used for remote command execution. This parameter is optional. .PARAMETER DomainUser The domain user used for remote command execution. This parameter is optional. .PARAMETER DomainUserPassword The domain user password used for remote command execution. This parameter is optional. #> Param( [Parameter(Mandatory=$true)] [String]$Name, [Parameter(Mandatory=$true)] [String]$GMSAName, [Parameter(Mandatory=$true)] [String]$ManagedIdentityResourceGroupName, [Parameter(Mandatory=$true)] [String]$ManagedIdentityName, [Parameter(Mandatory=$true)] [String]$VaultName, [Parameter(Mandatory=$true)] [String]$VaultGMSASecretName, [String]$DomainControllerAddress, [ValidateScript({ if($_.Split('\').Count -ne 2) { Throw "The domain user must be given as 'DOMAIN\USERNAME'." } $true })] [String]$DomainUser, [SecureString]$DomainUserPassword ) Write-Verbose "Getting the user-assigned managed identity info." $mi = Get-AzUserAssignedIdentity -ResourceGroupName $ManagedIdentityResourceGroupName -Name $ManagedIdentityName $remoteParams = Get-ADRemoteParams $DomainControllerAddress $DomainUser $DomainUserPassword Write-Verbose "Getting the AD domain info." $domain = Get-ADDomain @remoteParams Write-Verbose "Getting the AKV gMSA secret info." $secret = Get-AzKeyVaultSecret -VaultName $VaultName -Name $VaultGMSASecretName if(!$secret) { Throw "The secret '${VaultGMSASecretName}' was not found in the Azure key vault '${VaultName}'." } $secretUri = ([uri]$secret.Id).AbsoluteUri -replace "/$($secret.Version)$", '' return @{ "apiVersion" = "windows.k8s.io/v1alpha1" "kind" = "GMSACredentialSpec" "metadata" = @{ "name" = $Name } "credspec" = @{ "CmsPlugins" = @( "ActiveDirectory" ) "DomainJoinConfig" = @{ "Sid" = $domain.DomainSID.Value "MachineAccountName" = $GMSAName "Guid" = $domain.ObjectGUID.Guid "DnsTreeName" = $domain.Forest "DnsName" = $domain.DNSRoot "NetBiosName" = $domain.NetBIOSName } "ActiveDirectoryConfig" = @{ "GroupManagedServiceAccounts" = @( @{ "Name" = $GMSAName "Scope" = $domain.DNSRoot }, @{ "Name" = $GMSAName "Scope" = $domain.NetBIOSName } ) "HostAccountConfig" = @{ "PortableCcgVersion" = "1" "PluginGUID" = "{CCC2A336-D7F3-4818-A213-272B7924213E}" "PluginInput" = "ObjectId=$($mi.PrincipalId);SecretUri=${secretUri}" } } } } } function Get-GMSACredentialSpecClusterRoleManifest { <# .SYNOPSIS Get the RBAC ClusterRole Kubernetes manifest for the gMSA credential spec. .PARAMETER SpecName The spec name. #> Param( [Parameter(Mandatory=$true)] [String]$SpecName ) return @{ "apiVersion" = "rbac.authorization.k8s.io/v1" "kind" = "ClusterRole" "metadata" = @{ "name" = "${SpecName}-role" } "rules" = @( @{ "apiGroups" = @("windows.k8s.io") "resources" = @("gmsacredentialspecs") "verbs" = @("use") "resourceNames" = @($SpecName) } ) } } function Get-GMSACredentialSpecRoleBindingManifest { <# .SYNOPSIS Get the RBAC RoleBinding Kubernetes manifest for the gMSA credential spec. .PARAMETER SpecName The spec name. #> Param( [Parameter(Mandatory=$true)] [String]$SpecName ) return @{ "apiVersion" = "rbac.authorization.k8s.io/v1" "kind" = "RoleBinding" "metadata" = @{ "name" = "allow-default-svc-account-read-on-${SpecName}" "namespace" = "default" } "subjects" = @( @{ "kind" = "ServiceAccount" "name" = "default" "namespace" = "default" } ) "roleRef" = @{ "kind" = "ClusterRole" "name" = "${SpecName}-role" "apiGroup" = "rbac.authorization.k8s.io" } } } function Get-ADDCValidationPSScript { <# .SYNOPSIS Get the PowerShell script used to validate the connectivity to the Active Directory Domain Controller (AD DC) machine. .PARAMETER DomainDnsServer The AD DC DNS server. .PARAMETER RootDomainName The AD DC root domain name. #> Param( [Parameter(Mandatory=$true)] [String]$DomainDnsServer, [Parameter(Mandatory=$true)] [String]$RootDomainName ) return @" `$ErrorActionPreference = "Stop" # Checks that the root domain name can be resolved. Resolve-DnsName -Name $RootDomainName -Server $DomainDnsServer | Format-List * # Queries the Domain Name System (DNS) server for a list of domain controllers # and their corresponding IP addresses. nltest /dsgetdc:${RootDomainName} /server:${DomainDnsServer} if(`$LASTEXITCODE) { Throw "Validation command 'nltest /dsgetdc:${RootDomainName} /server:${DomainDnsServer}' had exit code `${LASTEXITCODE}." } "@ } function Get-GMSACredSpecValidationPSScript { <# .SYNOPSIS Get the PowerShell script used to validate the gMSA credential spec. .PARAMETER DomainName The Active Directory domain name. #> Param( [Parameter(Mandatory=$true)] [String]$DomainName ) return @" `$ErrorActionPreference = "Stop" # Returns the name of the parent domain of the server. nltest /parentdomain if(`$LASTEXITCODE) { Throw "Validation command 'nltest /parentdomain' had exit code `${LASTEXITCODE}." } # Checks that the configured DNS server can resolved the domain name. Resolve-DnsName -Name $DomainName # Queries the Domain Name System (DNS) server for a list of domain controllers and # their corresponding IP addresses. nltest /dsgetdc:${DomainName} if(`$LASTEXITCODE) { Throw "Validation command 'nltest /dsgetdc:${DomainName}' had exit code `${LASTEXITCODE}." } # Checks the status of the secure channel that the NetLogon service established. `$result = nltest /sc_verify:${DomainName} `$result if(`$LASTEXITCODE) { Throw "Validation command 'nltest /sc_verify:${DomainName}' had exit code `${LASTEXITCODE}." } `$successDCConnMsg = 'Trusted DC Connection Status Status = 0 0x0 NERR_Success' `$successVerificationStatusMsg = 'Trust Verification Status = 0 0x0 NERR_Success' if(!`$result.Contains(`$successDCConnMsg) -or !`$result.Contains(`$successVerificationStatusMsg)) { Throw "Validation command 'nltest /sc_verify:${DomainName}' reported errors." } # Validate if we can get a kerberos ticket granting ticket (TGT), and verify the # kerberos protocol for outbound and inbound auth. klist get krbtgt if(`$LASTEXITCODE) { Throw "Validation command 'klist get krbtgt' had exit code `${LASTEXITCODE}." } "@ } function Assert-ReadyDaemonSet { <# .SYNOPSIS Assert that the Kubernetes DaemonSet is ready. This function will assert that the ready pods count matches the desired pods count. .PARAMETER Name The DaemonSet name. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) $daemonSet = Invoke-Kubectl "get daemonset ${Name} -o json" | ConvertFrom-Json $readyCount = $daemonSet.status.numberReady $desiredCount = $daemonSet.status.desiredNumberScheduled if($readyCount -ne $desiredCount) { Throw "The DaemonSet '${Name}' is not ready yet. Ready pods count: ${readyCount}. Expected: ${desiredCount}." } } function Assert-ValidGMSACredentialSpecName { <# .SYNOPSIS Assert that the gMSA credential spec name is valid. .PARAMETER Name The gMSA credential spec name. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) if(!($_ -cmatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$")) { Throw "The gMSA credential spec name is invalid. The name must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." } return $true } function Assert-ValidAzureLocation { <# .SYNOPSIS Assert that the given Azure location is valid. .PARAMETER Location The Azure location. #> Param( [Parameter(Mandatory=$true)] [String]$Location ) foreach($loc in (Get-AzLocation)) { if($loc.Location -eq $Location) { return $true } } Throw "The Azure location '${Location}' is not valid." } function Assert-ValidAlphanumeric { <# .SYNOPSIS Assert that the given string is alphanumeric. .PARAMETER Name The string to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) if(!($Name -cmatch "^[a-zA-Z0-9]+$")) { Throw "Invalid input, '${Name}' is not alphanumeric." } return $true } function Assert-ValidResourceName { <# .SYNOPSIS This function will assert that the given string is a valid resource name. The allowed characters are letters, numbers, and the dash (-) character. The first character must be a letter, and the end character must be a letter or a number. .PARAMETER Name The string to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) if(!($Name -cmatch "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$")) { Throw "Invalid input given. The allowed characters are letters, numbers, and the dash (-) character. The first character must be a letter, and the end character must be a letter or a number." } return $true } function Assert-ValidAzureResourceGroupName { <# .SYNOPSIS Assert that the given Azure resource group name is valid. .PARAMETER Name The Azure resource group name. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) if(!($Name -match "^[-\w\._\(\)]*[-\w_\(\)]+$")) { Throw "Resource group names only allow alphanumeric characters, periods, underscores, hyphens and parenthesis and cannot end in a period." } if($Name.Length -gt 90) { Throw "Resource group names must be less than 90 characters." } return $true } function Assert-ValidDnsName { <# .SYNOPSIS This function will assert that the given string is a valid DNS name. .PARAMETER Name The string to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) if(!($Name -cmatch "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?(\.[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?)*$")) { Throw "Invalid input given, '${Name}' is not a valid DNS name." } return $true } function Assert-ValidDomainName { <# .SYNOPSIS This function will assert that the given string is a valid domain name (NetBIOS name or FQDN). .PARAMETER Name The string to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) $validNetBiosName = $true try { Assert-ValidAlphanumeric $Name } catch { $validNetBiosName = $false } $validFQDN = $true try { Assert-ValidDnsName $Name } catch { $validFQDN = $false } if(!$validNetBiosName -and !$validFQDN) { Throw "Invalid domain name. The domain name must be a valid NetBIOS name or a valid FQDN." } } function Assert-ValidDomainUserName { <# .SYNOPSIS This function will assert that the given string is a valid domain user name with the format 'DOMAIN\USERNAME'. .PARAMETER Name The string to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) $split = $Name.Split('\') if($split.Count -ne 2) { Throw "The domain user must be given as 'DOMAIN\USERNAME'." } # Validate the domain name (NetBIOS name or FQDN) and the user name return ((Assert-ValidDomainName $split[0]) -and (Assert-ValidResourceName $split[1])) } function Assert-ValidFQDNDomainUserName { <# .SYNOPSIS This function will assert that the given string is a valid domain user name with the format 'FQDN\USERNAME'. .PARAMETER Name The string to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Name ) $split = $Name.Split('\') if($split.Count -ne 2) { Throw "The domain user must be given as 'FQDN\USERNAME'." } return ((Assert-ValidDnsName $split[0]) -and (Assert-ValidResourceName $split[1])) } function Assert-ValidAksNodePoolNames { <# .SYNOPSIS This function will assert that the given AKS node pools' names are valid. .PARAMETER Names The node pools' names to validate. #> Param( [Parameter(Mandatory=$true)] [String]$Names ) foreach($name in $Names) { if($name.Length -gt 6) { Throw "The node pool name '$name' is too long. The maximum length is 6 characters." } Assert-ValidAlphanumeric $name | Out-Null } return $true } # SIG # Begin signature block # MIInvQYJKoZIhvcNAQcCoIInrjCCJ6oCAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCA0UsHQdOHhiUdV # 920RBLPvo2g5f8ywffcJjNNlrlGy9aCCDYUwggYDMIID66ADAgECAhMzAAACU+OD # 3pbexW7MAAAAAAJTMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjEwOTAyMTgzMzAwWhcNMjIwOTAxMTgzMzAwWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDLhxHwq3OhH+4J+SX4qS/VQG8HybccH7tnG+BUqrXubfGuDFYPZ29uCuHfQlO1 # lygLgMpJ4Geh6/6poQ5VkDKfVssn6aA1PCzIh8iOPMQ9Mju3sLF9Sn+Pzuaie4BN # rp0MuZLDEXgVYx2WNjmzqcxC7dY9SC3znOh5qUy2vnmWygC7b9kj0d3JrGtjc5q5 # 0WfV3WLXAQHkeRROsJFBZfXFGoSvRljFFUAjU/zdhP92P+1JiRRRikVy/sqIhMDY # +7tVdzlE2fwnKOv9LShgKeyEevgMl0B1Fq7E2YeBZKF6KlhmYi9CE1350cnTUoU4 # YpQSnZo0YAnaenREDLfFGKTdAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUlZpLWIccXoxessA/DRbe26glhEMw # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzQ2NzU5ODAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # AKVY+yKcJVVxf9W2vNkL5ufjOpqcvVOOOdVyjy1dmsO4O8khWhqrecdVZp09adOZ # 8kcMtQ0U+oKx484Jg11cc4Ck0FyOBnp+YIFbOxYCqzaqMcaRAgy48n1tbz/EFYiF # zJmMiGnlgWFCStONPvQOBD2y/Ej3qBRnGy9EZS1EDlRN/8l5Rs3HX2lZhd9WuukR # bUk83U99TPJyo12cU0Mb3n1HJv/JZpwSyqb3O0o4HExVJSkwN1m42fSVIVtXVVSa # YZiVpv32GoD/dyAS/gyplfR6FI3RnCOomzlycSqoz0zBCPFiCMhVhQ6qn+J0GhgR # BJvGKizw+5lTfnBFoqKZJDROz+uGDl9tw6JvnVqAZKGrWv/CsYaegaPePFrAVSxA # yUwOFTkAqtNC8uAee+rv2V5xLw8FfpKJ5yKiMKnCKrIaFQDr5AZ7f2ejGGDf+8Tz # OiK1AgBvOW3iTEEa/at8Z4+s1CmnEAkAi0cLjB72CJedU1LAswdOCWM2MDIZVo9j # 0T74OkJLTjPd3WNEyw0rBXTyhlbYQsYt7ElT2l2TTlF5EmpVixGtj4ChNjWoKr9y # TAqtadd2Ym5FNB792GzwNwa631BPCgBJmcRpFKXt0VEQq7UXVNYBiBRd+x4yvjqq # 5aF7XC5nXCgjbCk7IXwmOphNuNDNiRq83Ejjnc7mxrJGMIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGY4wghmKAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAAJT44Pelt7FbswAAAAA # AlMwDQYJYIZIAWUDBAIBBQCggbAwGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIN3l # bbO94h7LwYvFhlXCDkYYm8qfAz3UCh7rRPSGuqmLMEQGCisGAQQBgjcCAQwxNjA0 # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEcgBpodHRwczovL3d3dy5taWNyb3NvZnQu # Y29tIDANBgkqhkiG9w0BAQEFAASCAQArCX+BC3YnO3sOmBergZcTXebRBrr3et4C # KPyZz2Z6ilzMzlQawjFRk6lSYBbFK4pfoxyJ/05M3i5bkkx0hX0TQpFnbidd3MAE # brVN2W8YkzCcDhS352Hd+20xuoXqsmprkysRXrlk0VLUKCrPSiluDBDBNUBDr/94 # 3xZQ8SIv/B4SdBxr4jAv5NpZX4GDrHQC9W3XfrbHotAyLNEXvFfajVCJi6be8Y0d # nSsadSaWO7ZkT8EpOwsculGl2mW8MpOp2OO81cyTHMPf1JHQap0yzVuIu4Tj9PEq # iA3/Y+4/ZHlGl4Qk4rZe2Txr+MdP9Sgc4GCdKQvIGV+w2MIaczJXoYIXFjCCFxIG # CisGAQQBgjcDAwExghcCMIIW/gYJKoZIhvcNAQcCoIIW7zCCFusCAQMxDzANBglg # hkgBZQMEAgEFADCCAVkGCyqGSIb3DQEJEAEEoIIBSASCAUQwggFAAgEBBgorBgEE # AYRZCgMBMDEwDQYJYIZIAWUDBAIBBQAEIHcmPlRAC0rlUFBGCRMukGj3GyJdOu23 # 0dLxtr8gkdASAgZibEQuFA8YEzIwMjIwNTEzMTU0NTEwLjcwNVowBIACAfSggdik # gdUwgdIxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQH # EwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNV # BAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJhdGlvbnMgTGltaXRlZDEmMCQGA1UE # CxMdVGhhbGVzIFRTUyBFU046MkFENC00QjkyLUZBMDExJTAjBgNVBAMTHE1pY3Jv # c29mdCBUaW1lLVN0YW1wIFNlcnZpY2WgghFlMIIHFDCCBPygAwIBAgITMwAAAYZ4 # 5RmJ+CRLzAABAAABhjANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEG # A1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWlj # cm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFt # cCBQQ0EgMjAxMDAeFw0yMTEwMjgxOTI3MzlaFw0yMzAxMjYxOTI3MzlaMIHSMQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNy # b3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxl # cyBUU1MgRVNOOjJBRDQtNEI5Mi1GQTAxMSUwIwYDVQQDExxNaWNyb3NvZnQgVGlt # ZS1TdGFtcCBTZXJ2aWNlMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA # wI3G2Wpv6B4IjAfrgfJpndPOPYO1Yd8+vlfoIxMW3gdCDT+zIbafg14pOu0t0ekU # Qx60p7PadH4OjnqNIE1q6ldH9ntj1gIdl4Hq4rdEHTZ6JFdE24DSbVoqqR+R4Iw4 # w3GPbfc2Q3kfyyFyj+DOhmCWw/FZiTVTlT4bdejyAW6r/Jn4fr3xLjbvhITatr36 # VyyzgQ0Y4Wr73H3gUcLjYu0qiHutDDb6+p+yDBGmKFznOW8wVt7D+u2VEJoE6JlK # 0EpVLZusdSzhecuUwJXxb2uygAZXlsa/fHlwW9YnlBqMHJ+im9HuK5X4x8/5B5dk # uIoX5lWGjFMbD2A6Lu/PmUB4hK0CF5G1YaUtBrME73DAKkypk7SEm3BlJXwY/GrV # oXWYUGEHyfrkLkws0RoEMpoIEgebZNKqjRynRJgR4fPCKrEhwEiTTAc4DXGci4HH # Om64EQ1g/SDHMFqIKVSxoUbkGbdKNKHhmahuIrAy4we9s7rZJskveZYZiDmtAtBt # /gQojxbZ1vO9C11SthkrmkkTMLQf9cDzlVEBeu6KmHX2Sze6ggne3I4cy/5IULnH # Z3rM4ZpJc0s2KpGLHaVrEQy4x/mAn4yaYfgeH3MEAWkVjy/qTDh6cDCF/gyz3TaQ # DtvFnAK70LqtbEvBPdBpeCG/hk9l0laYzwiyyGY/HqMCAwEAAaOCATYwggEyMB0G # A1UdDgQWBBQZtqNFA+9mdEu/h33UhHMN6whcLjAfBgNVHSMEGDAWgBSfpxVdAF5i # XYP05dJlpxtTNRnpcjBfBgNVHR8EWDBWMFSgUqBQhk5odHRwOi8vd3d3Lm1pY3Jv # c29mdC5jb20vcGtpb3BzL2NybC9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENB # JTIwMjAxMCgxKS5jcmwwbAYIKwYBBQUHAQEEYDBeMFwGCCsGAQUFBzAChlBodHRw # Oi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY3Jvc29mdCUyMFRp # bWUtU3RhbXAlMjBQQ0ElMjAyMDEwKDEpLmNydDAMBgNVHRMBAf8EAjAAMBMGA1Ud # JQQMMAoGCCsGAQUFBwMIMA0GCSqGSIb3DQEBCwUAA4ICAQDD7mehJY3fTHKC4hj+ # wBWB8544uaJiMMIHnhK9ONTM7VraTYzx0U/TcLJ6gxw1tRzM5uu8kswJNlHNp7Re # dsAiwviVQZV9AL8IbZRLJTwNehCwk+BVcY2gh3ZGZmx8uatPZrRueyhhTTD2PvFV # Lrfwh2liDG/dEPNIHTKj79DlEcPIWoOCUp7p0ORMwQ95kVaibpX89pvjhPl2Fm0C # BO3pXXJg0bydpQ5dDDTv/qb0+WYF/vNVEU/MoMEQqlUWWuXECTqx6TayJuLJ6uU7 # K5QyTkQ/l24IhGjDzf5AEZOrINYzkWVyNfUOpIxnKsWTBN2ijpZ/Tun5qrmo9vNI # DT0lobgnulae17NaEO9oiEJJH1tQ353dhuRi+A00PR781iYlzF5JU1DrEfEyNx8C # WgERi90LKsYghZBCDjQ3DiJjfUZLqONeHrJfcmhz5/bfm8+aAaUPpZFeP0g0Iond # 6XNk4YiYbWPFoofc0LwcqSALtuIAyz6f3d+UaZZsp41U4hCIoGj6hoDIuU839bo/ # mZ/AgESwGxIXs0gZU6A+2qIUe60QdA969wWSzucKOisng9HCSZLF1dqc3QUawr0C # 0U41784Ko9vckAG3akwYuVGcs6hM/SqEhoe9jHwe4Xp81CrTB1l9+EIdukCbP0ky # zx0WZzteeiDN5rdiiQR9mBJuljCCB3EwggVZoAMCAQICEzMAAAAVxedrngKbSZkA # AAAAABUwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpX # YXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQg # Q29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRl # IEF1dGhvcml0eSAyMDEwMB4XDTIxMDkzMDE4MjIyNVoXDTMwMDkzMDE4MzIyNVow # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwggIiMA0GCSqGSIb3DQEBAQUA # A4ICDwAwggIKAoICAQDk4aZM57RyIQt5osvXJHm9DtWC0/3unAcH0qlsTnXIyjVX # 9gF/bErg4r25PhdgM/9cT8dm95VTcVrifkpa/rg2Z4VGIwy1jRPPdzLAEBjoYH1q # UoNEt6aORmsHFPPFdvWGUNzBRMhxXFExN6AKOG6N7dcP2CZTfDlhAnrEqv1yaa8d # q6z2Nr41JmTamDu6GnszrYBbfowQHJ1S/rboYiXcag/PXfT+jlPP1uyFVk3v3byN # pOORj7I5LFGc6XBpDco2LXCOMcg1KL3jtIckw+DJj361VI/c+gVVmG1oO5pGve2k # rnopN6zL64NF50ZuyjLVwIYwXE8s4mKyzbnijYjklqwBSru+cakXW2dg3viSkR4d # Pf0gz3N9QZpGdc3EXzTdEonW/aUgfX782Z5F37ZyL9t9X4C626p+Nuw2TPYrbqgS # Uei/BQOj0XOmTTd0lBw0gg/wEPK3Rxjtp+iZfD9M269ewvPV2HM9Q07BMzlMjgK8 # QmguEOqEUUbi0b1qGFphAXPKZ6Je1yh2AuIzGHLXpyDwwvoSCtdjbwzJNmSLW6Cm # gyFdXzB0kZSU2LlQ+QuJYfM2BjUYhEfb3BvR/bLUHMVr9lxSUV0S2yW6r1AFemzF # ER1y7435UsSFF5PAPBXbGjfHCBUYP3irRbb1Hode2o+eFnJpxq57t7c+auIurQID # AQABo4IB3TCCAdkwEgYJKwYBBAGCNxUBBAUCAwEAATAjBgkrBgEEAYI3FQIEFgQU # KqdS/mTEmr6CkTxGNSnPEP8vBO4wHQYDVR0OBBYEFJ+nFV0AXmJdg/Tl0mWnG1M1 # GelyMFwGA1UdIARVMFMwUQYMKwYBBAGCN0yDfQEBMEEwPwYIKwYBBQUHAgEWM2h0 # dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvRG9jcy9SZXBvc2l0b3J5Lmh0 # bTATBgNVHSUEDDAKBggrBgEFBQcDCDAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMA # QTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbL # j+iiXGJo0T2UkFvXzpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1p # Y3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0w # Ni0yMy5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3 # Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIz # LmNydDANBgkqhkiG9w0BAQsFAAOCAgEAnVV9/Cqt4SwfZwExJFvhnnJL/Klv6lwU # tj5OR2R4sQaTlz0xM7U518JxNj/aZGx80HU5bbsPMeTCj/ts0aGUGCLu6WZnOlNN # 3Zi6th542DYunKmCVgADsAW+iehp4LoJ7nvfam++Kctu2D9IdQHZGN5tggz1bSNU # 5HhTdSRXud2f8449xvNo32X2pFaq95W2KFUn0CS9QKC/GbYSEhFdPSfgQJY4rPf5 # KYnDvBewVIVCs/wMnosZiefwC2qBwoEZQhlSdYo2wh3DYXMuLGt7bj8sCXgU6ZGy # qVvfSaN0DLzskYDSPeZKPmY7T7uG+jIa2Zb0j/aRAfbOxnT99kxybxCrdTDFNLB6 # 2FD+CljdQDzHVG2dY3RILLFORy3BFARxv2T5JL5zbcqOCb2zAVdJVGTZc9d/HltE # AY5aGZFrDZ+kKNxnGSgkujhLmm77IVRrakURR6nxt67I6IleT53S0Ex2tVdUCbFp # AUR+fKFhbHP+CrvsQWY9af3LwUFJfn6Tvsv4O+S3Fb+0zj6lMVGEvL8CwYKiexcd # FYmNcP7ntdAoGokLjzbaukz5m/8K6TT4JDVnK+ANuOaMmdbhIurwJ0I9JZTmdHRb # atGePu1+oDEzfbzL6Xu/OHBE0ZDxyKs6ijoIYn/ZcGNTTY3ugm2lBRDBcQZqELQd # VTNYs6FwZvKhggLUMIICPQIBATCCAQChgdikgdUwgdIxCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5k # IE9wZXJhdGlvbnMgTGltaXRlZDEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046MkFE # NC00QjkyLUZBMDExJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZp # Y2WiIwoBATAHBgUrDgMCGgMVAAGu2DRzWkKljmXySX1korHL4fMnoIGDMIGApH4w # fDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMd # TWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwDQYJKoZIhvcNAQEFBQACBQDm # KI4VMCIYDzIwMjIwNTEzMTU1ODEzWhgPMjAyMjA1MTQxNTU4MTNaMHQwOgYKKwYB # BAGEWQoEATEsMCowCgIFAOYojhUCAQAwBwIBAAICCNAwBwIBAAICEd0wCgIFAOYp # 35UCAQAwNgYKKwYBBAGEWQoEAjEoMCYwDAYKKwYBBAGEWQoDAqAKMAgCAQACAweh # IKEKMAgCAQACAwGGoDANBgkqhkiG9w0BAQUFAAOBgQAPvW96ABBUz7l5dqxX9Bdo # sRQk3d7xUkLAnWzGyrJeXIMEcbsv5K8bc+T3VwcVbBASECye6jbRglndi7yIhmAD # lvZ+7UF91Na0HSLiCFR+1Zyu4F/Z7WxPhEU+tIWo8CkH6rc6aXmCJPgKXoE7xsJ1 # AoD90YrYBy4TTiav0VdLpDGCBA0wggQJAgEBMIGTMHwxCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0 # YW1wIFBDQSAyMDEwAhMzAAABhnjlGYn4JEvMAAEAAAGGMA0GCWCGSAFlAwQCAQUA # oIIBSjAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwLwYJKoZIhvcNAQkEMSIE # ILrCu48zA2H9BngkzTtZiuFEvswzDAOpgOoU+aexoQJuMIH6BgsqhkiG9w0BCRAC # LzGB6jCB5zCB5DCBvQQgGpmI4LIsCFTGiYyfRAR7m7Fa2guxVNIw17mcAiq8Qn4w # gZgwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4G # A1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYw # JAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAYZ45RmJ # +CRLzAABAAABhjAiBCDlCUkRGaGnZa0y/p3J5lHdF8OZFrFe80WSgjgIZU7kSzAN # BgkqhkiG9w0BAQsFAASCAgAQm4YVsH/CPqKC6REqyp+QqoVmLZ5ziP/8snpSRlvH # oy3atFdGj1DQv2IFyBSvUQgrxYGgGkjAb9mvWzdh0omPpeLqcVLwZVmmW5KXKcNZ # YoFHrObRKLFnvfYKijZddMvg7LXnFgiFOpi+PRA703JU+TfF6GwwmNbFoUgg5YZO # KOMVtGCdT7qG81jMuo7YaNkRlM+W4eVD6JU66wRp9Vp6iSEA0P3PoMqa6LtMRXT5 # QG9dBevDohMTF150Xdo1ZPkj1uyq3LM2ln0h14rsC80kkNTIHOPaDgwA8slrWfzg # UH+oxCqZkhSPFCSvpl0sN70agAUCxPiUPt05oRHwOb77toUnEv5M+UDJmNVR+Q7H # Q+2G0QVnbImYXgg9PxKjZdjZPVJSVW5g6zx/Nr6N5fWEViEc+H0YgyTzIno5cQAE # /FSa+lnG2sOyft3gKZ8g/sfIK5X1Tc034AZKlPuKORY13ZLmubWcP6/+jjdb5afi # HY+bGUj6/FUHqcSsGlzjKjgbTntVSeKcBA3U4a/f/2gU1/R8xl6fbPEHLFw+UCbw # pTUtE7WWRJ9MQ+mleBbtPyFXLg3WaGvYPJWpq6z4Q3anlEz/WRB3IvgiuJ9pf7Pz # pHPzyVvKXsAQHGyIOd9Go6fNeZAmMHT98hZfxx5wJjwvZUW4vuqFCnj4TMUfNFos # cw== # SIG # End signature block |