Common.psm1
######################################################################################### # # Copyright (c) Microsoft Corporation. All rights reserved. # # Common/Shared Functions, Types, and Variables # ######################################################################################### #requires -runasadministrator #region Enums Add-Type -TypeDefinition @" using System.Management.Automation; public enum InstallState { NotInstalled, Installing, InstallFailed, Updating, UpdateFailed, Uninstalling, UninstallFailed, Installed }; public enum DeploymentType { None, SingleNode, MultiNode }; public enum LoadBalancerType { unstacked_haproxy, stacked_kube_vip }; public enum OsType { Linux, Windows }; public enum OsSku { CBLMariner, Windows2019, Windows2022 }; public enum VmSize { Default, Standard_A2_v2, Standard_A4_v2, Standard_D2s_v3, Standard_D4s_v3, Standard_D8s_v3, Standard_D16s_v3, Standard_D32s_v3, Standard_DS2_v2, Standard_DS3_v2, Standard_DS4_v2, Standard_DS5_v2, Standard_DS13_v2, Standard_K8S_v1, Standard_K8S2_v1, Standard_K8S3_v1, Standard_NK6, Standard_NK12, }; public enum LoadBalancerSku { HAProxy, None, KubeVIP, SDNLoadBalancer }; public enum OfflineDownloadMode { full, minimum }; public class Command { public string cmdletName; public string processId; public string computerName; public System.DateTime startTime; public Command ( string cmdletName, string processId, string computerName, System.DateTime startTime ) { this.cmdletName = cmdletName; this.processId = processId; this.computerName = computerName; this.startTime = startTime; } } public class LoadBalancerSettings { public string Name; public LoadBalancerSku LoadBalancerSku; public LoadBalancerSku ServicesLoadBalancerSku; public VmSize VmSize; public int LoadBalancerCount; public LoadBalancerSettings ( string Name, LoadBalancerSku LoadBalancerSku, LoadBalancerSku ServicesLoadBalancerSku, VmSize VmSize, int LoadBalancerCount ) { this.Name = Name; this.LoadBalancerSku = LoadBalancerSku; this.ServicesLoadBalancerSku = ServicesLoadBalancerSku; this.VmSize = VmSize; this.LoadBalancerCount = LoadBalancerCount; } } public class VirtualNetwork { public string Name; public string VswitchName; public string IpAddressPrefix; public string Gateway; public string[] DnsServers; public string MacPoolName; public int Vlanid; public string VipPoolStart; public string VipPoolEnd; public string K8snodeIPPoolStart; public string K8snodeIPPoolEnd; public VirtualNetwork ( string Name, string VswitchName, string IpAddressPrefix, string Gateway, string[] DnsServers, string MacPoolName, int Vlanid, string VipPoolStart, string VipPoolEnd, string K8snodeIPPoolStart, string K8snodeIPPoolEnd ) { this.Name = Name; this.VswitchName = VswitchName; this.IpAddressPrefix = IpAddressPrefix; this.Gateway = Gateway; this.DnsServers = DnsServers; this.MacPoolName = MacPoolName; this.Vlanid = Vlanid; this.VipPoolStart = VipPoolStart; this.VipPoolEnd = VipPoolEnd; this.K8snodeIPPoolStart = K8snodeIPPoolStart; this.K8snodeIPPoolEnd = K8snodeIPPoolEnd; } public override string ToString() { return string.Format("{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}\n{7}\n{8}\n{9}", this.Name, this.IpAddressPrefix, this.Gateway, this.DnsServers != null ? string.Join(",", this.DnsServers) : "", this.MacPoolName, this.Vlanid, this.VipPoolStart, this.VipPoolEnd, this.K8snodeIPPoolStart, this.K8snodeIPPoolEnd); } } public class SecondaryNetworkPlugin { public string Name; public VirtualNetwork Network; public string CniType; public bool EnableDpdk; public bool EnableSriov; public SecondaryNetworkPlugin( string name, VirtualNetwork Network, string cniType, bool enableDpdk, bool enableSriov ) { this.Name = name.ToLower(); this.Network = Network; this.CniType = cniType.ToLower(); this.EnableDpdk = enableDpdk; this.EnableSriov = enableSriov; } } public class LinuxOsConfig { public string Name; public int hugePages2M; public int hugePages1G; public LinuxOsConfig( string name, int hugePages2M, int hugePages1G ) { this.Name = name.ToLower(); this.hugePages2M = hugePages2M; this.hugePages1G = hugePages1G; } } public class ProxySettings { public string Name; public string HTTP; public string HTTPS; public string NoProxy; public string CertFile; public PSCredential Credential; public ProxySettings ( PSCredential Credential, string Name = "", string HTTP = "", string HTTPS = "", string NoProxy = "", string CertFile = "" ) { this.Name = Name; this.HTTP = HTTP; this.HTTPS = HTTPS; this.NoProxy = NoProxy; this.CertFile = CertFile; this.Credential = Credential; } public override string ToString() { return string.Format("{0}\n{1}\n{2}\n{3}\n{4}\n{5}", this.Name, this.HTTP, this.HTTPS, this.NoProxy, this.CertFile, this.Credential.ToString()); } } public class ContainerRegistry { public string Server; public PSCredential Credential; public ContainerRegistry ( PSCredential Credential, string Server = "" ) { this.Server = Server; this.Credential = Credential; } public override string ToString() { return string.Format("{0}\n{1}", this.Server, this.Credential.ToString()); } } public class VipPoolSettings { public string Name; public string VipPoolStart; public string VipPoolEnd; public VipPoolSettings ( string Name, string VipPoolStart, string VipPoolEnd ) { this.Name = Name; this.VipPoolStart = VipPoolStart; this.VipPoolEnd = VipPoolEnd; } public override string ToString() { return string.Format("{0}\n{1}\n{2}", this.Name, this.VipPoolStart, this.VipPoolEnd); } } public class SSHConfiguration { public string Name; public string SSHPublicKey; public string SSHPrivateKey; public string CIDR; public string[] IPAddresses; public bool RestrictSSHCommands; public SSHConfiguration ( string Name, string SSHPublicKey, string SSHPrivateKey, string CIDR, string[] IPAddresses, bool RestrictSSHCommands ) { this.Name = Name; this.SSHPublicKey = SSHPublicKey; this.SSHPrivateKey = SSHPrivateKey; this.IPAddresses = IPAddresses; this.CIDR = CIDR; this.RestrictSSHCommands = RestrictSSHCommands; } public override string ToString() { return string.Format("{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}", this.Name, this.SSHPublicKey, this.SSHPrivateKey, this.IPAddresses != null ? string.Join(",", this.IPAddresses) : "", this.CIDR, this.RestrictSSHCommands); } } "@ Add-Type -Language CSharp -ReferencedAssemblies "System.Numerics.dll" @" using System; using System.Numerics; using System.Net; namespace AKSHCI { public class IPRange { public IPAddress start, end; public override string ToString() { return string.Format("{0} - {1}", start, end); } } public static class IPUtilities { public static int CompareIpAddresses(IPAddress ipAddress1, IPAddress ipAddress2) { byte[] ipAddress1Bytes = ipAddress1.GetAddressBytes(); byte[] ipAddress2Bytes = ipAddress2.GetAddressBytes(); Array.Reverse(ipAddress1Bytes); Array.Reverse(ipAddress2Bytes); Array.Resize<byte>(ref ipAddress1Bytes, 5); // Make sure the first byte is a 0 so BigInterger considers the number as unsigned Array.Resize<byte>(ref ipAddress2Bytes, 5); // Make sure the first byte is a 0 so BigInterger considers the number as unsigned BigInteger ipAddress1BigInt = new BigInteger(ipAddress1Bytes); BigInteger ipAddress2BigInt = new BigInteger(ipAddress2Bytes); return BigInteger.Compare(ipAddress1BigInt, ipAddress2BigInt); } public static IPAddress GetLastIpInCidr(IPAddress ipAddressStr, int prefixLength) { BigInteger fullMask = new BigInteger(0xFFFFFFFF); BigInteger mask = ((fullMask >> (prefixLength)) & fullMask); byte[] endAddress = mask.ToByteArray(); Array.Resize<byte>(ref endAddress, 4); Array.Reverse(endAddress); byte[] ipAddress = ipAddressStr.GetAddressBytes(); if(ipAddress.Length != endAddress.Length) { throw new System.InvalidOperationException("Address and prefix length are both expected to be IPv4 (" + ipAddress.Length + " != " + endAddress.Length + ")"); } for(int i = 0; i < ipAddress.Length; i++) { endAddress[i] = (byte) (endAddress[i] | ipAddress[i]); } return new IPAddress(endAddress); } public static IPAddress ToIPAddress(BigInteger bi) { var bytes = bi.ToByteArray(); Array.Resize<byte>(ref bytes, 4); Array.Reverse(bytes); return new IPAddress(bytes); } public static BigInteger ToBigInteger(IPAddress ip) { var ipBytes = ip.GetAddressBytes(); Array.Reverse(ipBytes); Array.Resize<byte>(ref ipBytes, ipBytes.Length + 1); // Make sure the first byte is a 0 so BigInterger considers the number as unsigned return new BigInteger(ipBytes); } public static void ToRange(string CIDR, out IPAddress start, out IPAddress end) { var s = CIDR.Split('/'); start = IPAddress.Parse(s[0]); var prefixLength = int.Parse(s[1]); end = AKSHCI.IPUtilities.GetLastIpInCidr(start, prefixLength); } public static bool ValidateRange(string rangeStart, string rangeEnd) { var start = IPAddress.Parse(rangeStart); var startBI = ToBigInteger(start); var end = IPAddress.Parse(rangeEnd); var endBI = ToBigInteger(end); if (endBI < startBI) { return false; } return true; } public static bool ValidateIPInCIDR(string ip, string CIDR) { var ipaddress = IPAddress.Parse(ip); var ipBI = ToBigInteger(ipaddress); IPAddress cidrStart, cidrEnd; ToRange(CIDR, out cidrStart, out cidrEnd); var cidrStartBI = ToBigInteger(cidrStart); var cidrEndBI = ToBigInteger(cidrEnd); if ((ipBI >= cidrStartBI) && (ipBI <= cidrEndBI)) { return true; } return false; } public static bool ValidateRangeInCIDR(string rangeStart, string rangeEnd, string CIDR) { if (ValidateIPInCIDR(rangeStart, CIDR) && ValidateIPInCIDR(rangeEnd, CIDR)) { return true; } return false; } public static IPRange[] GetVMIPPool(string vippoolStart, string vippoolEnd, string CIDR) { var start = IPAddress.Parse(vippoolStart); var startBI = ToBigInteger(start); var end = IPAddress.Parse(vippoolEnd); var endBI = ToBigInteger(end); IPAddress cidrStart, cidrEnd; ToRange(CIDR, out cidrStart, out cidrEnd); var cidrStartBI = ToBigInteger(cidrStart); var cidrEndBI = ToBigInteger(cidrEnd); if ((startBI == cidrStartBI) && (endBI == cidrEndBI)) { throw new Exception(string.Format("The VIP pool range ({0} - {1}) is too large. There is no space to allocate IP addresses for VM's. Try decreasing the size of the VIP pool.", vippoolStart, vippoolEnd)); } if (startBI == cidrStartBI) { var ippoolstart = ToIPAddress(endBI + 1); var ippoolend = ToIPAddress(cidrEndBI); return new IPRange[] { new IPRange{ start = ippoolstart, end = ippoolend } }; } else if (endBI == cidrEndBI) { var ippoolstart = ToIPAddress(cidrStartBI); var ippoolend = ToIPAddress(startBI - 1); return new IPRange[] { new IPRange { start = ippoolstart, end = ippoolend } }; } else { var ippool1start = ToIPAddress(cidrStartBI); var ippool1end = ToIPAddress(startBI - 1); var ippool2start = ToIPAddress(endBI + 1); var ippool2end = ToIPAddress(cidrEndBI); return new IPRange[] { new IPRange { start = ippool1start, end = ippool1end }, new IPRange { start = ippool2start, end = ippool2end } }; } } //This function checks if an IP address is part of an IP pool. //Example : IP is 10.10.10.1 and IP pool is 10.10.0.0 to 10.10.255.255 -> Return True -> overlap. public static bool CheckIPInIPPool(string ippoolStart, string ippoolEnd, string ipAddressStr) { //Get BigInteger for pool's Start IP var start = IPAddress.Parse(ippoolStart); var startBI = ToBigInteger(start); //Get BigInteger for pool's End IP var end = IPAddress.Parse(ippoolEnd); var endBI = ToBigInteger(end); //Get BigInteger for CIDR IP var ipAddr = IPAddress.Parse(ipAddressStr); var ipAddrBI = ToBigInteger(ipAddr); if ((ipAddrBI <= endBI) && (ipAddrBI >= startBI)) { //IP is present in IP Pool return true; } return false; } } } "@; #endregion #region Module constants $global:AksHciModule = "AksHci" $global:MocModule = "Moc" $global:KvaModule = "Kva" $global:DownloadModule = "DownloadSdk" $global:CommonModule = "Common" $global:configurationKeys = @{ $global:AksHciModule = "HKLM:SOFTWARE\Microsoft\${global:AksHciModule}PS"; $global:MocModule = "HKLM:SOFTWARE\Microsoft\${global:MocModule}PS"; $global:KvaModule = "HKLM:SOFTWARE\Microsoft\${global:KvaModule}PS"; } $global:repositoryName = "PSGallery" $global:repositoryNamePreview = "PSGallery" $global:repositoryUser = "" $global:repositoryPass = "" $global:loadBalancerTypeStr = @('unstacked-haproxy','unmanaged','stacked-kube-vip','sdn-load-balancer') #endregion #region VM size definitions $global:vmSizeDefinitions = @( # Name, CPU, MemoryGB, GPUType, GPUCount ([VmSize]::Default, "4", "4", "", ""), ([VmSize]::Standard_A2_v2, "2", "4", "", ""), ([VmSize]::Standard_A4_v2, "4", "8", "", ""), ([VmSize]::Standard_D2s_v3, "2", "8", "", ""), ([VmSize]::Standard_D4s_v3, "4", "16", "", ""), ([VmSize]::Standard_D8s_v3, "8", "32", "", ""), ([VmSize]::Standard_D16s_v3, "16", "64", "", ""), ([VmSize]::Standard_D32s_v3, "32", "128", "", ""), ([VmSize]::Standard_DS2_v2, "2", "7", "", ""), ([VmSize]::Standard_DS3_v2, "2", "14", "", ""), ([VmSize]::Standard_DS4_v2, "8", "28", "", ""), ([VmSize]::Standard_DS5_v2, "16", "56", "", ""), ([VmSize]::Standard_DS13_v2, "8", "56", "", ""), ([VmSize]::Standard_K8S_v1, "4", "2", "", ""), ([VmSize]::Standard_K8S2_v1, "2", "2", "", ""), ([VmSize]::Standard_K8S3_v1, "4", "6", "", ""), ([VmSize]::Standard_NK6, "6", "12", "Tesla T4", "1"), ([VmSize]::Standard_NK12, "12", "24", "Tesla T4", "2") ) #endregion #region Pod names and selectors $global:managementPods = @( ("Cloud Operator", "cloudop-system", "control-plane=controller-manager"), ("Cluster API core", "capi-system", "cluster.x-k8s.io/provider=cluster-api"), ("Bootstrap kubeadm", "capi-kubeadm-bootstrap-system", "cluster.x-k8s.io/provider=bootstrap-kubeadm"), ("Control Plane kubeadm", "capi-kubeadm-control-plane-system", "cluster.x-k8s.io/provider=control-plane-kubeadm"), ("Cluster API core Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=cluster-api"), ("Bootstrap kubeadm Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=bootstrap-kubeadm"), ("Control Plane kubeadm Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=control-plane-kubeadm"), ("AzureStackHCI Provider", "caph-system", "cluster.x-k8s.io/provider=infrastructure-azurestackhci"), ("AzureStackHCI Provider Webhook", "capi-webhook-system", "cluster.x-k8s.io/provider=infrastructure-azurestackhci") ) #endregion # -ErrorAction:SilentlyContinue Import-LocalizedData -BindingVariable "GenericLocMessage" -FileName commonLocalizationMessages #region Classes #region to capture CustomException error details enum ErrorTypes { IsUserErrorFlag #User Input Error IsInfraErrorFlag #Infrastructure Error IsErrorFlag #Error during execution } class CustomException: Exception { [ErrorTypes] $errorflag CustomException([string] $message, [ErrorTypes] $errorflag) : base($message) { $this.errorflag = $errorflag } CustomException() {} } #endregion class NetworkPlugin { [string] $Name static [string] $Default = "calico" NetworkPlugin( [string] $name ) { $curatedName = $name.ToLower() if ($curatedName -ne "flannel" -and $curatedName -ne "calico" -and $curatedName -ne "cilium") { throw [CustomException]::new("Invalid CNI '$curatedName'. The only supported CNIs are 'flannel', 'calico' and 'cilium'", ([ErrorTypes]::IsUserErrorFlag)) } $this.Name = $curatedName } NetworkPlugin() { $this.Name = [NetworkPlugin]::Default } } #endregion #region Script Constants $global:installDirectoryName = "AksHci" $global:workingDirectoryName = "AksHci" $global:imageDirectoryName = "AksHciImageStore" $global:yamlDirectoryName = "yaml" $global:cloudConfigDirectoryName = "wssdcloudagent" $global:nodeConfigDirectoryName = "wssdagent" $global:installDirectory = $($env:ProgramFiles + "\" + $global:installDirectoryName) $global:defaultworkingDir = $($env:SystemDrive + "\" + $global:workingDirectoryName) $global:defaultStagingShare = "" $global:nodeAgentBinary = "wssdagent.exe" $global:cloudAgentBinary = "wssdcloudagent.exe" $global:nodeCtlBinary = "nodectl.exe" $global:cloudCtlBinary = "mocctl.exe" $global:kubectlBinary = "kubectl.exe" $global:kvactlBinary = "kvactl.exe" $global:cloudOperatorYaml = "cloud-operator.yaml" $global:nodeAgentFullPath = [io.Path]::Combine($global:installDirectory, $global:nodeAgentBinary) $global:cloudAgentFullPath = [io.Path]::Combine($global:installDirectory, $global:cloudAgentBinary) $global:nodeCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:nodeCtlBinary) $global:cloudCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:cloudCtlBinary) $global:kubeCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:kubectlBinary) $global:kvaCtlFullPath = [io.Path]::Combine($global:installDirectory, $global:kvactlBinary) $script:psConfigKeyName = "psconfig" $script:psConfigJson = "psconfig.json" $global:psConfigDirectoryRoot = $($env:USERPROFILE) $global:mocMetadataRoot = $($env:USERPROFILE + "\.wssd") $global:mocMetadataDirectory = [io.Path]::Combine($global:mocMetadataRoot, "mocctl" ) $global:kvaMetadataDirectory = [io.Path]::Combine($global:mocMetadataRoot, "kvactl" ) $global:accessFileLocation = [io.Path]::Combine($global:mocMetadataDirectory, "cloudconfig") $global:accessFileDir = "CloudCfg" $global:accessFileDirMoc = "mocctl" $global:accessFileDirKva = "kvactl" $global:accessFileName = "cloudconfig" $global:multiAdminRelease = "1.0.13.10907" $global:defaultCloudConfigLocation = $($env:SystemDrive + "\programdata\" + $global:cloudConfigDirectoryName) $global:defaultNodeConfigLocation = $($env:SystemDrive + "\programdata\" + $global:nodeConfigDirectoryName) $global:defaultMgmtReplicas = 1 $global:defaultMgmtControlPlaneVmSize = [VmSize]::Standard_A4_v2 $global:defaultControlPlaneVmSize = [VmSize]::Standard_A4_v2 $global:defaultLoadBalancerVmSize = [VmSize]::Standard_A4_v2 $global:defaultWorkerVmSize = [VmSize]::Standard_K8S3_v1 $global:defaultNodePoolName = "nodepool1" $global:defaultWorkerNodeCount = 1 $global:defaultWorkerNodeOS = [OsType]::Linux $global:defaultWorkerNodeOsSku = [OsSku]::CBLMariner $global:defaultNodeAgentPort = 45000 $global:defaultNodeAuthorizerPort = 45001 $global:defaultCloudAgentPort = 55000 $global:defaultCloudAuthorizerPort = 65000 $global:defaultVipPoolName = "clusterVipPool" $global:defaultMacPoolStart = "" $global:defaultMacPoolEnd = "" $global:defaultVlanID = 0 $global:failoverCluster = $null $global:cloudAgentAppName = "ca" $global:cloudName = "moc-cloud" $global:defaultCloudLocation = "MocLocation" $global:cloudGroupPrefix = "clustergroup" $global:cloudStorageContainer = "MocStorageContainer" $global:cloudMacPool = "MocMacPool" $global:defaultCreateAutoConfigContainers = $true $global:defaultPodCidr = "10.244.0.0/16" $global:mgmtClusterCidr = "10.200.0.0/16" $global:mgmtControlPlaneCidr = "10.240.0.0/24" $global:workloadPodCidr = "10.244.0.0/16" $global:workloadServiceCidr = "10.96.0.0/12" $global:defaultProxyExemptions = "localhost,127.0.0.1,.svc,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" $global:credentialKey = (24,246,163,38,50,244,215,218,223,10,65,98,19,1,149,106,190,141,144,180,157,135,211,143) $global:clusterNameRegex = "^[a-z0-9][a-z0-9-]*[a-z0-9]$" $global:regexPatternVersionNumber = "^[a-z]{0,1}[0-9]+(?:\.[0-9]+)+$" $global:validTaints = @("NoSchedule","PreferNoSchedule","NoExecute") $global:regexPatternMacAddress = "^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$" $global:regexPatternCidrFormat = "^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$" $global:regexPatternTaintFormat = "^[a-zA-Z0-9-]+={1}[a-zA-Z0-9-]+\:{1}[a-zA-Z]+$" $global:regexPatternIpv4 = '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' $global:defaultLogLineCount = 500000 $global:operatorTokenValidity = 90 $global:addonTokenValidity = 90 $global:certificateValidityFactor = 1.0 $global:caCertificateValidityFactor = 1.0 $global:nodeCertificateValidityFactor = 1.0 $global:caCertRotationThreshold = 90 # Temporary until cross-platform signing is available $global:expectedAuthResponse = @{ "Status" = "Valid"; "SignatureType" = "Authenticode"; "StatusMessage" = "" } $global:cloudAgentTimeout = 300 $global:affinityRuleName = "Cloud Agent-CNO" #endregion #region User Configuration and Defaults $global:autoScalerProfileConfigToKvaYamlKeys = @{ "min-node-count" = "minnodecount"; "max-node-count" = "maxnodecount"; "max-nodes-total" = "maxnodestotal"; "scale-down-enabled" = "scaledownenabled"; "scan-interval" = "scaninterval"; "scale-down-delay-after-add" = "scaledowndelayafteradd"; "scale-down-delay-after-delete" = "scaledowndelayafterdelete"; "scale-down-delay-after-failure" = "scaledowndelayafterfailure"; "scale-down-unneeded-time" = "scaledownunneededtime"; "scale-down-unready-time" = "scaledownunreadytime"; "scale-down-utilization-threshold" = "scaledownutilizationthreshold"; "max-graceful-termination-sec" = "maxgracefulterminationsec"; "balance-similar-node-groups" = "balancesimilarnodegroups"; "expander" = "expander"; "skip-nodes-with-local-storage" = "skipnodeswithlocalstorage"; "skip-nodes-with-system-pods" = "skipnodeswithsystempods"; "max-empty-bulk-delete" = "maxemptybulkdelete"; "new-pod-scale-up-delay" = "newpodscaleupdelay"; "max-total-unready-percentage" = "maxtotalunreadypercentage"; "max-node-provision-time" = "maxnodeprovisiontime"; "ok-total-unready-count" = "oktotalunreadycount" } #endregion #region Configuration Functions function Set-VNetConfiguration { param ( [Parameter(Mandatory=$true)] [String] $module, [Parameter(Mandatory=$true)] [VirtualNetwork] $vnet ) Set-ConfigurationValue -name "vnetName" -value $vnet.Name -module $module Set-ConfigurationValue -name "vswitchName" -value $vnet.VswitchName -module $module Set-ConfigurationValue -name "ipaddressprefix" -value $vnet.IpAddressPrefix -module $module Set-ConfigurationValue -name "gateway" -value $vnet.Gateway -module $module Set-ConfigurationValue -name "dnsservers" -value ($vnet.DnsServers -join ",") -module $module Set-ConfigurationValue -name "macpoolname" -value $vnet.MacPoolName -module $module Set-ConfigurationValue -name "vlanid" -value $vnet.Vlanid -module $module Set-ConfigurationValue -name "vnetvippoolstart" -value $vnet.VipPoolStart -module $module Set-ConfigurationValue -name "vnetvippoolend" -value $vnet.VipPoolEnd -module $module Set-ConfigurationValue -name "k8snodeippoolstart" -value $vnet.K8snodeIPPoolStart -module $module Set-ConfigurationValue -name "k8snodeippoolend" -value $vnet.K8snodeIPPoolEnd -module $module } function Get-VNetConfiguration { param ( [Parameter(Mandatory=$true)] [String] $module ) # vnetName and vswitchName are mandatory fields. # So if both are empty we can assume the vnet was not set. $vnet_name = Get-ConfigurationValue -name "vnetName" -module $module $vnet_vswitchname = Get-ConfigurationValue -name "vswitchName" -module $module $isvnetEmpty = ([string]::IsNullOrWhiteSpace($vnet_name) -and [string]::IsNullOrWhiteSpace($vnet_vswitchname)) if ($isvnetEmpty) { return $null } $vnet_ipaddressprefix = Get-ConfigurationValue -name "ipaddressprefix" -module $module $vnet_gateway = Get-ConfigurationValue -name "gateway" -module $module $vnet_dnsservers = (Get-ConfigurationValue -name "dnsservers" -module $module) -split "," $vnet_macpoolname = Get-ConfigurationValue -name "macpoolname" -module $module $vnet_vlanid = Get-ConfigurationValue -name "vlanid" -module $module $vnet_vippoolstart = Get-ConfigurationValue -name "vnetvippoolstart" -module $module $vnet_vippoolend = Get-ConfigurationValue -name "vnetvippoolend" -module $module $vnet_k8snodeippoolstart = Get-ConfigurationValue -name "k8snodeippoolstart" -module $module $vnet_k8snodeippoolend = Get-ConfigurationValue -name "k8snodeippoolend" -module $module return [VirtualNetwork]::new($vnet_name, $vnet_vswitchname, $vnet_ipaddressprefix, $vnet_gateway, $vnet_dnsservers, $vnet_macpoolname, $vnet_vlanid, $vnet_vippoolstart, $vnet_vippoolend, $vnet_k8snodeippoolstart, $vnet_k8snodeippoolend) } function New-VirtualNetwork { <# .DESCRIPTION A wrapper around [VirutalNetwork]::new that Validates parameters before returning a VirtualNetwork object .PARAMETER name The name of the vnet .PARAMETER vswitchName The name of the vswitch .PARAMETER MacPoolName The name of the mac pool .PARAMETER vlanID The VLAN ID for the vnet .PARAMETER ipaddressprefix The address prefix to use for static IP assignment .PARAMETER gateway The gateway to use when using static IP .PARAMETER dnsservers The dnsservers to use when using static IP .PARAMETER vippoolstart The starting ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services' .PARAMETER vippoolend The ending ip address to use for the vip pool. The vip pool addresses will be used by the k8s API server and k8s services .PARAMETER k8snodeippoolstart The starting ip address to use for VM's in the cluster. .PARAMETER k8snodeippoolend The ending ip address to use for VM's in the cluster. .OUTPUTS VirtualNetwork object .EXAMPLE New-VirtualNetwork -name External -vswitchname External -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE New-VirtualNetwork -name "defaultswitch" -vswitchname "Default Switch" -ipaddressprefix 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 #> param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $name, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $vswitchName, [Parameter(Mandatory=$false)] [String] $MacPoolName = $global:cloudMacPool, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [Parameter(Mandatory=$false)] [String] $ipaddressprefix, [Parameter(Mandatory=$false)] [String] $gateway, [Parameter(Mandatory=$false)] [String[]] $dnsservers, [Parameter(Mandatory=$false)] [String] $vippoolstart, [Parameter(Mandatory=$false)] [String] $vippoolend, [Parameter(Mandatory=$false)] [String] $k8snodeippoolstart, [Parameter(Mandatory=$false)] [String] $k8snodeippoolend ) Test-ValidNetworkName -Name $name | Out-Null if ($dnsservers) { foreach ($dns in $dnsservers) { try { Test-ValidEndpoint -endpoint $dns } catch { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ip_list , $dnsservers))), ([ErrorTypes]::IsUserErrorFlag)) } } } if ($ipaddressprefix -or $gateway -or $dnsservers -or $k8snodeippoolstart -or $k8snodeippoolend) { if ((-not $ipaddressprefix) -or (-not $gateway) -or (-not $dnsservers) -or (-not $k8snodeippoolstart) -or (-not $k8snodeippoolend)) { throw [CustomException]::new(($($GenericLocMessage.comm_ip_param_missing)), ([ErrorTypes]::IsUserErrorFlag)) } } if ($ipaddressprefix) { Test-ValidCIDR -CIDR $ipaddressprefix | Out-Null } if ((-not [string]::IsNullOrWhiteSpace($vippoolstart)) -or (-not [string]::IsNullOrWhiteSpace($vippoolend))) { Test-ValidPool -PoolStart $vippoolstart -PoolEnd $vippoolend -CIDR $ipaddressprefix } if ($k8snodeippoolstart -and $k8snodeippoolend) { Test-ValidPool -PoolStart $k8snodeippoolstart -PoolEnd $k8snodeippoolend -CIDR $ipaddressprefix } return [VirtualNetwork]::new($name, $vswitchname, $ipaddressprefix, $gateway, $dnsservers, $MacPoolName, $vlanID, $vippoolstart, $vippoolend, $k8snodeippoolstart, $k8snodeippoolend) } #region SSH Configuration function Set-SSHConfiguration { param ( [Parameter(Mandatory=$true)] [String] $module, [Parameter(Mandatory=$true)] [SSHConfiguration] $sshConfig ) Set-ConfigurationValue -name "sshConfigName" -value $sshConfig.Name -module $module Set-ConfigurationValue -name "sshPublicKey" -value $sshConfig.SSHPublicKey -module $module Set-ConfigurationValue -name "sshPrivateKey" -value $sshConfig.SSHPrivateKey -module $module Set-ConfigurationValue -name "sshRestrictCommands" -value $sshConfig.RestrictSSHCommands -module $module Set-ConfigurationValue -name "sshCidr" -value $sshConfig.CIDR -module $module Set-ConfigurationValue -name "sshIPAddresses" -value ($sshConfig.IPAddresses -join ",") -module $module } function Get-SSHConfiguration { param ( [Parameter(Mandatory=$true)] [String] $module ) $sshName = Get-ConfigurationValue -name "sshConfigName" -module $module $sshPublicKey = Get-ConfigurationValue -name "sshPublicKey" -module $module $sshPrivateKey = Get-ConfigurationValue -name "sshPrivateKey" -module $module $restrictSSHCommands = Get-ConfigurationValue -name "sshRestrictCommands" -module $module $ipAddresses = (Get-ConfigurationValue -name "sshIPAddresses" -module $module) -split "," $cidr = (Get-ConfigurationValue -name "sshCidr" -module $module) -split "," return [SSHConfiguration]::new($sshName, $sshPublicKey, $sshPrivateKey, $cidr, $ipAddresses, $restrictSSHCommands) } function New-SSHConfiguration { <# .DESCRIPTION A wrapper around [SSHConfiguration]::new that Validates parameters before returning a SSHConfiguration object .PARAMETER name The name of the sshConfiguration .PARAMETER sshPublicKey The path to sshPublicKey file .PARAMETER sshPrivateKey The path to sshPublicKey file .PARAMETER restrictSSHCommands Restict SSH access to certain commands .PARAMETER ipAddressed Restict SSH access to certain ipaddresses .PARAMETER cidr Restict SSH access to a CIDR .OUTPUTS SSHConfiguration object .EXAMPLE New-SSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub .EXAMPLE New-SSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub -cidr 172.16.0.0/24 -gateway 172.16.0.1 -dnsservers 4.4.4.4, 8.8.8.8 -vippoolstart 172.16.0.0 -vippoolend 172.16.0.240 .EXAMPLE New-SSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub -cidr 172.16.0.0/24 .EXAMPLE New-SSHConfiguration -name sshConfig -sshPublicKey C:\AksHci\akshci_rsa.pub -ipAddresses 4.4.4.4,8.8.8.8 .EXAMPLE New-SSHConfiguration -name sshConfig -cidr 172.16.0.0/24 .EXAMPLE New-SSHConfiguration -name sshConfig -ipAddresses 4.4.4.4,8.8.8.8 .EXAMPLE New-SSHConfiguration -name sshConfig -ipAddresses 4.4.4.4,8.8.8.8 -restrictSSHCommands #> [CmdletBinding(DefaultParameterSetName = 'noip')] param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [String] $name, [Parameter(Mandatory=$false)] [String] $sshPublicKey, [Parameter(Mandatory=$false)] [String] $sshPrivateKey, [Parameter(Mandatory=$false)] [Switch] $restrictSSHCommands = $false, [Parameter(Mandatory=$true, ParameterSetName='ipaddresses')] [String[]] $ipAddresses, [Parameter(Mandatory=$true, ParameterSetName='cidr')] [String] $cidr ) if (($PSCmdlet.ParameterSetName -ieq "ipaddresses") -and $ipAddresses ) { foreach ($ip in $ipAddresses) { try { Test-ValidEndpoint -endpoint $ip } catch { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ip_list , $ipAddresses))), $true) } } } if (($PSCmdlet.ParameterSetName -ieq "cidr") -and ![string]::IsNullOrEmpty($cidr)) { Test-ValidCIDR -CIDR $cidr | Out-Null } return [SSHConfiguration]::new($name, $sshPublicKey, $sshPrivateKey, $cidr, $ipAddresses, $restrictSSHCommands) } #endregion #region configuration function Save-ConfigurationDirectory { <# .DESCRIPTION Saves the workingDir of configuration in registry. Handles multinode as well. .PARAMETER WorkingDir WorkingDir to be persisted .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName, [Parameter(Mandatory=$true)] [String] $WorkingDir ) if (Test-MultiNodeDeployment) { # *. If Multinode, replicate this across all nodes Get-ClusterNode -ErrorAction Stop | ForEach-Object { Save-ConfigurationDirectoryNode -moduleName $moduleName -nodeName $_.Name -WorkingDir $WorkingDir } } else { # *. If Standalone, store it locally Save-ConfigurationDirectoryNode -moduleName $moduleName -nodeName ($env:computername) -WorkingDir $WorkingDir } } function Save-ConfigurationDirectoryNode { <# .DESCRIPTION Saves the workingDir of configuration in registry. .PARAMETER nodeName Name of node to save the configuration directory .PARAMETER WorkingDir WorkingDir to be persisted .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName, [Parameter(Mandatory=$true)] [string] $nodeName, [Parameter(Mandatory=$true)] [String] $WorkingDir ) $configDir = [io.Path]::Combine($WorkingDir, "." + $moduleName) Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_save_config_dir , $configDir)) -moduleName $moduleName Invoke-Command -ComputerName $NodeName -ScriptBlock { $regPath = $args[0] $regKey = $args[1] $regValue = $args[2] if (!(Test-Path ($regPath))) { New-Item -Path $regPath | Out-Null } Set-ItemProperty -Path $regPath -Name $regKey -Value $regValue -Force | Out-Null } -ArgumentList @($global:configurationKeys[$moduleName], $script:psConfigKeyName, $configDir) } function Delete-ConfigurationDirectoryNode { <# .DESCRIPTION Deletes the configuration in registry. .PARAMETER nodeName Name of node to save the configuration directory .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName, [Parameter(Mandatory=$true)] [string] $nodeName ) Invoke-Command -ComputerName $NodeName -ScriptBlock { $regPath = $args[0] $regKey = $args[1] Clear-ItemProperty -Path $regPath -Name $regKey -Force | Out-Null } -ArgumentList @($global:configurationKeys[$moduleName], $script:psConfigKeyName) } function Set-SecurePermissionFolder { <# .DESCRIPTION Initialize folder with appropriate permissions .PARAMETER Path path of config folder. #> param ( [parameter(Mandatory=$true)] [string]$Path ) $acl = Get-Acl $Path $acl.SetAccessRuleProtection($true,$false) $adminGroup = New-Object System.Security.Principal.SecurityIdentifier([System.Security.Principal.WellKnownSidType]::BuiltinAdministratorsSid, $null) $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($adminGroup, "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow") $acl.SetAccessRule($accessRule) $acl | Set-Acl $Path } function Set-SecurePermissionFile { <# .DESCRIPTION Initialize file with appropriate permissions .PARAMETER Path path of file. #> param ( [parameter(Mandatory=$true)] [string]$Path ) # ACL the yaml so that it is only readable by administrator $acl = Get-Acl $Path $acl.SetAccessRuleProtection($true,$false) $adminGroup = New-Object System.Security.Principal.SecurityIdentifier([System.Security.Principal.WellKnownSidType]::BuiltinAdministratorsSid, $null) $accessRule = New-Object System.Security.AccessControl.FileSystemAccessRule($adminGroup, "FullControl", "Allow") $acl.SetAccessRule($accessRule) $acl | Set-Acl $Path } function Reset-ConfigurationDirectory { <# .DESCRIPTION Cleanup workingDir info in registry that has been saved .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) Write-Status $($GenericLocMessage.comm_reset_config_dir) -moduleName $moduleName if (Test-MultiNodeDeployment) { # *. If Multinode, remove this across all nodes Get-ClusterNode -ErrorAction Stop | ForEach-Object { Invoke-Command -ComputerName $_.Name -ScriptBlock { $regPath = $args[0] Remove-Item -Path $regPath -Force -ErrorAction SilentlyContinue | Out-Null } -ArgumentList @($global:configurationKeys[$moduleName]) } } else { # *. If Standalone, remove it locally Remove-Item -Path $global:configurationKeys[$moduleName] -Force -ErrorAction SilentlyContinue | Out-Null } } function Get-ConfigurationDirectory { <# .DESCRIPTION Gets the Working Directory of configuration from the registry .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # 1. If the psconfig path is available, try to use it. $regVal = Get-ItemPropertyValue -Path $global:configurationKeys[$moduleName] -Name $script:psConfigKeyName -ErrorAction SilentlyContinue if ($regVal) { return $regVal } # 2. If not, use the default path return [io.Path]::Combine($global:psConfigDirectoryRoot, "." + $moduleName) } function Get-ConfigurationFile { <# .DESCRIPTION Get the configuration file to be used for persisting configurations .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) $configFile = [io.Path]::Combine((Get-ConfigurationDirectory -moduleName $moduleName), $script:psConfigJson) # Write-Status "Configuration file for $moduleName => [$configFile]" return $configFile } function Test-Configuration { <# .DESCRIPTION Tests if a configuration exists .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # Write-Status "Testing Configuration for $moduleName" return Test-Path -Path (Get-ConfigurationFile -moduleName $moduleName) } function Reset-Configuration { <# .DESCRIPTION Resets the configuration Resets also the configuration info that persisted in registry. Does a double cleanup so the one in working dir as well as user directory is removed .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) Write-Status $($GenericLocMessage.comm_reset_config) -moduleName $moduleName # 1. Remove the shared configuration if (Test-Configuration -moduleName $moduleName) { Remove-Item -Path (Get-ConfigurationDirectory -moduleName $moduleName) -Recurse -Force -ErrorAction SilentlyContinue } Reset-ConfigurationDirectory -moduleName $moduleName # 2. Remove the local configuration, if any if (Test-Configuration -moduleName $moduleName) { Remove-Item -Path (Get-ConfigurationDirectory -moduleName $moduleName) -Recurse -Force -ErrorAction SilentlyContinue } $global:config[$moduleName] = @{} } function Save-Configuration { <# .DESCRIPTION saves a configuration to persisted storage .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) $configFile = Get-ConfigurationFile -moduleName $moduleName $configDir = [IO.Path]::GetDirectoryName($configFile) if (!(Test-Path $configDir)) { New-Item -ItemType Directory -Force -Path $configDir | Out-Null } Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_save_config , $moduleName, $configFile)) -moduleName $moduleName ConvertTo-Json -InputObject $global:config[$moduleName] | Out-File -FilePath $configFile Set-SecurePermissionFile -Path $configFile } function Import-Configuration { <# .DESCRIPTION Loads a configuration from persisted storage .PARAMETER moduleName Name of the module #> param ( [Parameter(Mandatory=$true)] [string] $moduleName ) # Write-Status "Importing Configuration for $moduleName" $tmp = ConvertFrom-Json -InputObject (Get-Content (Get-ConfigurationFile -moduleName $moduleName) -Raw) $global:config[$moduleName] = @{} $tmp.psobject.Properties | ForEach-Object { $global:config[$moduleName][$_.Name] = $_.Value} } function Set-ConfigurationValue { <# .DESCRIPTION Persists a configuration value to the registry .PARAMETER name Name of the configuration value .PARAMETER moduleName Name of the module .PARAMETER value Value to be persisted #> param ( [String] $name, [Parameter(Mandatory=$true)] [String] $module, [Object] $value ) $global:config[$module][$name] = $value Save-Configuration -moduleName $module } function Get-ConfigurationValue { <# .DESCRIPTION Retrieves a configuration value from the registry .PARAMETER type The expected type of the value being retrieved .PARAMETER module module of the module .PARAMETER name Name of the configuration value #> param ( [Type] $type = [System.String], [Parameter(Mandatory=$true)] [String] $module, [String] $name ) $value = $null if (Test-Configuration -moduleName $module) { Import-Configuration -moduleName $module $value = $global:config[$module][$name] } switch($type.Name) { "Boolean" { if (!$value) {$value = 0} return [System.Convert]::ToBoolean($value) } "UInt32" { if (!$value) {$value = 0} return [System.Convert]::ToUInt32($value) } "VmSize" { if (!$value) {$value = 0} return [Enum]::Parse([VmSize], $value, $true) } "DeploymentType" { if (!$value) {$value = 0} return [Enum]::Parse([DeploymentType], $value, $true) } "InstallState" { if (!$value) {$value = 0} return [Enum]::Parse([InstallState], $value, $true) } "LoadBalancerType" { if (!$value) {$value = 0} return [Enum]::Parse([LoadBalancerType], $value, $true) } Default { if (!$value) {$value = ""} return $value } } } #endregion function ConvertTo-SystemVersion { <# .DESCRIPTION Converts a string representation of a version to a System.Version object. .PARAMETER Version The version string to be converted. #> param ( [String] $Version ) $converted = $null if ([System.Version]::TryParse($Version, [ref] $converted)) { return $converted } throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_version_str_err , $Version)), ([ErrorTypes]::IsErrorFlag)) } function Compare-Versions { <# .DESCRIPTION Compares two string versions and returns an indication of their relative values. The comparison is performed by compariing the major version numbers for equality (per SemVer specification). The return value is a signed integer that indicates the relative values of the two objects: Less than zero - Version has a major version lower than the ComparisonVersion. Zero - The two major versions are the same (i.e. they are compatible). Greater than zero - Version has a newer major version than the ComparisonVersion. .PARAMETER Version The current version number. .PARAMETER ComparisonVersion A version number to be compared with the CurrentVersion. .OUTPUTS System.Int32 #> param ( [String] $Version, [String] $ComparisonVersion ) $current = ConvertTo-SystemVersion -Version $Version $comparison = ConvertTo-SystemVersion -Version $ComparisonVersion if ($current.Major -eq $comparison.Major) { return 0 } if ($current.Major -lt $comparison.Major) { return -1 } return 1 } function Test-IsProductInstalled() { <# .DESCRIPTION Tests if the desired product/module is installed (or installing). Note that we consider some failed states (e.g. UninstallFailed) to represent that the product is still installed, albeit in a unknown/failed state. .PARAMETER moduleName The module name to test for installation state .PARAMETER activity Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $moduleName, [Parameter()] [String]$activity = $MyInvocation.MyCommand.Name ) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $($GenericLocMessage.comm_verify_prod_installation_state) $currentState = Get-InstallState -module $moduleName if (-not $currentState) { return $false } switch($currentState) { $([InstallState]::NotInstalled) { return $false } $([InstallState]::InstallFailed) { return $false } } return $true } function Get-InstallState { <# .DESCRIPTION Returns the installation state for a product/module. May return $null. .PARAMETER moduleName The module name to query #> param ( [String] $moduleName ) $state = Get-ConfigurationValue -module $moduleName -name "InstallState" -type ([Type][InstallState]) if (-not $state) { $state = [InstallState]::NotInstalled } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_installation_state , $state)) return $state } function Test-ValidEndpoint { <# .DESCRIPTION Validates that an endpoint is a valid ip address This function exists to validate that a endpoint is a valid ip address. If the enpoint is not a valid ip address, it throws. .PARAMETER endpoint A string representing an IP address #> param ( [String] $endpoint ) if (-not [ipaddress]::TryParse($endpoint,[ref]$null)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_invalid_ip , $endpoint)), ([ErrorTypes]::IsErrorFlag)) } } function Test-ValidK8sObjectName { <# .DESCRIPTION Validates the format of a name that will be used as the name of a kubernetes object This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name The name of an k8s object .PARAMETER Type Currently we are only using k8s names for clusters and networks #> param ( [String] $Name, [ValidateSet("cluster","network","nodepool")] [String] $Type ) if (-not ($Name -cmatch $clusterNameRegex)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_inavlid_name , $Name, $Type, $clusterNameRegex)) } return $true } function Test-ValidClusterName { <# .DESCRIPTION Validates the format of a cluster name. This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name A cluster name #> param ( [String] $Name ) return Test-ValidK8sObjectName -Name $Name -Type "cluster" } function Test-ValidNetworkName { <# .DESCRIPTION Validates the format of a network name. This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name A network name #> param ( [String] $Name ) return Test-ValidK8sObjectName -Name $Name -Type "network" } function Test-ValidNodePoolName { <# .DESCRIPTION Validates the format of a node pool name. This function exists so that multiple other functions can re-use the same validation rather than us repeating it all over the script (ValidatePattern does not allow variables to be used for the regex string as it must be a constant). We throw to provide a more specific error message to the caller. .PARAMETER Name A node pool name #> param ( [String] $Name ) return Test-ValidK8sObjectName -Name $Name -Type "nodepool" } function Test-ForUpdates { <# .DESCRIPTION Check if a module is up to date and provide the option to update it. .PARAMETER moduleName The name of the module. .PARAMETER repositoryName Powershell repository name. .PARAMETER repositoryUser Powershell repository username. .PARAMETER repositoryPass Powershell repository password. #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '')] param ( [String] $moduleName, [String] $repositoryName, [String] $repositoryUser, [String] $repositoryPass ) if ($global:config[$moduleName]["skipUpdates"]) { return } Write-Status $($GenericLocMessage.comm_module_updates) -moduleName $moduleName $proxyParameters = @{} $proxyConfig = Get-ProxyConfiguration -moduleName $moduleName if ($proxyConfig.HTTPS) { $proxyParameters.Add("Proxy", $proxyConfig.HTTPS) } elseif ($proxyConfig.HTTP) { $proxyParameters.Add("Proxy", $proxyConfig.HTTP) } if ($proxyConfig.Credential) { $proxyParameters.Add("ProxyCredential", $proxyConfig.Credential) } $current = Get-InstalledModule -Name "PowershellGet" -ErrorAction SilentlyContinue if (($null -eq $current) -or ($current.version -lt 1.6.0)) { Write-SubStatus $($GenericLocMessage.comm_psget_update) -moduleName $moduleName $parameters = @{ Name = "PowershellGet" Force = $true Confirm = $false SkipPublisherCheck = $true } $parameters += $proxyParameters Install-Module @parameters Write-SubStatus $($GenericLocMessage.comm_psget_update_done) -moduleName $moduleName exit 0 } $patToken = $repositoryPass | ConvertTo-SecureString -AsPlainText -Force $repositoryCreds = New-Object System.Management.Automation.PSCredential($repositoryUser, $patToken) $current = Get-InstalledModule -Name $moduleName -ErrorAction SilentlyContinue # TODO: Skip alpha/preview updates for now if ($current.Repository -ieq $global:repositoryNamePreview) { Write-SubStatus $($GenericLocMessage.comm_current_version_pre_release) -moduleName $moduleName return } $parameters = @{ Name = $moduleName Repository = $repositoryName Credential = $repositoryCreds ErrorAction = "SilentlyContinue" } $parameters += $proxyParameters $latest = Find-Module @parameters if (($null -eq $current) -or ($null -eq $latest)) { Write-SubStatus $($GenericLocMessage.comm_update_unable) -moduleName $moduleName return } Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_installed_version , $current.version)) -moduleName $moduleName Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_latest_version , $latest.version)) -moduleName $moduleName if ([System.Version]$current.version -ge [System.Version]$latest.version) { Write-SubStatus $($GenericLocMessage.comm_already_uptodate) -moduleName $moduleName return } Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_new_version , $moduleName)) -moduleName $moduleName $title = 'Recommended update' $question = "Do you want to update to the latest version of the module/binaries? If you choose 'Yes' then we will perform a full cleanup of your deployment before applying the updates.`n" $choices = '&Yes', '&No' $decision = $Host.UI.PromptForChoice($title, $question, $choices, 0) if ($decision -eq 0) { Write-Status $($GenericLocMessage.comm_installing_updates) -moduleName $moduleName $parameters = @{ Name = $moduleName Credential = $repositoryCreds Force = $true } $parameters += $proxyParameters Update-Module @parameters $current = Get-InstalledModule -Name $moduleName -ErrorAction SilentlyContinue Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_installed_version_now , $current.version)+"`n") -moduleName $moduleName if ([System.Version]$current.version -ge [System.Version]$latest.version) { Write-SubStatus $($GenericLocMessage.comm_remove_older_version+"`n") -moduleName $moduleName Get-InstalledModule -Name $moduleName -AllVersions | Where-Object {$_.Version -ne $current.Version} | Uninstall-Module -Force -Confirm:$false -ErrorAction:SilentlyContinue Write-SubStatus $($GenericLocMessage.comm_update_successful) -moduleName $moduleName exit 0 } throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_update_failed , $current.version, $latest.version)) } } function Initialize-Environment { <# .DESCRIPTION Executes steps to prepare the environment for operations. This includes checking to ensure that the module is up to date and that a configuration is present. .PARAMETER moduleName The name of the module. .PARAMETER repositoryName Powershell repository name. .PARAMETER repositoryUser Powershell repository username. .PARAMETER repositoryPass Powershell repository password. .PARAMETER checkForUpdates Should the script check for updates before proceeding to prepare the environment. .PARAMETER createIfNotPresent Should the script create a new deployment configuration if one is not present. .PARAMETER activity Activity name #> param ( [Parameter()] [String] $moduleName, [Parameter()] [String] $repositoryName = $global:repositoryName, [Parameter()] [String] $repositoryUser = $global:repositoryUser, [Parameter()] [String] $repositoryPass = $global:repositoryPass, [Parameter()] [Switch]$checkForUpdates, [Parameter()] [Switch]$createIfNotPresent, [Parameter()] [String]$activity ) Initialize-ProxyEnvironment -moduleName $moduleName $commands = Get-InvokedPSCommands -module $moduleName $currentCommand = [Command]::new($activity, [System.Diagnostics.Process]::GetCurrentProcess().Id, $(hostname), $(get-date)) $commands += $currentCommand Set-ConfigurationValue -name "commands" -value $commands -module $moduleName if ($checkForUpdates.IsPresent) { Test-ForUpdates -moduleName $moduleName -repositoryName $repositoryName -repositoryUser $repositoryUser -repositoryPass $repositoryPass } } function Uninitialize-Environment { <# .DESCRIPTION Executes steps to teardown the environment for operations. .PARAMETER moduleName The name of the module. .PARAMETER activity Activity name #> param ( [Parameter()] [String] $moduleName, [Parameter()] [String]$activity ) $commands = Get-InvokedPSCommands -module $moduleName $processId = [System.Diagnostics.Process]::GetCurrentProcess().Id $commandName = $activity $resultCommands = @() ForEach ($command in $commands) { if ((($command).cmdletName -eq $commandName) -and (($command).processId -eq $processId) -and (($command).computerName -eq $(hostname))) { # Write-Warning "Removing command $(($command).cmdletName)" continue } $resultCommands += $command } if ($resultCommands.Count -lt 10) { Set-ConfigurationValue -name "commands" -value $resultCommands -module $moduleName return } # cleanup the commands which are executed by powershell which is no longer active $resultRunningCommands = @() $runningPowershellProcesses = $(Get-Process -Name powershell) ForEach ($command in $resultCommands) { ForEach ($runningPowershellProcesss in $runningPowershellProcesses) { if ($runningPowershellProcesss.Id -eq ($command).processId) { $resultRunningCommands += $command break } } } Set-ConfigurationValue -name "commands" -value $resultRunningCommands -module $moduleName } function Get-InvokedPSCommands { <# .DESCRIPTION Gets the commands from config which are running .PARAMETER moduleName The name of the module. #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $commands = Get-ConfigurationValue -Name "commands" -module $moduleName if (-not $commands) { return @() } # Due to automatic unrolling in powershell, we need the below if condition # https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_return?view=powershell-7.2#return-values-and-the-pipeline if ($commands.GetType() -eq [Type][System.Management.Automation.PSCustomObject]) { $result = @($commands) Write-Output -NoEnumerate $result return } # casting to array return [System.Array]$commands } function Get-RunningPSCommands { <# .DESCRIPTION Gets the commands from config which are running .PARAMETER Name Name of the command .PARAMETER moduleName The name of the module. #> [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $Name, [Parameter(Mandatory=$true)] [String] $moduleName ) $commands = Get-InvokedPSCommands -module $moduleName $result = @() Foreach ($command in $commands) { if ($command.cmdletName -match $Name) { # Write-Host "Already there is a command with $Name in commands history" $process = $(Get-Process -PID $command.processId -ComputerName $command.computerName -ErrorAction SilentlyContinue) if ($process) { $result += $command } else { # Write-Host "Command ${command.cmdletName} with ${command.processId} is not running" } } } return $($result | Sort-Object -Property @{Expression={$_.processId}} -Unique) } function Get-FirewallRules { <# .DESCRIPTION Obtains the firewall rules needed for agent communication #> $firewallRules = @( ("wssdagent GRPC server port", "TCP", $global:config[$global:MocModule]["nodeAgentPort"]), ("wssdagent GRPC authentication port", "TCP", $global:config[$global:MocModule]["nodeAgentAuthorizerPort"]), ("wssdcloudagent GRPC server port", "TCP", $global:config[$global:MocModule]["cloudAgentPort"]), ("wssdcloudagent GRPC authentication port", "TCP", $global:config[$global:MocModule]["cloudAgentAuthorizerPort"]) ) return $firewallRules } #endregion #region General helper functions function Get-URLResponseCode { <# .DESCRIPTION Get URL response code .PARAMETER URL The URL to test connection with .PARAMETER timeoutSec Specifics how long the request can be pending before it times out. .PARAMETER proxySettings Proxy setting on Host #> param ( [String] $URL, [int] $timeoutSec = 15, [AllowNull()][Object] $proxySettings ) $result = $null try { if ($proxySettings -and $proxySettings.IsProxyConfigured) { $result = (Invoke-WebRequest -uri $URL -UseBasicParsing -Proxy $proxySettings.ProxyServer -ProxyCredential $proxySettings.Credential -TimeoutSec $timeoutSec).StatusCode } else { $result = (Invoke-WebRequest -uri $URL -UseBasicParsing -TimeoutSec $timeoutSec).StatusCode } } catch [System.Net.WebException]{ $result = $($_.Exception.Response.StatusCode.Value__) if ([string]::IsNullOrEmpty($result)){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_http_no_response, $URL, $proxySettings)) } } return $result } function Update-DirectoryPath { <# .DESCRIPTION Return normailize directory path .PARAMETER directoryPath The directory path to be normalized .PARAMETER useBackwardSlashes Convert all slashes to back slashes, or forward slashes if set to false #> param ( [Parameter(Mandatory=$true)] [AllowEmptyString()] [String] $directoryPath, [bool] $useBackslashes = $true ) $ret = "" if (-not [string]::IsNullOrWhiteSpace($directoryPath)) { $slash = '\' $prefix = "" if($useBackslashes) { $directoryPath = $directoryPath.Replace('/','\') } else { $directoryPath = $directoryPath.Replace('\', '/') $slash = '/' } $doubleSlashes = $slash + $slash if($directoryPath.StartsWith($doubleSlashes)) { $directoryPath = $directoryPath.Substring(2) $prefix = $doubleSlashes } elseif($directoryPath.StartsWith($slash)) { $directoryPath = $directoryPath.Substring(1) $prefix = $slash } $directoryPath = $directoryPath.TrimEnd($slash) $strs = $directoryPath.Split($slash) foreach($str in $strs){ if($str.length -gt 0){ $ret += $str + $slash } } $ret = $prefix + $ret.TrimEnd($slash) } return $ret } function Write-StatusWithProgress { <# .DESCRIPTION Outputs status to progress and to console .PARAMETER status The status message .PARAMETER activity The progress activity .PARAMETER percentage The progress percentage. 100% will output progress completion .PARAMETER moduleName The module name. Will become a prefix for console output #> [CmdletBinding()] param ( [String] $status = "", [String] $activity = "Status", [Int] $percentage = -1, [Switch] $completed, [Parameter(Mandatory=$true)] [String] $moduleName ) # Propagate verbose preference across modules from the caller, if not explicitly specified if (-not $PSBoundParameters.ContainsKey('Verbose')) { $script:VerbosePreference = $PSCmdlet.GetVariableValue('VerbosePreference') } $message = $status if ($moduleName) { $eventmessage = $("$activity - $message") $message = $("$moduleName - $message") Write-ModuleEventLog -moduleName $moduleName -entryType Information -eventId 1 -message $eventmessage } Write-Progress -Activity $activity -Status $message -PercentComplete $percentage -Completed:$completed.IsPresent Write-Status -msg $status -moduleName $moduleName } function Write-Status { <# .DESCRIPTION Outputs status to the console with a prefix for readability .PARAMETER msg The message/object to output .PARAMETER moduleName The module name. Will be used as a prefix #> [CmdletBinding()] param ( [Object]$msg, [Parameter(Mandatory=$true)] [String]$moduleName ) # Propagate verbose preference across modules from the caller, if not explicitly specified if (-not $PSBoundParameters.ContainsKey('Verbose')) { $script:VerbosePreference = $PSCmdlet.GetVariableValue('VerbosePreference') } if ($msg) { $time = Get-Date -DisplayHint Time Write-Verbose "[$time] [$moduleName] $msg`n" } } function Write-SubStatus { <# .DESCRIPTION Outputs sub-status to the console with a indent for readability .PARAMETER msg The message/object to output .PARAMETER moduleName The module name. Will be used as a prefix .PARAMETER indentChar Char to use as the indent/bulletpoint of the status message #> [CmdletBinding()] param ( [Object]$msg, [Parameter(Mandatory=$true)] [String]$moduleName, [String]$indentChar = "`t`t" ) # Propagate verbose preference across modules from the caller, if not explicitly specified if (-not $PSBoundParameters.ContainsKey('Verbose')) { $script:VerbosePreference = $PSCmdlet.GetVariableValue('VerbosePreference') } Write-Status -msg $($indentChar + $msg) -moduleName $moduleName } function GetDefaultLinuxNodepoolName { <# .DESCRIPTION Gets the Default Linux Nodepool name. .PARAMETER clusterName Cluster Name. #> param ( [String]$clusterName ) # GA build named the default nodepools as $Name-default-linux-nodepool # fallback if $clusterName-linux is not found. $linuxNodepool = Invoke-Kubectl -ignoreError -arguments $("get akshcinodepools/$clusterName-default-linux-nodepool -o json") | ConvertFrom-Json if ($null -eq $linuxNodepool) { return "$clusterName-linux" } return "$clusterName-default-linux-nodepool" } function GetDefaultWindowsNodepoolName { <# .DESCRIPTION Gets the Default Windows Nodepool Name .PARAMETER clusterName Cluster Name. #> param ( [String]$clusterName ) # GA build named the default nodepools as $Name-default-windows-nodepool # fallback if $clusterName-windows is not found. $windowsNodepool = Invoke-Kubectl -ignoreError -arguments $("get akshcinodepools/$clusterName-default-windows-nodepool -o json") | ConvertFrom-Json if ($null -eq $windowsNodepool) { return "$clusterName-windows" } return "$clusterName-default-windows-nodepool" } function Test-Process { <# .DESCRIPTION Test if a process is running. .PARAMETER processName The process name to test. .PARAMETER nodeName The node to test on. #> param ( [String]$processName, [String]$nodeName ) try { Invoke-Command -ComputerName $nodeName -ScriptBlock { $processName = $args[0] $GenericLocMessage = $args[1] $process = Get-Process -Name $processName -ErrorAction SilentlyContinue if(!$process) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_process_not_running , $processName, $env:computername)) } } -ArgumentList $processName, $GenericLocMessage } catch { throw [CustomException]::new($_, ([ErrorTypes]::IsInfraErrorFlag)) } } function Test-LocalFilePath { <# .DESCRIPTION Returns true if the path appears to be local. False otherwise. .PARAMETER path Path to be tested. #> param( [String]$path ) $path = [System.Environment]::ExpandEnvironmentVariables($path) $clusterStorage = Join-Path -Path $([System.Environment]::ExpandEnvironmentVariables("%systemdrive%")) -ChildPath 'clusterstorage' if($path.StartsWith('\\') -or $path.ToLower().StartsWith($clusterStorage.ToLower())) { return $false } return $true } function Stop-AllProcesses { <# .DESCRIPTION Kill all instance of the given process name. #> param ( [string]$processName, [string]$computerName = "localhost" ) Invoke-Command -ComputerName $computerName { $processName = $args[0] Get-Process -Name $processName -ErrorAction SilentlyContinue | ForEach-Object { $_ | Stop-Process -Force } } -ArgumentList $processName } function Copy-FileLocal { <# .DESCRIPTION Copies a file locally. .PARAMETER source File source. .PARAMETER destination File destination. #> param ( [String]$source, [String]$destination ) Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_copy , $source, $destination)) -moduleName $global:CommonModule Copy-Item -Path $source -Destination $destination -Recurse } function Copy-FileToRemoteNode { <# .DESCRIPTION Copies a file to a remote node. .PARAMETER source File source. .PARAMETER destination File destination. .PARAMETER remoteNode The remote node to copy to. #> param ( [String]$source, [String]$destination, [String]$remoteNode ) $remotePath = "\\$remoteNode\" + ($destination).Replace(":", "$") Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_copy , $source, $remotePath)) -moduleName $global:CommonModule $remoteDir = [IO.Path]::GetDirectoryName($remotePath) if (!(Test-Path $remoteDir)) { New-Item -ItemType Directory -Force -Path $remoteDir | Out-Null } while($true) { try { Copy-Item -Path $source -Destination $remotePath return } catch { Write-Warning $_.Exception.Message -ErrorAction Continue Start-Sleep -Seconds 5 $process = [io.path]::GetFileNameWithoutExtension($destination) Stop-AllProcesses -computerName $remoteNode -processName $process } } } function Test-ForWindowsFeatures { <# .DESCRIPTION Installs any missing required OS features. .PARAMETER features The features to check for. .PARAMETER nodeName The node to execute on. #> param ( [String[]]$features, [String]$nodeName ) Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_req_os_feature , $nodeName)) -moduleName $global:MocModule $nodeEditionName = Invoke-Command -ComputerName $nodeName -ScriptBlock { return (get-itemproperty "hklm:\software\microsoft\windows nt\currentversion").editionid } if (-not ($nodeEditionName -match 'server')) { # Comment out this check # throw $($GenericLocMessage.comm_support_server_editions) } try { $remoteRebootRequired = Invoke-Command -ComputerName $nodeName -ScriptBlock { $rebootRequired = $false $GenericLocMessage = $args[1] foreach($feature in $args[0]) { write-verbose $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_checking_status_of_feature, $feature)) $wf = Get-WindowsFeature -Name "$feature" if ($null -eq $wf) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_feature_not_found , $feature)) } if ($wf.InstallState -ine "Installed") { write-verbose $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_installing_missing_feature, $feature)) $result = Install-WindowsFeature -Name "$feature" -WarningAction SilentlyContinue if ($result.RestartNeeded) { $rebootRequired = $true } } } return $rebootRequired } -ArgumentList $features, $GenericLocMessage } catch { throw [CustomException]::new($_, ([ErrorTypes]::IsInfraErrorFlag)) } if ($remoteRebootRequired) { Write-Status $($GenericLocMessage.comm_reboot_req) -moduleName $global:MocModule Read-Host $("Press enter when you are ready to reboot $nodeName ...") Restart-Computer -ComputerName $nodeName -Force } } function Enable-Remoting { <# .DESCRIPTION Enables powershell remoting on the local machine. #> Write-Status $($GenericLocMessage.comm_ps_remote) -moduleName $global:MocModule Enable-PSRemoting -Force -Confirm:$false winrm quickconfig -q -force } function Get-HostRoutingInfo { <# .DESCRIPTION Obtains the host routing information. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) try { return Invoke-Command -ComputerName $nodeName -ScriptBlock { $GenericLocMessage = $args[0] $oldProgressPreference = $global:progressPreference $global:progressPreference = 'silentlyContinue' $computerName = "www.msftconnecttest.com" $routingInfo = Test-NetConnection -DiagnoseRouting -ComputerName $computerName -ErrorAction SilentlyContinue $global:progressPreference = $oldProgressPreference if (!$routingInfo -or !$routingInfo.RouteDiagnosticsSucceeded) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_conn_test_failed , $computerName)) } return $routingInfo } -ArgumentList $GenericLocMessage } catch { throw [CustomException]::new($_, ([ErrorTypes]::IsInfraErrorFlag)) } } function Get-HostAdapterName { <# .DESCRIPTION Obtains the name of the best host network adapter. Uses routing to determine the host interface name to return. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) $routingInfo = Get-HostRoutingInfo -nodeName $nodeName return $routingInfo.OutgoingInterfaceAlias } function Get-HostIp { <# .DESCRIPTION Obtains the hosts IP address. Uses routing to determine the best host IPv4 address to use. .PARAMETER nodeName The node to execute on. #> param ( [String]$nodeName ) $routingInfo = Get-HostRoutingInfo -nodeName $nodeName return $routingInfo.SelectedSourceAddress.IPv4Address } function Test-Binary { <# .DESCRIPTION A basic sanity test to make sure that this system is ready to deploy kubernetes. .PARAMETER nodeName The node to execute on. .PARAMETER binaryName Binary to check. #> param ( [String]$nodeName, [String]$binaryName ) try { Invoke-Command -ComputerName $nodeName -ScriptBlock { $GenericLocMessage = $args[1] if ( !(Get-Command $args[0] -ErrorAction SilentlyContinue )) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_missing_binary , $args[0])) } } -ArgumentList $binaryName, $GenericLocMessage } catch { throw [CustomException]::new($_, ([ErrorTypes]::IsInfraErrorFlag)) } } function Get-CloudFqdn { <# .DESCRIPTION Determines the right FQDN to use for cloudagent based on the type of deployment and script args #> return $global:config[$global:MocModule]["cloudFqdn"] } function Get-SshPublicKey { <# .DESCRIPTION Get the SSH Public Key that is configured to be used by the deployment #> return $global:config[$global:MocModule]["sshPublicKey"] } function Get-SshIPAddress { <# .DESCRIPTION Get SSH IP addresses for VM restrictions #> if ([string]::IsNullOrWhiteSpace($global:config[$global:MocModule]["sshIPAddresses"])) { return $global:config[$global:MocModule]["sshCIDR"] } return $global:config[$global:MocModule]["sshIPAddresses"] } function Get-IsSSHRestricted { <# .DESCRIPTION Get if ssh should be restricted #> return $global:config[$global:MocModule]["sshRestrictCommands"] } function Get-SshPrivateKey { <# .DESCRIPTION Get the SSH Private Key that is configured to be used by the deployment #> return $global:config[$global:MocModule]["sshPrivateKey"] } function Get-SSHAuthConfigSudoRestrictions { <# .DESCRIPTION Get the SSH authorized key config for config #> $publicKey = Get-Content -Path (Get-SshPublicKey) $restrictedSSH = Get-IsSSHRestricted $sshIPAddress = Get-SshIPAddress if ($restrictedSSH) { $retPublicKey = "command=""/usr/sbin/ssh-commands.sh""" } if ($restrictedSSH -and (-not [string]::IsNullOrWhiteSpace($sshIPAddress))) { $retPublicKey += "," } if (-not [string]::IsNullOrWhiteSpace($sshIPAddress)) { $retPublicKey += "from=""" + $sshIPAddress + """" } if ($restrictedSSH -or (-not [string]::IsNullOrWhiteSpace($sshIPAddress))) { $retPublicKey += " " } $retPublicKey += $publicKey return $retPublicKey } function Get-SSHAuthConfig { <# .DESCRIPTION Get the SSH authorized key config for config #> $publicKey = Get-Content -Path (Get-SshPublicKey) $sshIPAddress = Get-SshIPAddress if (-not [string]::IsNullOrWhiteSpace($sshIPAddress)) { $retPublicKey += "from=""" + $sshIPAddress + """ " } $retPublicKey += $publicKey return $retPublicKey } function ConvertTo-ArgString { <# .DESCRIPTION Takes a dictionary of parameters and converts them to a string representation. Convert-ParametersToString is designed for powershell Cmdlet arguments. However this one is for binaries. It doesn't add "-" in front parameter names and joins the value by comma if its an array. .PARAMETER argDictionary Dictionary of arguments (e.g. obtained from $PSBoundParameters). .PARAMETER boolFlags List of boolean flags to be passed .PARAMETER separator Separates flag and value; default: " " (single space) #> param ( [System.Collections.IDictionary] $argDictionary, [string[]] $boolFlags, [string] $separator = " " ) [string[]] $argsList = @() if ($argDictionary) { foreach ($key in $argDictionary.Keys) { $argsList += ('{0}{1}"{2}"' -f $key, $separator, $($argDictionary[$key] -join ",")) } } if ($boolFlags) { $argsList += $boolFlags } return $argsList -join " " } #region invoke methods function Invoke-CommandLine { <# .DESCRIPTION Executes a command and optionally ignores errors. .PARAMETER command Comamnd to execute. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. .PARAMETER showOutputAsProgress Optionally, show output from the executing command as progress bar updates. .PARAMETER progressActivity The activity name to display when showOutputAsProgress was requested. .PARAMETER moduleName The calling module name to show in output logging. #> param ( [String]$command, [String]$arguments, [Switch]$ignoreError, [Switch]$showOutput, [Switch]$showOutputAsProgress, [String]$progressActivity, [Parameter(Mandatory=$true)] [String]$moduleName ) $currentErrorActionPreference=$ErrorActionPreference # We need to set erroractionpreference 'continue' to get multiline errors $ErrorActionPreference="continue" try { if ($showOutputAsProgress.IsPresent) { $result = (& $command $arguments.Split(" ") | ForEach-Object { $status = $_ -replace "`t"," - "; Write-StatusWithProgress -activity $progressActivity -moduleName $moduleName -Status $status }) 2>&1 } elseif ($showOutput.IsPresent) { $result = (& $command $arguments.Split(" ") | Out-Default) 2>&1 } else { $result = (& $command $arguments.Split(" ") 2>&1) } } catch { if ($ignoreError.IsPresent) { return } throw } finally { $ErrorActionPreference=$currentErrorActionPreference } $out = $result | Where-Object {$_.gettype().Name -ine "ErrorRecord"} # On a non-zero exit code, this may contain the error #$outString = ($out | Out-String).ToLowerInvariant() if ($LASTEXITCODE) { $err = $result | Where-Object {$_.gettype().Name -eq "ErrorRecord"} $errMessage = "$command $arguments $GenericLocMessage.generic_non_zero $LASTEXITCODE [$err]" if ($ignoreError.IsPresent) { $ignoreMessage = "[IGNORED ERROR] $errMessage" Write-Status -msg $ignoreMessage -moduleName $moduleName Write-ModuleEventLog -moduleName $moduleName -entryType Warning -eventId 2 -message $errMessage return } throw [CustomException]::new($errMessage, ([ErrorTypes]::IsErrorFlag)) } return $out } function Invoke-Kubectl { <# .DESCRIPTION Executes a kubectl command. .PARAMETER kubeconfig The kubeconfig file to use. Defaults to the management kubeconfig. .PARAMETER arguments Arguments to pass to the command. .PARAMETER ignoreError Optionally, ignore errors from the command (don't throw). .PARAMETER showOutput Optionally, show live output from the executing command. #> param ( [string] $kubeconfig = $global:config[$global:KvaModule]["kubeconfig"], [string] $arguments, [switch] $ignoreError, [switch] $showOutput ) return Invoke-CommandLine -command $global:kubeCtlFullPath -arguments $("--kubeconfig=""$kubeconfig"" $arguments") -showOutput:$showOutput.IsPresent -ignoreError:$ignoreError.IsPresent -moduleName $global:KvaModule } #end region function Compress-Directory { <# .DESCRIPTION Util for zipping folders .PARAMETER ZipFilename output zip file name .PARAMETER SourceDir directory to compress #> param ( [Parameter(Mandatory=$true)] [String]$ZipFilename, [Parameter(Mandatory=$true)] [String]$SourceDir ) if (Test-Path $ZipFilename) { $title = 'ZipFile already exists' $question = "Do you want to overwrite it?`n" $choices = '&Yes', '&No' $decision = $Host.UI.PromptForChoice($title, $question, $choices, 0) if ($decision -eq 0) { Remove-Item -Path $ZipFilename -Force } else { throw $([System.IO.IOException] $GenericLocMessage.generic_file_exists) } } Add-Type -Assembly System.IO.Compression.FileSystem $compressionLevel = [System.IO.Compression.CompressionLevel]::Optimal [System.IO.Compression.ZipFile]::CreateFromDirectory($SourceDir,$ZipFilename, $compressionLevel, $false) } #endregion #region Resource limit helper functions function Convert-ParametersToString { <# .DESCRIPTION Takes a dictionary of parameters and converts them to a string representation. .PARAMETER argDictionary Dictionary of arguments (e.g. obtained from $PSBoundParameters). .PARAMETER stripAsJob Optionally remove 'AsJob' if it's included in the argDictionary. #> param ( [System.Collections.IDictionary] $argDictionary, [Switch] $stripAsJob ) $strArgs = "" foreach ($key in $argDictionary.Keys) { if (($key -ieq "AsJob") -and ($stripAsJob.IsPresent)) { continue } $seperator = " " $val = $argDictionary[$key] if (($val -eq $true) -or ($val -eq $false)) { $seperator = ":$" } $strArgs += " -$key$seperator$val" } return $strArgs } #endregion #region prechecks function Test-HCIRegistration { <# .DESCRIPTION Check the SKU of node and if HCI substrate then check the registration status of the node #> $osResult = Get-CimInstance -Namespace root/CimV2 -ClassName Win32_OperatingSystem -Property OperatingSystemSKU # PRODUCT_AZURESTACKHCI_SERVER_CORE = 406 # Check if the substrate is HCI if ($osResult.OperatingSystemSKU -eq 406) { $hciStatus = Get-AzureStackHCI #Check if the HCI node is registered else throw error $regStatus = ($hciStatus).RegistrationStatus if ($regStatus -ine "Registered") { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unregistered_node , $env:computername, $regStatus)), ([ErrorTypes]::IsInfraErrorFlag)) } if ($hciStatus.ConnectionStatus -ne "Connected") { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_outofpolicy , $hciStatus.AzureResourceName, $hciStatus.ConnectionStatus)), ([ErrorTypes]::IsInfraErrorFlag)) } # $hciSubStatus = Get-AzureStackHCISubscriptionStatus | Where-Object { $_.SubscriptionName -eq "Azure Stack HCI" } # if ($hciSubStatus.Status -ne "Subscribed") # { # throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_hcistatusnotsubscribed , $hciSubStatus.Status)) # } } } function Test-ClusterHealth { <# .DESCRIPTION Check if cluster node and cluster network are up #> #Check the ClusterNode Get-ClusterNode -ErrorAction Stop | ForEach-Object { if ($_.State -ine "Up") { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_cluster_node_state , $_, $($_.State))), ([ErrorTypes]::IsInfraErrorFlag)) } } Get-ClusterNetwork -ErrorAction Stop | ForEach-Object { # we only want to fail when the network state is not 'up' AND the network role is not 'None' if ($_.State -ine "Up" -and $_.Role -ine "None") { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_cluster_network_state , $_, $($_.State))) } } } #endregion #region Asynchronous job helpers function Get-BackgroundJob { <# .DESCRIPTION Returns a background job by name (supports wildcards and may return multiple jobs) .PARAMETER name Name of the job(s). #> param ( [Parameter(Mandatory=$true)] [String] $name ) return Get-Job -Name $name -ErrorAction SilentlyContinue } function New-BackgroundJob { <# .DESCRIPTION Creates a new background job. .PARAMETER name Name of the job. .PARAMETER cmdletName Cmdlet to execute as a job .PARAMETER argDictionary Argument dictionary to pass to the job cmdlet. .PARAMETER scheduledJob Optionally, use a scheduled job instead of a regular job. This allows the job to execute fully in the background and be accessible cross-session. Note that less progress information is available while this type of job is running. .PARAMETER allowDuplicateJobs Optionally, allow a new job to be created even if a job with the same name already exists and is still executing. #> param ( [Parameter(Mandatory=$true)] [String] $name, [Parameter(Mandatory=$true)] [String] $cmdletName, [Parameter(Mandatory=$true)] [System.Collections.IDictionary] $argDictionary, [Parameter()] [Switch] $scheduledJob, [Parameter()] [Switch] $allowDuplicateJobs ) if (-not $allowDuplicateJobs.IsPresent) { $jobs = Get-BackgroundJob -name $name foreach ($job in $jobs) { if (($job.State -ieq "Completed") -or ($job.State -ieq "Failed")) { continue } throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_job_exists , $name)) } } $strArgs = Convert-ParametersToString($argDictionary) -stripAsJob if ($scheduledJob.IsPresent) { if (-not $strArgs) { # Always pass at least one arg $strArgs = '-Verbose:$false' } $options = New-ScheduledJobOption -RunElevated -HideInTaskScheduler return Register-ScheduledJob -Name $name -ScheduledJobOption $options -RunNow -ScriptBlock { param($p1,$p2) #$VerbosePreference = "continue" Invoke-Expression $("$p1 $p2") } -ArgumentList $cmdletName,$strArgs } else { return Start-Job -Name $name -ScriptBlock { Invoke-Expression $("$using:cmdletName $using:strArgs") } } } #endregion function Get-FailoverCluster { <# .DESCRIPTION Safe wrapper around checking for the presence of a failover cluster. #> # Check if failover cluster powershell module was installed # and only run Get-Cluster in that case if (Get-Command "Get-Cluster" -errorAction SilentlyContinue) { return Get-Cluster -ErrorAction SilentlyContinue } return $null } function Get-KubernetesGalleryImageName { <# .DESCRIPTION Returns the appropriate gallery image name based on kubernetes versions .PARAMETER k8sVersion Kubernetes version .PARAMETER imageType Image Type .PARAMETER osSku SKU of the image: CBLMariner, Windows2019 or Windows2022 #> param ( [Parameter(Mandatory=$true)] [String]$k8sVersion, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String]$imageType, [Parameter(Mandatory=$true)] [String]$releaseVersion, [String]$osSku, [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } switch ($imageType) { "Windows" { $imageName = "Windows_k8s" # if ossku specified for ws2022 if ($osSku -ieq "Windows2022") { $imageName += "_2022" } break } "Linux" { $tmpVersion = ($k8sVersion.TrimStart("v").Replace('.', '-')) $imageName = "Linux_k8s_" + $tmpVersion break } } return $imageName + "_" + $releaseVersion } function Get-LegacyKubernetesGalleryImageName { <# .DESCRIPTION Returns the appropriate gallery image name based on kubernetes versions using the legacy naming convention .PARAMETER k8sVersion Kubernetes version .PARAMETER imageType Image Type .PARAMETER osSku SKU of the image: CBLMariner, Windows2019 or Windows2022 #> param ( [Parameter(Mandatory=$true)] [String]$k8sVersion, [Parameter(Mandatory=$true)] [ValidateSet("Windows", "Linux")] [String]$imageType, [String]$osSku, [String] $activity ) if (-not $activity) { $activity = "$($MyInvocation.MyCommand.Name) - $Name" } $tmpVersion = ($k8sVersion.TrimStart("v").Replace('.', '-')) # if ossku specified for ws2022 if ($osSku -ieq "Windows2022") { return "Windows_k8s_2022" } switch ($imageType) { "Windows" { return "Windows_k8s" } "Linux" { return "Linux_k8s_" + $tmpVersion } } } function Test-MultiNodeDeployment { <# .DESCRIPTION Returns true if the script believes this is a multi-node deployment. False otherwise. #> $failoverCluster = Get-FailoverCluster return ($null -ne $failoverCluster) } function Get-Ipv4MaskFromPrefix { <# .DESCRIPTION Transforms an IP prefix length to an IPv4 net mask. .PARAMETER PrefixLength Length of the prefix #> param ( [Parameter(Mandatory=$true)] [int] $PrefixLength ) Add-Type -Language CSharp -ReferencedAssemblies "System.Numerics.dll" @" using System; using System.Numerics; namespace AKSHCI { public static class Ipv4MaskCompute { public static byte[] GetIpv4MaskFromPrefix(int prefixLength) { BigInteger fullMask = new BigInteger(0xFFFFFFFF); BigInteger mask = ((fullMask << (32 - prefixLength)) & fullMask); return mask.ToByteArray(); } } } "@; if ($PrefixLength -lt 0) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_invalid_prefix_length , $PrefixLength)) } if ($PrefixLength -eq 0) { return "0.0.0.0" } if ($PrefixLength -gt 32) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_invalid_prefix_length , $PrefixLength)) } $maskArray = [AKSHCI.Ipv4MaskCompute]::GetIpv4MaskFromPrefix($PrefixLength) return "$($maskArray[3]).$($maskArray[2]).$($maskArray[1]).$($maskArray[0])" } function Get-ClusterNetworkPrefixForIp { <# .DESCRIPTION Returns the cluster network prefix length associated with the given IP, or $null if not found. .PARAMETER IpAddress Find the cluster network associated with this IP address #> param ( [Parameter(Mandatory=$true)] [System.Net.IPAddress] $IpAddress ) $v4Networks = (Get-ClusterNetwork -ErrorAction SilentlyContinue | Where-Object { $_.Ipv4Addresses.Count -gt 0 }) foreach($v4Network in $v4Networks) { for($i = 0; $i -lt $v4Network.Ipv4Addresses.Count; $i++) { [System.Net.IPAddress]$ipv4 = $null $clusIpv4 = $v4Network.Ipv4Addresses[$i] if (-Not [System.Net.IPAddress]::TryParse($clusIpv4, [ref] $ipv4)) { Write-Warning $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_ignore_failover_ip , $clusIpv4)) continue } $lastIp = [AKSHCI.IPUtilities]::GetLastIpInCidr($ipv4, $v4Network.Ipv4PrefixLengths[$i]) if([AKSHCI.IPUtilities]::CompareIpAddresses($ipAddress, $ipv4) -ge 0 -AND [AKSHCI.IPUtilities]::CompareIpAddresses($ipAddress, $lastIp) -le 0) { return $v4Network.Ipv4PrefixLengths[$i] } } } throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_cluster_network_not_found , $IpAddress)), ([ErrorTypes]::IsInfraErrorFlag)) } function Get-FailoverClusterResourceState { <# .DESCRIPTION Gets the status information of the resource .PARAMETER clusterResource Cluster Resource #> param ( [Parameter(Mandatory=$true)] [string] $resourceName ) $extendedStatus = "" $tmp = Get-ClusterResource -Name $resourceName -ErrorAction Ignore if ($tmp) { $extendedStatus = "CurrentState: $($tmp.State), LastOperationStatusCode: $($tmp.LastOperationStatusCode), StatusInformation: $($tmp.StatusInformation)." } return $extendedStatus } function Get-FailoverClusterGroupState { <# .DESCRIPTION Gets the status information of the group .PARAMETER clusterGroupName Cluster group #> param ( [Parameter(Mandatory=$true)] [string] $clusterGroupName ) $extendedStatus = "" $tmp = Get-ClusterGroup -Name $clusterGroupName -ErrorAction Ignore if ($tmp) { $extendedStatus = "CurrentState: $($tmp.State), LastOperationStatusCode: $($tmp.LastOperationStatusCode), StatusInformation: $($tmp.StatusInformation)." } return $extendedStatus } function Start-FailoverClusterResource { <# .DESCRIPTION Starts a failover cluster resource .PARAMETER resourceName Name of the resource .PARAMETER waitTimeMinutes waitTime in minutes (Example: -waitTimeMinutes 5) #> param ( [Parameter(Mandatory=$true)] [string] $resourceName, [int] $waitTimeMinutes = 5 ) Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_starting_cluster_resource , $resourceName)) -moduleName $global:MocModule # 1. Start the cluster resource try { Start-ClusterResource -Name $resourceName -ErrorAction Stop | Out-Null } catch [Exception] { $extendedError = Get-FailoverClusterResourceState -resourceName $resourceName throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_failed_to_start_cluster_resource, $resourceName, $extendedError, $($_.Exception.Message.ToString()))) } # 2. Wait for up to 5 minutes for the transition to happen $clusterResource = (Get-ClusterResource -Name $resourceName -ErrorAction Stop) $waitStart = [DateTime]::get_Now() while ($clusterResource.State -eq [Microsoft.FailoverClusters.PowerShell.ClusterResourceState]::Pending) { $extendedState = Get-FailoverClusterResourceState -resourceName $resourceName Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_waiting_for_resource, $resourceName, $extendedState)) -moduleName $global:MocModule if (([DateTime]::get_Now() - $waitStart) -ge [Timespan]::FromMinutes($waitTimeMinutes)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_timed_out_waiting_for_resource, $resourceName, $extendedState)), ([ErrorTypes]::IsInfraErrorFlag)) } Start-Sleep 5 $clusterResource = (Get-ClusterResource -Name $resourceName -ErrorAction Stop) } Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_started_cluster_resource , $resourceName)) -moduleName $global:MocModule } function Start-FailoverClusterGroup { <# .DESCRIPTION Start and wait for cluster group .PARAMETER clusterGroupName Name of the cluster group (Example: ca-2f87825b-a4af-473f-8a33-8e3bdd5f9b61) .PARAMETER waitTimeMinutes waitTime in minutes (Example: -waitTimeMinutes 5) #> param ( [Parameter(Mandatory=$true)] [string] $clusterGroupName, [int] $waitTimeMinutes = 5 ) Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_starting_cluster_group , $clusterGroupName)) -moduleName $global:MocModule # 1. Start the cluster resource try { Start-ClusterGroup -Name $clusterGroupName -ErrorAction Stop | Out-Null } catch [Exception] { $extendedError = Get-FailoverClusterGroupState -clusterGroupName $clusterGroupName throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_failed_to_start_cluster_group, $clusterGroupName, $extendedError, $($_.Exception.Message.ToString()))), ([ErrorTypes]::IsInfraErrorFlag)) } # 2. Wait for up to 5 minutes for the transition to happen - ie ip resource to become ready $clusterGroup = (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop) $waitStart = [DateTime]::get_Now() while ($clusterGroup.State -eq [Microsoft.FailoverClusters.PowerShell.ClusterGroupState]::Pending) { $extendedState = Get-FailoverClusterGroupState -clusterGroupName $clusterGroupName Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_waiting_for_group, $clusterGroupName, $extendedState)) -moduleName $global:MocModule if (([DateTime]::get_Now() - $waitStart) -ge [Timespan]::FromMinutes($waitTimeMinutes)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_timed_out_waiting_for_group_in_failover_cluster, $extendedState)), ([ErrorTypes]::IsInfraErrorFlag)) } Start-Sleep 5 $clusterGroup = (Get-ClusterGroup -Name $clusterGroupName -ErrorAction Stop) } Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_started_cluster_group , $clusterGroupName)) -moduleName $global:MocModule } function Get-ClusterGroupUUID { <# .DESCRIPTION Failover Cluster Group's name is localized into different languages and we need to use the UUID of the ClusterGroup to identiy instead of using the name #> $clusterGroupUUID = (Get-ItemProperty "HKLM:\Cluster").ClusterGroup if ([string]::IsNullOrEmpty($clusterGroupUUID)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unable_to_retrieve_cluster_group)), ([ErrorTypes]::IsInfraErrorFlag)) } return $clusterGroupUUID } function Get-ClusterNameResourceUUID { <# .DESCRIPTION Like Failover Cluster Group, cluster name resource is also localized into different languages and we need to use the uuid of the cluster name resource instead of using the name. #> $clusterNameResourceUUID = ((Get-ItemProperty "HKLM:\Cluster").ClusterNameResource) if ([string]::IsNullOrEmpty($clusterNameResourceUUID)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unable_to_retrieve_cluster_name_resource)), ([ErrorTypes]::IsInfraErrorFlag)) } return $clusterNameResourceUUID } function Add-FailoverClusterNetworkResource { <# .DESCRIPTION Creates the failover cluster resource for DNS Network Name and IP Addresses. This method is called only when ClusterRoleName is not default .PARAMETER clusterGroupName Name of the cluster group .PARAMETER staticIpCidr Static IP and network prefix, using CIDR format #> param( [string] $staticIpCidr, [parameter(Mandatory=$true)] [string] $clusterGroupName ) # 1. Create and start the resource in order: DNS Name, IP Addresses $dnsName = Add-ClusterResource -Name "$clusterGroupName" -ResourceType "Network Name" -Group $clusterGroupName -ErrorAction Stop $dnsName | Set-ClusterParameter -Multiple @{"Name"="$clusterGroupName";"DnsName"="$clusterGroupName"} -ErrorAction Stop # 2. Create and Start the resources in order - IP resource, Service and then start the Resource Group if ([string]::IsNullOrWhiteSpace($staticIpCidr)) { # 1.a DHCP Case $networkList = Get-ClusterNetwork -ErrorAction SilentlyContinue | Where-Object { $_.Role -eq "ClusterAndClient" } foreach ($network in $networkList) { $IPResourceName = "IPv4 Address on $($network.Address)" $IPAddress = Add-ClusterResource -Name $IPResourceName -ResourceType "IP Address" -Group $clusterGroupName -ErrorAction Stop $IPAddress | Set-ClusterParameter -Multiple @{"Network"=$network;"EnableDhcp"=1} -ErrorAction Stop Add-ClusterResourceDependency -Resource "$clusterGroupName" -Provider $IPResourceName -ErrorAction Stop | Out-Null try { Start-FailoverClusterResource -resourceName $IPResourceName -waitTimeMinutes 5 } catch { $errorMessage = Write-ModuleEventException -message "Start-FailoverClusterResource failed on resource $IPResourceName" -exception $_ -moduleName $global:MocModule throw [CustomException]::new($([System.Exception]::new([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_failover_cluster_networks_error, $errorMessage), $_.Exception)), ([ErrorTypes]::IsInfraErrorFlag)) } } } else { # 1.b Static IP Case $staticIpCidrArray = $staticIpCidr.Split("/") #Split a string of format x.x.x.x/pp to an array of x.x.x.x and pp if ($staticIpCidrArray.Length -eq 1) { $prefixLength = Get-ClusterNetworkPrefixForIp -IpAddress $staticIpCidrArray[0] } else { $prefixLength = $staticIpCidrArray[1] } $subnetMask = Get-Ipv4MaskFromPrefix -PrefixLength $prefixLength $IPResourceName = "IPv4 Address $($staticIpCidrArray[0])" $IPAddress = Add-ClusterResource -Name $IPResourceName -ResourceType "IP Address" -Group $clusterGroupName -ErrorAction Stop $IPAddress | Set-ClusterParameter -Multiple @{"Address"=$staticIpCidrArray[0];"SubnetMask"=$subnetMask;"EnableDhcp"=0} -ErrorAction Stop Add-ClusterResourceDependency -Resource "$clusterGroupName" -Provider $IPResourceName -ErrorAction Stop | Out-Null try { Start-FailoverClusterResource -resourceName $IPResourceName -waitTimeMinutes 5 } catch { $errorMessage = Write-ModuleEventException -message "Start-FailoverClusterResource failed." -exception $_ -moduleName $global:MocModule throw [CustomException]::new($([System.Exception]::new([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ip_address, $($staticIpCidrArray[0]), $errorMessage), $_.Exception)), ([ErrorTypes]::IsInfraErrorFlag)) } } } function Add-FailoverClusterCloudServiceResource { <# .DESCRIPTION Creates the failover cluster resouce for cloud service parameters .PARAMETER serviceDisplayName Display name of the service .PARAMETER serviceName Name of the service binary .PARAMETER serviceParameters Service start parameters .PARAMETER clusterGroupName Name of the cluster group #> param( [Parameter(Mandatory=$true)] [string] $serviceDisplayName, [Parameter(Mandatory=$true)] [string] $clusterGroupName, [Parameter(Mandatory=$true)] [string] $serviceName, [Parameter(Mandatory=$true)] [string] $serviceParameters, [Parameter()] [int] $useNetworkName = 0, [Parameter(Mandatory=$true)] [bool] $useUpdateFailoverClusterCreationFlow ) #0. Get other parameters if (!$useUpdateFailoverClusterCreationFlow) { $useNetworkName = 1 } # 1. Add cloud service parameter to cluster group $ServiceConfig = Add-ClusterResource -Name $serviceDisplayName -ResourceType "Generic Service" -Group $clusterGroupName -ErrorAction Stop # only add DNS name as cluster resource dependency if customer is bring their own DNS if (!$useUpdateFailoverClusterCreationFlow) { Add-ClusterResourceDependency -Resource $serviceDisplayName -Provider "$clusterGroupName" -ErrorAction Stop | Out-Null } $ServiceConfig | Set-ClusterParameter -Multiple @{"ServiceName"=$serviceName;"StartupParameters"=$serviceParameters;"UseNetworkName"=$useNetworkName} -ErrorAction Stop # 1.a Add cluster affinity rule if we are attaching failover cluster group to existing cluster "Cluster Group" if ($useUpdateFailoverClusterCreationFlow) { $clusterGroupUUID = Get-ClusterGroupUUID New-ClusterAffinityRule -name "$global:affinityRuleName" -ruletype SameNode -ErrorAction Stop Add-ClusterGroupToAffinityRule -name "$global:affinityRuleName" -groups "$clusterGroupName", "$clusterGroupUUID" -ErrorAction Stop } # 2. Start the service . try { Start-FailoverClusterResource -resourceName $serviceDisplayName -waitTimeMinutes 5 } catch { $errorMessage = Write-ModuleEventException -message "Start-FailoverClusterResource failed " -exception $_ -moduleName $global:MocModule throw [CustomException]::new($([System.Exception]::new([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_issue_while_registering_resource_name, $errorMessage), $_.Exception)), ([ErrorTypes]::IsInfraErrorFlag)) } } function Add-FailoverClusterGenericRole { <# .DESCRIPTION Creates a generic service role in failover cluster (similar to Add-ClusterGenericServiceRole), but allows for fine tuning network configuration. .PARAMETER staticIpCidr Static IP and network prefix, using the CIDR format (Example: 192.168.1.2/16) .PARAMETER serviceDisplayName Display name of the service (Example: "WSSD cloud agent service") .PARAMETER clusterGroupName Name of the cluster group (Example: ca-2f87825b-a4af-473f-8a33-8e3bdd5f9b61) .PARAMETER serviceName Name of the service binary (Example: wssdcloudagent) .PARAMETER serviceParameters Service start parameters (Example: --fqdn ca-2f87825b-a4af-473f-8a33-8e3bdd5f9b61.contoso.com) #> param ( [string] $staticIpCidr, [Parameter(Mandatory=$true)] [string] $serviceDisplayName, [Parameter(Mandatory=$true)] [string] $clusterGroupName, [Parameter(Mandatory=$true)] [string] $serviceName, [Parameter(Mandatory=$true)] [string] $serviceParameters ) # 0 - Prerequisite Add-ClusterGroup -Name $clusterGroupName -GroupType GenericService -ErrorAction Stop | Out-Null $useUpdateFailoverClusterCreationFlow = $global:config[$global:MocModule]["useUpdatedFailoverClusterCreationLogic"] try { # 1 - Only create network resource if customer is not using default cluster role name # - cx's environment does not have *-affinityRuleCommand present meaning running a build older then WS2019 if (!$useUpdateFailoverClusterCreationFlow) { Add-FailoverClusterNetworkResource -staticIpCidr $staticIpCidr -clusterGroupName $clusterGroupName } # 2 - Add and start wssd cloud agent service resource to the failover cluster Add-FailoverClusterCloudServiceResource -serviceDisplayName $serviceDisplayName -clusterGroupName $clusterGroupName -serviceName $serviceName ` -serviceParameters $serviceParameters -useUpdateFailoverClusterCreationFlow $useUpdateFailoverClusterCreationFlow # 3. Start the cluster group and wait at most 5 minutes for it to come online Start-FailoverClusterGroup -clusterGroupName $clusterGroupName -waitTimeMinutes 5 } catch [Exception] { Remove-ClusterGroup -Name $clusterGroupName -RemoveResources -Force -ErrorAction Ignore throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_cloudagent_service_failed_to_start, $clusterGroupName, $($_.Exception.Message.ToString()))), ([ErrorTypes]::IsInfraErrorFlag)) } } function Test-UseUpdatedFailoverClusterCreationFlow { <# .DESCRIPTION After Feb 2023 release, there are 2 versions on how we create the cloud agent's failover cluster group and assign network name/IP address to cloud agent V1 (Old method): Regardless of whether customer provides a custom clusterRoleName or not, we will spin up the network name and IP address for cloud agent. customer can also pass in a cloud service IP to be used for their cloud agent service. The cloud service IP must be preregistered. If customer is using a static network setup, a cloud service IP is required. V2 (New method): If customer chooses the default clusterRoleName and the customer is running a WS2019 or later OS Build, we will use "Cluster Group's" existing network name and IP Address. Thus we will not bring up any network resources (Netwokr Name, IP Address) for cloud agent. If the customer provides their own preregistered cluster role name, we will bring up the network resources (network name, IP Address) for customer. For static Networking case, cloud service IP is not required if customer is using default cluster role name, but is required if customer is using custom cluster role name. V1 v.s V2: If customer's environment has "*-ClusterAffinityRule" command and is using a default cluster role name, then we will use V2 creation flow. For all other scenarios, we will use the V1 creation flow. V1 Prechecks: 1. All Previous checks in Test-ClusterNetworkProperties with addition 2. Test-FailoverClusterCreate V2 Prechecks: 1. All previous checks in Test-ClusterNetworkProperties with exception 2. Test CIDR required when customer is providing their own cluster role name and static network setup and addition 3. Testing Cluster Group's network resource is being brought up (Cluster IP && Cluster Name) #> # return True only when useDefaultClusterRoleName is True AND *-ClusterAffinityRule command is present if (-not (Get-Command "*-ClusterAffinityRule" -ErrorAction SilentlyContinue)) { return $false } else { $cloudServiceIP = $global:config[$global:MocModule]["cloudServiceCidr"] return ($global:config[$global:MocModule]["useDefaultClusterRoleName"] -and [string]::IsNullOrEmpty($cloudServiceIP)) } } #region Logging and Monitoring #region Cleanup Functions function Install-Binaries { <# .DESCRIPTION Copies AksHci binaries to a node .PARAMETER nodeName The node to execute on. .PARAMETER module The module #> param ( [Parameter(Mandatory=$true)] [String]$nodeName, [Parameter(Mandatory=$true)] [String]$module, [Parameter(Mandatory=$true)] [Hashtable] $binariesMap ) Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_install_binaries , $module, $nodeName)) -moduleName $module Invoke-Command -ComputerName $nodeName -ScriptBlock { $path = $args[0] New-Item -ItemType Directory -Force -Path $path $envPath = [Environment]::GetEnvironmentVariable("PATH") if($envPath -notlike $("*$path*")) { [Environment]::SetEnvironmentVariable("PATH", "$path;$envPath") [Environment]::SetEnvironmentVariable("PATH", "$path;$envPath", "Machine") } } -ArgumentList $global:installDirectory | out-null $binariesMap.Keys | foreach-object { Copy-FileToRemoteNode -source $([io.Path]::Combine($global:config[$module]["installationPackageDir"], $_)) -remoteNode $nodeName -destination $binariesMap[$_] } } function Uninstall-Binaries { <# .DESCRIPTION Copies AksHci binaries to a node .PARAMETER nodeName The node to execute on. .PARAMETER module The module #> param ( [Parameter(Mandatory=$true)] [String]$nodeName, [Parameter(Mandatory=$true)] [String]$module, [Parameter(Mandatory=$true)] [Hashtable] $binariesMap ) Write-Status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_uninstall_binaries , $module, $nodeName)) -moduleName $module Invoke-Command -ComputerName $nodeName -ScriptBlock { $binaries = $args[0] $binaries.Keys | Foreach-Object { Remove-Item -Path $binaries[$_] -force -ErrorAction SilentlyContinue } } -ArgumentList $binariesMap | out-null } #endregion #region Catalog helpers function Get-Catalog { <# .DESCRIPTION Get the Catalog for AksHci. This would include a set of product release versions .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String]$moduleName ) $cacheFile = $global:config[$moduleName]["manifestCache"] if ((Test-Path $cacheFile)) { return (Get-Content $cacheFile | ConvertFrom-Json) } $provider = Get-DownloadProvider -module $moduleName $downloadParams = @{ Name = $global:config[$moduleName]["catalog"] Audience = $global:config[$moduleName]["ring"] Provider = $provider } if ($global:config[$moduleName]["useStagingShare"] -or ($global:config[$moduleName]["offlineDownload"] -and $global:config[$moduleName]["offsiteTransferCompleted"])) { $downloadParams.Add("Endpoint", $global:config[$moduleName]["stagingShare"]) } $catalog = Get-DownloadSdkCatalog @downloadParams $cacheFile = $global:config[$moduleName]["manifestCache"] $catalogJson = $catalog | ConvertTo-Json -depth 100 Set-Content -path $cacheFile -value $catalogJson -encoding UTF8 return $catalog } function Get-LatestCatalog { <# .DESCRIPTION Get the latest catalog for AksHci by clearing the cache and redownloading the latest .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) # Clean the catalog cache, so we download the latest Clear-CatalogCache -moduleName $moduleName return Get-Catalog -moduleName $moduleName } function Clear-CatalogCache { <# .DESCRIPTION Removes any cached copy of the catalog .PARAMETER moduleName Module name #> $cacheFile = $global:config[$moduleName]["manifestCache"] if ((Test-Path $cacheFile)) { Remove-Item $cacheFile -Force } # Sometimes this path wouldnt exist. Try to initialize it here $dirPath = [io.Path]::GetDirectoryName($cacheFile) if (!(Test-Path $dirPath)) { New-Item -ItemType Directory -force -Path $dirPath } } function Get-ProductRelease { <# .DESCRIPTION Gets the Product Release manifest for the specified Version .PARAMETER version The requested release version .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter(Mandatory=$true)] [String] $moduleName ) $catalog = Get-Catalog -moduleName $moduleName foreach($productRelease in $catalog.ProductStreamRefs[0].ProductReleases) { if ($productRelease.Version -ieq $version) { return $productRelease } } throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_release_not_found , $version))), ([ErrorTypes]::IsUserErrorFlag)) } function Get-ProductReleasesUptoVersion { <# .DESCRIPTION Get all of the Product Release Manifests up to the specified Version .PARAMETER version Requested version #> param ( [String]$version, [Parameter(Mandatory=$true)] [String]$moduleName ) # Assumption here is that the ordering of values in catalog release stream is latest at top. $releaseList = @() $catalog = Get-Catalog -moduleName $moduleName foreach($productRelease in $catalog.ProductStreamRefs[0].ProductReleases) { $releaseList += $productRelease if ($productRelease.Version -ieq $version) { break } } if ($releaseList.Count -eq 0) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_not_found , $version)), ([ErrorTypes]::IsErrorFlag)) } return $releaseList } function Get-LatestRelease { <# .DESCRIPTION Get the latest release of AksHci by refreshing the catalog and returning the latest release from the updated catalog .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $catalog = Get-LatestCatalog -moduleName $moduleName if (-not $catalog) { throw [CustomException]::new(($($GenericLocMessage.comm_no_catalog_retreive)), ([ErrorTypes]::IsUserErrorFlag)) } return $catalog.ProductStreamRefs[0].ProductReleases[0] } function Get-ReleaseDownloadParameters { <# .DESCRIPTION .PARAMETER name Release name to be downloaded .PARAMETER version Release version to be downloaded .PARAMETER destination The destination for the download .PARAMETER parts How many download parts to use (concurrency) .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $name, [Parameter(Mandatory=$true)] [String] $version, [Parameter()] [String] $destination, [Parameter(Mandatory=$false)] [Int] $parts = 1, [Parameter(Mandatory=$true)] [String] $moduleName ) $provider = Get-DownloadProvider -module $moduleName if(0 -eq $parts) { $parts = 1 } $downloadParams = @{ Provider = $provider Name = $name Version = $version Destination = $destination Parts = $parts } if ($global:config[$moduleName]["useStagingShare"] -or ($global:config[$moduleName]["offlineDownload"] -and $global:config[$moduleName]["offsiteTransferCompleted"])) { $downloadParams.Add("Endpoint", $global:config[$moduleName]["stagingShare"]) $downloadParams.Add("CatalogName", $global:config[$moduleName]["catalog"]) $downloadParams.Add("Audience", $global:config[$moduleName]["ring"]) } return $downloadParams } function Get-DownloadProvider { <# .DESCRIPTION Returns an appropriate download provider based on the current module configuration .PARAMETER moduleName Module name #> param ( [Parameter(Mandatory=$true)] [String]$moduleName ) if ($global:config[$moduleName]["useStagingShare"] -or ($global:config[$moduleName]["offlineDownload"] -and $global:config[$moduleName]["offsiteTransferCompleted"])) { $endpoint = $($global:config[$moduleName]["stagingShare"]) if ($endpoint.StartsWith("http")) { return "http" } elseif ($endpoint.StartsWith("//") -or $endpoint.StartsWith("\\") -or $endpoint.Contains(":")) { return "local" } else { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unsupported_endpoint , $endpoint)), ([ErrorTypes]::IsErrorFlag)) } } return "sfs" } #endregion #region File download helpers function Test-AuthenticodeBinaries { <# .DESCRIPTION Validates binary integrity via authenticode .PARAMETER workingDir Location of the binaries to be tested .PARAMETER binaries The list of binaries to be tested #> param ( [Parameter(Mandatory=$true)] [string] $workingDir, [Parameter(Mandatory=$true)] [string[]] $binaries ) Write-Status $($GenericLocMessage.comm_verify_binaries) -moduleName $global:DownloadModule $workingDir = $workingDir -replace "\/", "\" foreach ($binary in $binaries) { $name = $("$workingDir/$binary") -replace "\/", "\" Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_check_binary_sign , $name)) -moduleName $global:DownloadModule $auth = Get-AuthenticodeSignature -FilePath $name if (($global:expectedAuthResponse.status -ne $auth.status) -or ($global:expectedAuthResponse.SignatureType -ne $auth.SignatureType)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_authenticode_failed , "Binary "+$name, $($global:expectedAuthResponse.status), $($global:expectedAuthResponse.SignatureType), $($auth.status), $($auth.SignatureType))), ([ErrorTypes]::IsErrorFlag)) } Write-SubStatus $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_verify_sign , $name)) -moduleName $global:DownloadModule } } #endregion #region EventLog function New-ModuleEventLog { <# .DESCRIPTION Tests if the desired product/module is installed (or installing). Note that we consider some failed states (e.g. UninstallFailed) to represent that the product is still installed, albeit in a unknown/failed state. .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) New-EventLog -LogName "AKSHCI" -Source $moduleName -ErrorAction Ignore } function Write-ModuleEventException { <# .DESCRIPTION Write Exception message with stacktrace to event log .PARAMETER moduleName The module name to test for installation state .PARAMETER exception The message that has to be logged #> param ( [Parameter(Mandatory=$true)] [String] $moduleName, [Parameter(Mandatory=$true)] [String] $message, [Parameter(Mandatory=$true)] [Object] $exception ) $errorMessage = $message $errorMessage += "`r`nException [" + $exception.Exception.Message + "]" $errorMessage += "`r`nStacktrace [" + $($exception.ScriptStackTrace) + "]" if ($exception.Exception.InnerException) { $errorMessage += "`r`nInnerException[" + $exception.Exception.InnerException.Message + "]" } Write-ModuleEventLog -moduleName $moduleName -message $errorMessage -eventId 2 -entryType Warning Write-Warning $errorMessage return $errorMessage } function Write-ModuleEventLog { <# .DESCRIPTION Tests if the desired product/module is installed (or installing). Note that we consider some failed states (e.g. UninstallFailed) to represent that the product is still installed, albeit in a unknown/failed state. .PARAMETER moduleName The module name to test for installation state .PARAMETER message The message that has to be logged .PARAMETER entryType Activity name to use when writing progress .PARAMETER eventId Activity name to use when writing progress #> param ( [Parameter(Mandatory=$true)] [String] $moduleName, [Parameter(Mandatory=$true)] [String] $message, [Parameter(Mandatory=$true)] [System.Diagnostics.EventLogEntryType] $entryType, [Parameter(Mandatory=$true)] [int] $eventId ) $start = 0 $end = 32760 $tmpMessage = $message # Try to trim down the message if its too big if ($message.Length -gt $end) { $tmpMessage = $message.Substring($start, $end) } Write-EventLog -LogName "AKSHCI" -Source $moduleName -EventID $eventId -EntryType $entryType -Message $tmpMessage } #endregion #region Validation functions function Test-ValidCIDR { <# .DESCRIPTION This function validates that CIDR is valid by checking: 1. That the CIDR notation is correct (IP/prefix) 2. That the IP is valid. 3. That the prefix length is between 1 and 30 .PARAMETER CIDR The CIDR in the form IP/prefixlength. E.g. 10.0.0.0/24 #> param ( [Parameter(Mandatory=$true)] [string] $CIDR ) $x = $CIDR.Split('/') if ($x.Length -ne 2) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_cidr , $CIDR)) } $ip = $x[0] $prefix = [int]::Parse($x[1]) Test-ValidEndpoint -endpoint $ip # The minimum prefix length is /30 (which leaves 2 usable ip addresses. 1 for mgmt cluster VM, 1 for mgmt k8s VIP) if (($prefix -lt 1) -or ($prefix -gt 30)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_prefix_len , $prefix)) } return $true } function Test-ValidPool { <# .DESCRIPTION This function validates that the pool start/end are valid by checking: 1. That the pool start/end are valid ip addresses 2. That the pool end comes after or is equal (1 IP) to the pool start. 3. If CIDR is also given, it validates the range is within the CIDR. .PARAMETER PoolStart The starting ip address of the pool .PARAMETER PoolEnd The ending ip address of the pool .PARAMETER CIDR The CIDR from where the pool should come from. Note that if CIDR is given, it is expected to have already been validated. #> param ( [Parameter(Mandatory=$true)] [string] $PoolStart, [Parameter(Mandatory=$true)] [string] $PoolEnd, [string] $CIDR ) Test-ValidEndpoint -endpoint $PoolStart Test-ValidEndpoint -endpoint $PoolEnd $valid = [AKSHCI.IPUtilities]::ValidateRange($PoolStart, $PoolEnd) if (-not $valid) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ip_pool_range , $PoolStart, $PoolEnd)) } if ($CIDR) { $valid = [AKSHCI.IPUtilities]::ValidateRangeInCIDR($PoolStart, $PoolEnd, $CIDR) if (-not $valid) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_ip_pool_not_in_cidr , $PoolStart, $PoolEnd, $CIDR)) } } } function Test-VipPoolAgainstVnicAddressPrefix { <# .DESCRIPTION This functions validates that the pool belongs to the subnet by: 1. Finding the adapters attached to the switch (`$switchName`) 2. Filtering by internet facing adapters 3. Checking the subnets of the adapters against the pool .PARAMETER switchName The name of the switch .PARAMETER multiNode Whether Multi Node Configuration .PARAMETER PoolStart The starting ip address of the pool .PARAMETER PoolEnd The ending ip address of the pool .PARAMETER vlanID Vlan ID .PARAMETER nodeName The node to execute on. #> [cmdletbinding()] param( [parameter(Mandatory)] [String] $switchName, [Switch] $multiNode, [parameter(Mandatory)] [String] $PoolStart, [parameter(Mandatory)] [String] $PoolEnd, [Parameter(Mandatory=$false)] [int] $vlanID = $global:defaultVlanID, [parameter(Mandatory)] [String] $nodeName ) $isVlanSetup = $($vlanID -ne $global:defaultVlanID) $inputArgs = @( $nodeName, $switchName, $multiNode.IsPresent, $isVlanSetup, $vlanID ) try { $subnetRanges = Invoke-Command -ComputerName $nodeName { $hostName = $args[0] $switchName = $args[1] $isMultiNode = $args[2] $isVlanSetup = $args[3] $vlanID = $args[4] # validate if switch exists Write-Verbose $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_checking_for_virtual_switch_with_name, $switchName, $hostName)) $switch = $null # For Multi Node configuration the vSwitch needs to be external if ($isMultiNode) { $switch = Get-VMSwitch -Name $switchName -SwitchType External -ErrorAction SilentlyContinue if ($null -eq $switch) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMsg.comm_external_switch_missing, $switchName, $hostName)) } } else { $switch = Get-VMSwitch -Name $switchName -ErrorAction SilentlyContinue if ($null -eq $switch) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMsg.comm_switch_missing, $switchName, $hostName)) } } [string[]] $subnetRanges = @() [hashtable] $adapterDeviceIdToMac = @{} # Populate Adapter device id to MAC Address map (Get-VMNetworkAdapter -ManagementOS -SwitchName "$switchName" -ErrorAction SilentlyContinue) | ForEach-Object { $adapterDeviceIdToMac.Add($_.DeviceId, $_.MacAddress) } if ($adapterDeviceIdToMac.Count -eq 0) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMsg.comm_adapter_not_connected, $switchName, $hostName)) } $adapters = (Get-NetAdapter | Where-Object { $adapterDeviceIdToMac.Keys -contains $_.InterfaceGuid }) # Get internet facing adapters; only for multi node if ($isMultiNode) { $adapters = $adapters | Where-Object { $null -ne (Get-NetRoute "0.0.0.0/0" -InterfaceIndex $_.ifIndex -ErrorAction SilentlyContinue) # Route 0.0.0.0/0 is always pointing to the internet } if ($adapters.Count -eq 0) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMsg.comm_adapter_not_found, $hostName)) } } foreach ($adapter in $adapters) { if ($isVlanSetup) { # If on a different vlan; skip validation # TODO validate in case different vlan if (($null -eq $adapter.VlanID) -or ($adapter.VlanID -ne $vlanID)) { continue } } $adapterInstanceID = $adapter.InterfaceGuid $mac = $adapterDeviceIdToMac[$adapterInstanceID] $macAddressBytes = [System.Net.NetworkInformation.PhysicalAddress]::Parse($mac).GetAddressBytes() $mac = (($macAddressBytes|ForEach-Object ToString X2) -join ':') # docs : https://docs.microsoft.com/en-us/windows/win32/cimwin32prov/win32-networkadapterconfiguration $adapterConfig = (Get-WmiObject Win32_NetworkAdapterConfiguration | Where-Object IPEnabled | Where-Object MACAddress -eq $mac | Select-Object IPSubnet, IpAddress, DHCPEnabled) # For static IP configuration skip validation # TODO if ($adapterConfig.DHCPEnabled -eq $false) { continue } for($i = 0; $i -lt $adapterConfig.IpAddress.Count; $i++) { [IPAddress] $ip = $adapterConfig.IpAddress[$i] if ($ip.AddressFamily -ine "InterNetwork") # Skip if not IPv4 { continue } [IPAddress] $netmask = $adapterConfig.IPSubnet[$i] # convert mask to cidr $octets = $netmask.IPAddressToString.Split('.') $cidr=0 foreach($octet in $octets) { while(0 -ne $octet) { $octet = ($octet -shl 1) -band [byte]::MaxValue; $cidr++ } } $ipAddressBytes = $ip.GetAddressBytes() $netmaskAddressbytes = $netmask.GetAddressBytes() $startIP = (((0..3) | ForEach-Object { $ipAddressBytes[$_] -band $netmaskAddressbytes[$_] }) -join '.') $subnetRange = $startIP+'/'+$cidr $subnetRanges += $subnetRange } } return $subnetRanges } -ArgumentList $inputArgs }#end of try catch { throw [CustomException]::new($_, ([ErrorTypes]::IsInfraErrorFlag)) } foreach ($subnetRange in $subnetRanges) { if([AKSHCI.IPUtilities]::ValidateRangeInCIDR($PoolStart, $PoolEnd, $subnetRange)) # if any one range is valid, pass, return early { return } } if ($subnetRanges.Count -ne 0) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMsg.comm_poolstart_poolend_outside_subnet_range, $PoolStart, $PoolEnd, $switchName)), ([ErrorTypes]::IsInfraErrorFlag)) } } function Validate-IPV4Address { <# .DESCRIPTION returns true if the given ip address is in ipv4 format. False otherwise #> param( [parameter(Mandatory = $true)] [string] $ip ) if ($ip -notmatch $regexPatternIpv4){ return $false } return $true } function Test-ValidVersionNumber { <# .DESCRIPTION This function validates whether the given input version number is valid. currently it accepts one of the two types of version numbers: 1) v1.22.3.4.556 2) 1.2.3 .PARAMETER versionNumber The input version number #> param( [string] $versionNumber ) if ([string]::IsNullOrEmpty($versionNumber)){ return $true } if (-not ($versionNumber -cmatch $regexPatternVersionNumber)){ throw $([System.string]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_version_number, $versionNumber)) } return $true } function Test-ValidTaints { <# .DESCRIPTION This function validates the input taint array is valid by checking the given taint 1) is in ["NoSchedule", "PreferNoSchedule" ,"NoExecute"] 2) adheres to the format "key1=val1:TAINT1" .PARAMETER taints The list of taints #> param( [string[]] $taints ) if ([string]::IsNullOrEmpty($taints)){ return $true } foreach ($taint in $taints){ if (-not ($taint -cmatch $regexPatternTaintFormat)){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_taints)) } $taintElemArr = $taint.Split(":") $curTaint = $taintElemArr[-1] if (-not ($validTaints -contains $curTaint)){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_taints)) } } return $true } function Test-ValidCredSpecJsonPath { <# .DESCRIPTION This function validates the input json path is valid by checking 1) the actual json file exists .PARAMETER jsonPath json path of CredSpec #> param( [string] $jsonPath ) if ([string]::IsNullOrEmpty($jsonPath)){ return $true } if (-not (Test-Path $jsonPath)){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_json_file_path, $jsonPath)) } return $true } function Test-ValidMacPoolAddress { <# .DESCRIPTION This function validates whether the given mac address is valid .PARAMETER macpooladdress The mac address #> param( [string] $macPoolAddress ) if ([string]::IsNullOrEmpty($macPoolAddress)){ return $true } if (-not ($macPoolAddress -cmatch $regexPatternMacAddress)){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_mac_address, $macPoolAddress)) } return $true } function Test-ValidIpv4Address { <# .DESCRIPTION This function validates whether a given ipv4 address is valid. .PARAMETER ipv4 The ipv4 address #> param( [string] $ipv4 ) if ([string]::IsNullOrEmpty($ipv4)){ return $true } $isValidIpv4 = Validate-IPV4Address -ip $ipv4 if (-not $isValidIpv4){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ipv4_address, $ipv4)) } return $true } function Test-ValidDirectoryPath { <# .DESCRIPTION returns true if the input dirPath is valid. False otherwise .PARAMETER dirPath the directory path #> param( [string] $dirPath ) if ([string]::IsNullOrEmpty($dirPath)){ return $true } if (-not (Test-Path $dirPath -IsValid)){ throw $([System.string]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_file_path_exists, $dirPath)) } return $true } function Test-ValidDNSServers { <# .DESCRIPTION returns true if the input dns server array is valid. False otherwise .PARAMETER dnsServers #> param( [string[]] $dnsServers ) if ([string]::IsNullOrEmpty($dnsServers)){ return $true } foreach ($ip in $dnsServers){ if ([string]::IsNullOrEmpty($ip)){ return $true } if (-not (Validate-IPV4Address -ip $ip)){ throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_ipv4_address, $ip)) } } return $true } function Test-ValidIPPrefix { <# .DESCRIPTION returns true if the input is a valid ip prefix address. .PARAMETER ipprefix the ip prefix #> param( [string] $ipprefix ) if ([string]::IsNullOrEmpty($ipprefix)){ return $true } return Test-ValidCIDR -CIDR $ipprefix } #endregion #region Proxy server functions function Set-ProxyConfiguration { <# .DESCRIPTION Sets the proxy server configuration for a module .PARAMETER proxySettings Proxy server settings .PARAMETER moduleName The module name #> param ( [Parameter()] [ProxySettings] $proxySettings, [Parameter(Mandatory=$true)] [String] $moduleName ) $http = "" $https = "" $noProxy = "" $certFile = "" $user = "" $pass = "" if ($proxySettings) { $http = $proxySettings.HTTP $https = $proxySettings.HTTPS $noProxy = $proxySettings.NoProxy $certFile = $proxySettings.CertFile Test-ProxyConfiguration -http $http -https $https -noProxy $noProxy -certFile $certFile if ($proxySettings.Credential.Username) { $user = $proxySettings.Credential.UserName } if ($proxySettings.Credential.Password) { $pass = $proxySettings.Credential.Password | ConvertFrom-SecureString -Key $global:credentialKey } } Set-ConfigurationValue -name "proxyServerUsername" -value $user -module $moduleName Set-ConfigurationValue -name "proxyServerPassword" -value $pass -module $moduleName Set-ConfigurationValue -name "proxyServerHTTP" -value $http -module $moduleName Set-ConfigurationValue -name "proxyServerHTTPS" -value $https -module $moduleName Set-ConfigurationValue -name "proxyServerNoProxy" -value $noProxy -module $moduleName Set-ConfigurationValue -name "proxyServerCertFile" -value $certFile -module $moduleName Initialize-ProxyEnvironment -moduleName $moduleName } function Initialize-ProxyEnvironment { <# .DESCRIPTION Applies proxy settings to the current process environment .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $proxySettings = Get-ProxyConfiguration -moduleName $moduleName if ($proxySettings.HTTP -or $proxySettings.HTTPS) { Set-DownloadSdkProxy -Http "$($proxySettings.HTTP)" -Https "$($proxySettings.HTTPS)" -NoProxy "$($proxySettings.NoProxy)" } } function Get-ProxyConfiguration { <# .DESCRIPTION Returns a custom PSObject containing the complete HTTP, HTTPS, NoProxy, and CertFile setting strings .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$true)] [String] $moduleName ) $proxyHTTP = "" $proxyHTTPS = "" $proxyCertName = "" $proxyCertContentB64 = "" if ($global:config[$moduleName]["proxyServerHTTP"]) { $proxyHTTP = Get-ProxyWithCredentials -proxyServer $($global:config[$moduleName]["proxyServerHTTP"]) -proxyUsername $($global:config[$moduleName]["proxyServerUsername"]) -proxyPass $($global:config[$moduleName]["ProxyServerPassword"]) } if ($global:config[$moduleName]["proxyServerHTTPS"]) { $proxyHTTPS = Get-ProxyWithCredentials -proxyServer $($global:config[$moduleName]["proxyServerHTTPS"]) -proxyUsername $($global:config[$moduleName]["proxyServerUsername"]) -proxyPass $($global:config[$moduleName]["ProxyServerPassword"]) } if (($global:config[$moduleName]["proxyServerCertFile"]) -and (Test-Path $global:config[$moduleName]["proxyServerCertFile"])) { $content = Get-Content -Encoding Byte -Path $global:config[$moduleName]["proxyServerCertFile"] if ($content) { $proxyCertName = "proxy-cert.crt" $proxyCertContentB64 = [Convert]::ToBase64String($content) } } $proxyServer = "" $isProxyConfigured = $false if (-not [String]::IsNullOrWhiteSpace($proxySettings.http)) { $proxyServer = $proxySettings.HTTP $isProxyConfigured = $true } if (-not [String]::IsNullOrWhiteSpace($proxySettings.https)) { $proxyServer = $proxySettings.HTTPS $isProxyConfigured = $true } $proxyConfig = [ordered]@{ 'HTTP' = $proxyHTTP; 'HTTPS' = $proxyHTTPS; 'NoProxy' = $global:config[$moduleName]["proxyServerNoProxy"]; 'CertPath' = $global:config[$moduleName]["proxyServerCertFile"]; 'CertName' = $proxyCertName; 'CertContent' = $proxyCertContentB64 'IsProxyConfigured'= $isProxyConfigured 'ProxyServer' = $proxyServer } if ($($global:config[$moduleName]["proxyServerUsername"]) -and $($global:config[$moduleName]["ProxyServerPassword"])) { $securePass = $($global:config[$moduleName]["ProxyServerPassword"]) | ConvertTo-SecureString -Key $global:credentialKey $credential = New-Object System.Management.Automation.PSCredential -ArgumentList $($global:config[$moduleName]["proxyServerUsername"]), $securePass $proxyConfig.Add("Credential", $credential) } $result = @() $result += New-Object -TypeName PsObject -Property $proxyConfig return $result } function Get-ProxyWithCredentials { <# .DESCRIPTION Returns a complete proxy string with credentials in the URI format (e.g. http://user:pass@server.com:8080) .PARAMETER proxyServer Proxy server string URI .PARAMETER proxyUsername Proxy server username .PARAMETER proxyPass Proxy server password (this is a secure string representation, not plaintext) #> param ( [Parameter(Mandatory=$true)] [String] $proxyServer, [Parameter()] [String] $proxyUsername, [Parameter()] [String] $proxyPass ) $uri = Test-ProxyServerValue -proxyServer $proxyServer $proxyString = $($uri.Scheme + "://") if ($proxyUsername -and $proxyPass) { $securePass = $proxyPass | ConvertTo-SecureString -Key $global:credentialKey $credential = New-Object System.Management.Automation.PSCredential -ArgumentList $proxyUsername, $securePass $proxyString += $($credential.UserName + ":" + $credential.GetNetworkCredential().Password + "@") } $proxyString += $uri.Authority if ($uri.IsDefaultPort) { $proxyString += (":" + $uri.Port) } return $proxyString } function Test-ProxyConfiguration { <# .DESCRIPTION Validates the provided proxy server configuration. On failure, would throw. .PARAMETER http HTTP proxy server configuration .PARAMETER https HTTPS proxy server configuration .PARAMETER noProxy Proxy server exemption/bypass list .PARAMETER certFile Path to a CA certificate file used to establish trust with a HTTPS proxy server #> param ( [Parameter()] [String] $http, [Parameter()] [String] $https, [Parameter()] [String] $noProxy, [Parameter()] [String] $certFile ) if ($http) { Test-ProxyServerValue -proxyServer $http | Out-Null } if ($https) { Test-ProxyServerValue -proxyServer $https | Out-Null } $http_proxy_env = [System.Environment]::GetEnvironmentVariable("HTTP_PROXY", "Machine") $https_proxy_env = [System.Environment]::GetEnvironmentVariable("HTTPS_PROXY", "Machine") $no_proxy_env = [System.Environment]::GetEnvironmentVariable("NO_PROXY", "Machine") if ($http_proxy_env -or $https_proxy_env) { if (-not $no_proxy_env) { throw [CustomException]::new(($($GenericLocMessage.comm_missing_NoProxy_configuration )), ([ErrorTypes]::IsInfraErrorFlag)) } } if ($noProxy -and $noProxy.contains("*")) { throw [CustomException]::new(($($GenericLocMessage.comm_NoProxy_list_invalid_wildcard )), ([ErrorTypes]::IsUserErrorFlag)) } if ($noProxy -and $noProxy.contains(";")) { throw [CustomException]::new(($($GenericLocMessage.comm_NoProxy_input_list_invalid_delimiter)), ([ErrorTypes]::IsUserErrorFlag)) } if ($certFile) { if (-not (Test-Path $certFile)) { throw [CustomException]::new(($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_proxy_cert_not_found , $certFile))), ([ErrorTypes]::IsUserErrorFlag)) } } } function Test-ProxyServerValue { <# .DESCRIPTION Validates the provided proxy server string .PARAMETER proxyServer Proxy server string in absolute URI format (e.g. http://proxy.com:3128) #> param ( [String] $proxyServer ) $uri = $null $result = [System.URI]::TryCreate($proxyServer, [System.UriKind]::Absolute, [ref]$uri) if (-not $result) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_proxy_invalid_uri , $proxyServer)) } switch($uri.Scheme) { $([System.URI]::UriSchemeHttp) { break } $([System.URI]::UriSchemeHttps) { break } Default { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_proxy_invalid_uri , $proxyServer)) } } $proxyConnectionResult = test-netconnection $uri.Host -port $uri.Port if (-not $proxyConnectionResult.TcpTestSucceeded) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_proxy_connection_failure , $proxyServer)) } return $uri } function Test-ValidAutoScalerProfileConfig { <# .DESCRIPTION Validates the keys provided in a AutoScalerProfile config hashtable. .PARAMETER AutoScalerProfileConfig An AutoScalerProfile config #> param ( [hashtable] $AutoScalerProfileConfig ) if ($AutoScalerProfileConfig.Count -lt 1) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_empty_autoscalerprofile_config)) } foreach($k in $AutoScalerProfileConfig.Keys) { if (-not $global:autoScalerProfileConfigToKvaYamlKeys.ContainsKey($k)) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $KvaLocMessage.kva_invalid_autoscalerprofile_config_key, $k)) } } return $true } function New-VipPoolSettings { <# .DESCRIPTION A wrapper around [New-VipPoolSettings]::new that Validates parameters before returning a New VipPoolSettings object .PARAMETER name The name of the vip pool .PARAMETER vipPoolStart The starting ip address to use for the vip pool. .PARAMETER vippoolend The ending ip address to use for the vip pool. .OUTPUTS VipPoolSettings object .EXAMPLE #> param ( [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [string] $name, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [String] $vipPoolStart, [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] [String] $vipPoolEnd ) Test-ValidPool -PoolStart $vipPoolStart -PoolEnd $vipPoolEnd return [VipPoolSettings]::new($name, $vipPoolStart, $vipPoolEnd) } #endregion # function Get-ImageReleaseManifest { <# .DESCRIPTION Discovers the requested image and returns the release manifest for it .PARAMETER imageVersion Image release version to target .PARAMETER operatingSystem Image operating system to target .PARAMETER osSku SKU of the image: CBLMariner, Windows2019 or Windows2022 .PARAMETER k8sVersion Kubernetes version to target .PARAMETER moduleName The module name #> param( [parameter(Mandatory=$true)] [String] $imageVersion, [parameter(Mandatory=$true)] [String] $operatingSystem, [String] $osSku, [Parameter(Mandatory=$true)] [String] $k8sVersion, [Parameter(Mandatory=$true)] [String] $moduleName ) if ([string]::IsNullOrEmpty($osSku)) { switch ($operatingSystem) { "Windows" { $osSku = [OsSku]::Windows2019 } "Linux" { $osSku = [OsSku]::CBLMariner } } } $k8sVersion = $k8sVersion.TrimStart("v") $productRelease = Get-ProductRelease -version $imageVersion -moduleName $moduleName foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { $vhdInfo = Get-ImageReleaseVhdInfo -release $subProductRelease if (-not $vhdInfo) { continue } if ($vhdInfo.CustomData.BaseOSImage.OperatingSystem -ine $operatingSystem) { continue } if (-not [string]::IsNullOrEmpty($vhdInfo.CustomData.BaseOSImage.SKU) -and ($vhdInfo.CustomData.BaseOSImage.SKU -ine $osSku)) { continue } foreach($pkg in $vhdInfo.CustomData.K8SPackages) { if ($pkg.Version -ieq $k8sVersion) { return $subProductRelease } } } } throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_image_not_found, $imageVersion, $operatingSystem, $k8sVersion)), ([ErrorTypes]::IsErrorFlag)) } function Get-ImageReleaseVhdInfo { <# .DESCRIPTION Discovers and returns vhd information for the specified image release .PARAMETER imageRelease Image release manifest #> param( [parameter(Mandatory=$True)] [PSCustomObject] $release ) foreach ($fileRelease in $release.ProductFiles) { if ($fileRelease.CustomData.Type -ieq "vhd") { return $fileRelease } } return $null } function Get-ImageRelease { <# .DESCRIPTION Download the specified image release. .PARAMETER imageRelease Image release manifest .PARAMETER imageDir Directory for local image store. .PARAMETER moduleName The module name .PARAMETER releaseVersion Version of the release .PARAMETER imageGalleryName Name to use for image #> param( [parameter(Mandatory=$True)] [PSCustomObject] $imageRelease, [parameter(Mandatory=$True)] [String] $imageDir, [parameter(Mandatory=$True)] [String] $moduleName, [Parameter(Mandatory=$True)] [String] $releaseVersion, [Parameter(Mandatory=$false)] [String] $imageGalleryName ) $downloadpath = "$imageDir\$([System.IO.Path]::GetRandomFileName().Split('.')[0])" New-Item -ItemType Directory -Force -Confirm:$false -Path $downloadpath | Out-Null try { $vhdInfo = Get-ImageReleaseVhdInfo -release $imageRelease if ([string]::IsNullOrEmpty($imageGalleryName)) { $k8sVersion = $vhdInfo.CustomData.K8SPackages[0].Version $imageGalleryName = Get-KubernetesGalleryImageName -imagetype $vhdInfo.CustomData.BaseOSImage.OperatingSystem -osSku $vhdInfo.CustomData.BaseOSImage.SKU -k8sVersion $k8sVersion -releaseVersion $releaseVersion } $imageVersionCurrent = $imageRelease.Version $imageVersionManifestPath = "$ImageDir\$imageGalleryName.json" $destinationpath = $("$ImageDir\$imageGalleryName.vhdx" -replace "\/", "\") if (Test-Path $imageVersionManifestPath) { $OldImageData = get-content $imageVersionManifestPath | ConvertFrom-Json $OldImageVersion = $OldImageData.Version Write-SubStatus -moduleName $moduleName "Existing image $imageVersionManifestPath has version $OldImageVersion . Requested version is $imageVersionCurrent" if ($imageVersionCurrent -ieq $OldImageVersion) { if (Test-Path -Path $destinationpath) { Write-SubStatus -moduleName $moduleName $($GenericLocMessage.comm_existing_image_upto_date) return $destinationpath } Write-SubStatus -moduleName $moduleName $($GenericLocMessage.comm_existing_image_not_present) } } Write-Status -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_download_image_with_version, $($imageRelease.ProductStream), $imageVersionCurrent, $downloadPath)) $imageConcurrentDownloads = $global:config[$modulename]["concurrentDownloads"] if (($global:config[$moduleName]["useStagingShare"] -or ($global:config[$moduleName]["offlineDownload"] -and $global:config[$moduleName]["offsiteTransferCompleted"])) -and -not [string]::IsNullOrEmpty($releaseVersion)) { $imageVersionCurrent = $releaseVersion } $downloadParams = Get-ReleaseDownloadParameters -name $imageRelease.ProductStream -version $imageVersionCurrent -destination $downloadPath -parts $imageConcurrentDownloads -moduleName $moduleName $releaseInfo = Get-DownloadSdkRelease @downloadParams if ($global:config[$moduleName]["useStagingShare"]) { $imageFile = $releaseInfo.Files[0] -replace "\/", "\" } elseif ($global:config[$moduleName]["offlineDownload"] -and -not $global:config[$moduleName]["offsiteTransferCompleted"]) { return $destinationpath } else { $imageFile = Expand-SfsImage -files $releaseInfo.Files -destination $downloadPath -workingDirectory $downloadPath -moduleName $moduleName } $imageJson = $imageRelease | ConvertTo-Json -depth 100 Set-Content -path $imageversionManifestPath -value $imageJson -encoding UTF8 -Confirm:$false if (test-path $destinationpath) { Remove-Item $destinationpath -Force -Confirm:$false } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_moving_image, $imageFile, $destinationpath)) Move-Item -Path $imageFile -Destination $destinationpath -Confirm:$false } finally { Remove-Item -Force $downloadpath -Recurse -Confirm:$false } return $destinationpath } function Expand-SfsImage { <# .DESCRIPTION Expand and verify a SFS image download using authenticode. NOTE: This is temporary until cross-platform signing is available in Download SDK. .PARAMETER files The downloaded image files (expected to be a image file zip and a companion cab). .PARAMETER destination Destination for the expanded image file. .PARAMETER workingDirectory Working directory to use for zip and cab file expansion. .PARAMETER moduleName The module name #> param( [Parameter(Mandatory=$True)] [String[]] $files, [Parameter(Mandatory=$True)] [String] $destination, [Parameter(Mandatory=$True)] [String] $workingDirectory, [parameter(Mandatory=$True)] [String] $moduleName ) Write-Status -moduleName $moduleName $($GenericLocMessage.comm_verifying_image_companion_file_download) [String[]]$cabfile = $files | Where-Object { $_ -match "\.cab$"} if (($null -eq $cabfile) -or ($cabfile.count -ne 1)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_wrong_cab_file_count, $cabfile.count)), ([ErrorTypes]::IsErrorFlag)) } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_verify_authenticode_signature, $cabFile)) $auth = Get-AuthenticodeSignature -filepath $cabfile if (($global:expectedAuthResponse.status -ne $auth.status) -or ($global:expectedAuthResponse.SignatureType -ne $auth.SignatureType)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_authenticode_failed, "KVA image companion file", $global:expectedAuthResponse.status, $global:expectedAuthResponse.SignatureType, $auth.status, $auth.SignatureType)), ([ErrorTypes]::IsErrorFlag)) } Write-Status -moduleName $moduleName $($GenericLocMessage.comm_expanding_image_companion_file) $expandDir = "$WorkingDirectory\expand_" + [System.IO.Path]::GetRandomFileName().Split('.')[0] New-Item -ItemType Directory -Force -Confirm:$false -Path $expandDir | Out-Null $expandoutput = expand.exe $cabfile $expandDir Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_expand_output, $expandoutput)) $manifest = Get-ChildItem $expandDir | select-object -first 1 $manifestcontents = get-content $manifest.fullname | convertfrom-json $packageAlgo = $manifestcontents.PackageVerification.VerificationDescriptor.Algorithm $packageHash = $manifestcontents.PackageVerification.VerificationDescriptor.FileHash $packageName = $manifestcontents.PackageVerification.VerificationDescriptor.Filename Write-Status -moduleName $moduleName $($GenericLocMessage.comm_verifying_image_file_download) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_verification_request, $packageName, $packageAlgo, $packageHash)) [string[]]$imagezip = $files | Where-Object { $_ -match "$packageName$"} if ($null -eq $imagezip) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unable_to_locate_image_file, $packageName)), ([ErrorTypes]::IsErrorFlag)) } Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_calculating_hash_for_archive, $packageAlgo, $imagezip[0])) $hash = Get-Base64Hash -file $imagezip[0] -algorithm $packageAlgo -moduleName $moduleName if ($packageHash -ne $hash) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unexpected_hash, $moduleName, $packageHash, $($imagezip[0]), $hash)), ([ErrorTypes]::IsErrorFlag)) } Write-Status -moduleName $moduleName $($GenericLocMessage.comm_expanding_image_file_archive) $contentsDir = "$expandDir\contents" New-Item -ItemType Directory -Force -Confirm:$false -Path $contentsdir | Out-Null Expand-Archive -path $imagezip[0] -destinationpath $contentsDir -Confirm:$false | Out-Null if (-not $global:config[$moduleName]["offlineDownload"]) { Remove-Item $imagezip[0] -Confirm:$false } [System.IO.FileInfo[]]$workimage = Get-ChildItem -r $contentsDir | Where-Object { (-not $_.psiscontainer) -and ($_.name -match "\.vhdx$")} if ($workimage.count -ne 1) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_invalid_file_count_expansion, $($workimage.count))), ([ErrorTypes]::IsErrorFlag)) } $content0 = $manifestcontents.ContentVerification[0].VerificationDescriptor Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_calculating_hash_for_file, $packageAlgo, $workimage[0].fullname)) $hash = Get-Base64Hash -file $workimage[0].fullname -algorithm $content0.algorithm -moduleName $moduleName if ($content0.FileHash -ne $hash) { throw $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_unexpected_hash, $moduleName, $($content0.FileHash), $($workimage[0].fullname), $hash)) } $image = $("$destination\" + $workimage[0].name) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_moving_image_file, $image)) Move-item -Path $workimage[0].fullname -Destination $image -Confirm:$false return $image } function Get-Base64Hash { <# .DESCRIPTION Obtain the base64 byte hash of the specified file. Used to verify SFS binaries .PARAMETER file File to generate the base64 hash for .PARAMETER algorithm Hashing algorithm to use .PARAMETER moduleName The module name #> param ( [Parameter(Mandatory=$True)] [string] $file, [Parameter(Mandatory=$True)] [string] $algorithm, [parameter(Mandatory=$True)] [String] $moduleName ) Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_generating_base64_hash, $file, $algorithm)) $diskhash_hex = Get-FileHash -algo $algorithm -path $file [byte[]] $diskhash_bin = for ([int]$i = 0; $i -lt $diskhash_hex.hash.length; $i += 2) { [byte]::parse($diskhash_hex.hash.substring($i, 2), [System.Globalization.NumberStyles]::HexNumber) } return [convert]::ToBase64String($diskhash_bin) } function Get-AvailableKubernetesVersions { <# .DESCRIPTION Returns the kubernetes versions (by OS) that are supported by the specified AksHci release .PARAMETER akshciVersion AksHci Release version. Defaults to the version of the current deployment .PARAMETER moduleName The module name #> param ( [Parameter()] [String] $akshciVersion, [Parameter(Mandatory=$true)] [String] $moduleName ) $result = @() if (-not $akshciVersion) { $akshciVersion = Get-AksHciVersion } # Get the Manifest for the specified Version $productRelease = Get-ProductRelease -version $akshciVersion -module $moduleName foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { foreach ($fileRelease in $subProductRelease.ProductFiles) { if (-not $fileRelease.CustomData.K8sPackages) { continue } $fileRelease.CustomData.K8SPackages | ForEach-Object { $version = [ordered]@{ 'OrchestratorType' = "Kubernetes"; 'OrchestratorVersion' = $("v"+$_.Version); 'OS' = $fileRelease.CustomData.BaseOSImage.OperatingSystem; 'SKU' = $fileRelease.CustomData.BaseOSImage.SKU; 'IsPreview' = $false } $result += New-Object -TypeName PsObject -Property $version } } } } return $result } function Get-LatestKubernetesVersion { <# .DESCRIPTION Get the latest Kubernetes version for given AksHci version. OS is limited to Linux for now as New-KvaClusterInternal set the controlplane os to be Linux .PARAMETER akshciVersion AksHci Release version. Defaults to the version of the current deployment .PARAMETER moduleName The module name #> [CmdletBinding()] param ( [String]$akshciVersion, [Parameter(Mandatory=$true)] [String] $moduleName ) if (-not $akshciVersion) { $akshciVersion = Get-AksHciVersion } $k8sVersions = Get-AvailableKubernetesVersions -akshciVersion $akshciVersion -moduleName $moduleName | Where-Object -Property OS -eq Linux $latestVersion = $k8sVersions[0].OrchestratorVersion foreach ($k8sVersion in $k8sVersions) { if ([Version]$k8sVersion.OrchestratorVersion.TrimStart("v") -ge [Version]$latestVersion.TrimStart("v")) { $latestVersion = $k8sVersion.OrchestratorVersion } } return $latestVersion } function Get-EarliestKubernetesVersion { <# .DESCRIPTION Get the earliest Kubernetes version for given AksHci version. OS is limited to Linux for now as New-KvaClusterInternal set the controlplane os to be Linux .PARAMETER akshciVersion AksHci Release version. Defaults to the version of the current deployment .PARAMETER moduleName The module name #> [CmdletBinding()] param ( [String]$akshciVersion, [Parameter(Mandatory=$true)] [String] $moduleName ) if (-not $akshciVersion) { $akshciVersion = Get-AksHciVersion } $k8sVersions = Get-AvailableKubernetesVersions -akshciVersion $akshciVersion -moduleName $moduleName | Where-Object -Property OS -eq Linux $latestVersion = $k8sVersions[0].OrchestratorVersion foreach ($k8sVersion in $k8sVersions) { if ([Version]$k8sVersion.OrchestratorVersion.TrimStart("v") -le [Version]$latestVersion.TrimStart("v")) { $latestVersion = $k8sVersion.OrchestratorVersion } } return $latestVersion } function Get-ReleaseContent { <# .DESCRIPTION Download all required files and packages for the specified release .PARAMETER version Release version .PARAMETER destination Destination directory for the content .PARAMETER activity Activity name to use when writing progress .PARAMETER moduleName The module name .PARAMETER mode Different modes for choosing different Linux kubernetes versions to download. #> param ( [Parameter(Mandatory=$true)] [String] $version, [Parameter(Mandatory=$true)] [string] $destination, [Parameter()] [String] $activity = $MyInvocation.MyCommand.Name, [Parameter(Mandatory=$true)] [string] $moduleName, [Parameter()] [OfflineDownloadMode] $mode = "full" ) $akshci = $false $destination = $destination -replace "\/", "\" Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_discovering_release_content, $moduleName)) $productRelease = Get-ProductRelease -version $version -moduleName $moduleName # find requested release foreach($releaseStream in $productRelease.ProductStreamRefs) { foreach($subProductRelease in $releaseStream.ProductReleases) { if ($subProductRelease.ProductName -ieq "client-cred-plugin" -and -not ($global:config[$modulename]["offlineDownload"] -and -not $global:config[$moduleName]["offsiteTransferCompleted"])) { continue } $vhdInfo = Get-ImageReleaseVhdInfo -release $subProductRelease if (-not $vhdInfo -or ($global:config[$modulename]["offlineDownload"] -and -not $global:config[$moduleName]["offsiteTransferCompleted"])) { $akshci = $true $name = $subProductRelease.ProductName.Substring(0, 3) $versionManifestPath = [io.Path]::Combine($destination, $("$name-release.json")) Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_downloading_release_content_to, $name, $destination)) $downloadVersion = $subProductRelease.Version if (($global:config[$moduleName]["useStagingShare"]) -or ($global:config[$modulename]["offlineDownload"] -and $global:config[$moduleName]["offsiteTransferCompleted"])) { $downloadVersion = $version } if ($vhdInfo) { if ($vhdInfo.CustomData.BaseOSImage.OperatingSystem -ieq "Linux") { $k8sVersion = $subProductRelease.ProductFiles.CustomData.K8sPackages.Version if ($mode -ieq "minimum" -and $k8sVersion -ne $productRelease.CustomData.ManagementNodeImageK8sVersion) { continue } } else { if ($mode -ieq "minimum") { continue } } } $downloadParams = Get-ReleaseDownloadParameters -name $subProductRelease.ProductStream -version $downloadVersion -destination $destination -parts 3 -moduleName $moduleName $releaseInfo = Get-DownloadSdkRelease @downloadParams if (-not ($global:config[$moduleName]["useStagingShare"]) -and -not ($global:config[$moduleName]["offlineDownload"] -and -not $global:config[$moduleName]["offsiteTransferCompleted"])) { if ($releaseInfo.Files.Count -ne 1) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_wrong_release_files_count, $moduleName, $releaseInfo.Files.Count)), ([ErrorTypes]::IsErrorFlag)) } $packagename = $releaseInfo.Files[0] -replace "\/", "\" # Temporary until cross-platform signing is available $auth = Get-AuthenticodeSignature -filepath $packagename if (($global:expectedAuthResponse.status -ne $auth.status) -or ($auth.SignatureType -ne $global:expectedAuthResponse.SignatureType)) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_authenticode_failed, $global:expectedAuthResponse.status, $global:expectedAuthResponse.SignatureType, $auth.status, $auth.SignatureType)), ([ErrorTypes]::IsErrorFlag)) } Write-StatusWithProgress -activity $activity -moduleName $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_expanding_package, $moduleName, $packagename, $destination)) $expandoutput = expand.exe -r $packagename $destination -f:* Write-SubStatus -moduleName $moduleName $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.generic_expand_result, $expandoutput)) } $versionJson = $subProductRelease | ConvertTo-Json -depth 100 set-content -path $versionManifestPath -value $versionJson -encoding UTF8 } else { if ($vhdInfo.CustomData.BaseOSImage.OperatingSystem -ieq "Linux") { $k8sVersion = $subProductRelease.ProductFiles.CustomData.K8sPackages.Version } else { $k8sVersion = $productRelease.CustomData.ManagementNodeImageK8sVersion } Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_obtain_download_information_for_image, $subProductRelease.ProductName)) $imageRelease = Get-ImageReleaseManifest -imageVersion $version -operatingSystem $vhdInfo.CustomData.BaseOSImage.OperatingSystem -osSku $vhdInfo.CustomData.BaseOSImage.SKU -k8sVersion $k8sVersion -moduleName $moduleName Write-Output $subProductRelease.ProductName if ($global:config[$moduleName]["offlineDownload"] -and $global:config[$moduleName]["offsiteTransferCompleted"]) { Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_using_offline_download_image, $subProductRelease.ProductName)) } else { Write-StatusWithProgress -activity $activity -module $moduleName -status $([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_downloading_image, $subProductRelease.ProductName)) } Get-ImageRelease -imageRelease $imageRelease -imageDir $destination -moduleName $moduleName -releaseVersion $version } } } if(-not $akshci) { throw [CustomException]::new($([System.String]::Format([System.Globalization.CultureInfo]::InvariantCulture, $GenericLocMessage.comm_no_release_content, $version)), ([ErrorTypes]::IsErrorFlag)) } return } #endregion #region Multi-Admin common functions function Get-UserSSHPublicKey { <# .DESCRIPTION Get the SSH Public Key that is set at the user scope to be used by the deployment #> $version = Get-AksHciVersion if ([version]$version -lt [version]$global:multiAdminRelease){ return Get-SshPublicKey } $workingDir = $global:config[$global:MocModule]["workingDir"] if(!(Test-Path -Path "$env:USERPROFILE\.ssh")){ New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\.ssh" | Out-Null } Copy-Item -Path "$workingDir\.ssh\akshci_rsa.pub" -Destination $($env:USERPROFILE + "\.ssh") Copy-Item -Path "$workingDir\.ssh\akshci_rsa" -Destination $($env:USERPROFILE + "\.ssh") return "$env:USERPROFILE\.ssh\akshci_rsa.pub" } function Get-UserSSHPrivateKey { <# .DESCRIPTION Get the SSH Private Key that is set at the user scope to be used by the deployment #> $version = Get-AksHciVersion if ([version]$version -lt [version]$global:multiAdminRelease){ return Get-SshPrivateKey } $workingDir = $global:config[$global:MocModule]["workingDir"] if(!(Test-Path -Path "$env:USERPROFILE\.ssh")){ New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\.ssh" | Out-Null } Copy-Item -Path "$workingDir\.ssh\akshci_rsa.pub" -Destination $($env:USERPROFILE + "\.ssh") Copy-Item -Path "$workingDir\.ssh\akshci_rsa" -Destination $($env:USERPROFILE + "\.ssh") return "$env:USERPROFILE\.ssh\akshci_rsa" } #endregion # SIG # Begin signature block # MIIn0QYJKoZIhvcNAQcCoIInwjCCJ74CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCDts8AcnT+yvCV/ # GxYK3bmPoJCK3jnp7lT0lM//fDPAK6CCDYUwggYDMIID66ADAgECAhMzAAADTU6R # phoosHiPAAAAAANNMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI4WhcNMjQwMzE0MTg0MzI4WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDUKPcKGVa6cboGQU03ONbUKyl4WpH6Q2Xo9cP3RhXTOa6C6THltd2RfnjlUQG+ # Mwoy93iGmGKEMF/jyO2XdiwMP427j90C/PMY/d5vY31sx+udtbif7GCJ7jJ1vLzd # j28zV4r0FGG6yEv+tUNelTIsFmmSb0FUiJtU4r5sfCThvg8dI/F9Hh6xMZoVti+k # bVla+hlG8bf4s00VTw4uAZhjGTFCYFRytKJ3/mteg2qnwvHDOgV7QSdV5dWdd0+x # zcuG0qgd3oCCAjH8ZmjmowkHUe4dUmbcZfXsgWlOfc6DG7JS+DeJak1DvabamYqH # g1AUeZ0+skpkwrKwXTFwBRltAgMBAAGjggGCMIIBfjAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUId2Img2Sp05U6XI04jli2KohL+8w # VAYDVR0RBE0wS6RJMEcxLTArBgNVBAsTJE1pY3Jvc29mdCBJcmVsYW5kIE9wZXJh # dGlvbnMgTGltaXRlZDEWMBQGA1UEBRMNMjMwMDEyKzUwMDUxNzAfBgNVHSMEGDAW # gBRIbmTlUAXTgqoXNzcitW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8v # d3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIw # MTEtMDctMDguY3JsMGEGCCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDEx # XzIwMTEtMDctMDguY3J0MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIB # ACMET8WuzLrDwexuTUZe9v2xrW8WGUPRQVmyJ1b/BzKYBZ5aU4Qvh5LzZe9jOExD # YUlKb/Y73lqIIfUcEO/6W3b+7t1P9m9M1xPrZv5cfnSCguooPDq4rQe/iCdNDwHT # 6XYW6yetxTJMOo4tUDbSS0YiZr7Mab2wkjgNFa0jRFheS9daTS1oJ/z5bNlGinxq # 2v8azSP/GcH/t8eTrHQfcax3WbPELoGHIbryrSUaOCphsnCNUqUN5FbEMlat5MuY # 94rGMJnq1IEd6S8ngK6C8E9SWpGEO3NDa0NlAViorpGfI0NYIbdynyOB846aWAjN # fgThIcdzdWFvAl/6ktWXLETn8u/lYQyWGmul3yz+w06puIPD9p4KPiWBkCesKDHv # XLrT3BbLZ8dKqSOV8DtzLFAfc9qAsNiG8EoathluJBsbyFbpebadKlErFidAX8KE # usk8htHqiSkNxydamL/tKfx3V/vDAoQE59ysv4r3pE+zdyfMairvkFNNw7cPn1kH # Gcww9dFSY2QwAxhMzmoM0G+M+YvBnBu5wjfxNrMRilRbxM6Cj9hKFh0YTwba6M7z # ntHHpX3d+nabjFm/TnMRROOgIXJzYbzKKaO2g1kWeyG2QtvIR147zlrbQD4X10Ab # rRg9CpwW7xYxywezj+iNAc+QmFzR94dzJkEPUSCJPsTFMIIHejCCBWKgAwIBAgIK # YQ6Q0gAAAAAAAzANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlm # aWNhdGUgQXV0aG9yaXR5IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEw # OTA5WjB+MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UE # BxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYD # VQQDEx9NaWNyb3NvZnQgQ29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG # 9w0BAQEFAAOCAg8AMIICCgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+la # UKq4BjgaBEm6f8MMHt03a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc # 6Whe0t+bU7IKLMOv2akrrnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4D # dato88tt8zpcoRb0RrrgOGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+ # lD3v++MrWhAfTVYoonpy4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nk # kDstrjNYxbc+/jLTswM9sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6 # A4aN91/w0FK/jJSHvMAhdCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmd # X4jiJV3TIUs+UsS1Vz8kA/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL # 5zmhD+kjSbwYuER8ReTBw3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zd # sGbiwZeBe+3W7UvnSSmnEyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3 # T8HhhUSJxAlMxdSlQy90lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS # 4NaIjAsCAwEAAaOCAe0wggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRI # bmTlUAXTgqoXNzcitW2oynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTAL # BgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBD # uRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jv # c29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFf # MDNfMjIuY3J0MIGfBgNVHSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEF # BQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1h # cnljcHMuaHRtMEAGCCsGAQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkA # YwB5AF8AcwB0AGEAdABlAG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn # 8oalmOBUeRou09h0ZyKbC5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7 # v0epo/Np22O/IjWll11lhJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0b # pdS1HXeUOeLpZMlEPXh6I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/ # KmtYSWMfCWluWpiW5IP0wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvy # CInWH8MyGOLwxS3OW560STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBp # mLJZiWhub6e3dMNABQamASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJi # hsMdYzaXht/a8/jyFqGaJ+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYb # BL7fQccOKO7eZS/sl/ahXJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbS # oqKfenoi+kiVH6v7RyOA9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sL # gOppO6/8MO0ETI7f33VtY5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtX # cVZOSEXAQsmbdlsKgEhr/Xmfwb1tbWrJUnMTDXpQzTGCGaIwghmeAgEBMIGVMH4x # CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt # b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01p # Y3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTECEzMAAANNTpGmGiiweI8AAAAA # A00wDQYJYIZIAWUDBAIBBQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQw # HAYKKwYBBAGCNwIBCzEOMAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIIzi # xVrZks/UzszDxx8DJ/NZa7Ph8l06VTJDHEiB0KIPMEIGCisGAQQBgjcCAQwxNDAy # oBSAEgBNAGkAYwByAG8AcwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20wDQYJKoZIhvcNAQEBBQAEggEAaX+oK6wp8v7tK9eiSIWJVlxOMCzmrYRFH5wG # Wwwb1PyA1hsBIa4e0tx6ECDPTfKXifA3tkjn7ib3ezG3JJJveX5jCachOWpy0qr1 # jMOYhOLBDYelfHZGzLynM81DjsrKJVqIge5tdf+/a8bdazP7Hn+jmMUOzP0SnQGi # fQ3kBLnwE0yKAzHqPHsG4qs5hjG1cuD9sVE0AYYd0ho/kq6Ac57F0YIzefqkF4YH # YTMee57Lkir7Fl5KncozC/i/gdEi2SH9C/4cYNvIJZzDVXiArS1dp6/8fp+bzDGo # 2vPjf3xGH1tTFNH57xDjKuf77nkQCjwK6JErKggigPOzhbGe+KGCFywwghcoBgor # BgEEAYI3AwMBMYIXGDCCFxQGCSqGSIb3DQEHAqCCFwUwghcBAgEDMQ8wDQYJYIZI # AWUDBAIBBQAwggFZBgsqhkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGE # WQoDATAxMA0GCWCGSAFlAwQCAQUABCA3NG1sjp/iZqiJHTXXIMFyWjJn8fHvDwpa # wh6Ruq/MFwIGZJNEQjLeGBMyMDIzMDcwNzIwMjkwOC4xNTVaMASAAgH0oIHYpIHV # MIHSMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQL # EyRNaWNyb3NvZnQgSXJlbGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsT # HVRoYWxlcyBUU1MgRVNOOkZDNDEtNEJENC1EMjIwMSUwIwYDVQQDExxNaWNyb3Nv # ZnQgVGltZS1TdGFtcCBTZXJ2aWNloIIRezCCBycwggUPoAMCAQICEzMAAAG59gAN # ZVRPvAMAAQAAAbkwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAg # UENBIDIwMTAwHhcNMjIwOTIwMjAyMjE3WhcNMjMxMjE0MjAyMjE3WjCB0jELMAkG # A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx # HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9z # b2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMg # VFNTIEVTTjpGQzQxLTRCRDQtRDIyMDElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt # U3RhbXAgU2VydmljZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAONJ # Pslh9RbHyQECbUIINxMF5uQkyN07VIShITXubLpWnANgBCLvCcJl7o/2HHORnsRc # mSINJ/qclAmLIrOjnYnrbocAnixiMEXC+a1sZ84qxYWtEVY7VYw0LCczY+86U/8s # hgxqsaezKpWriPOcpV1Sh8SsOxf30yO7jvld/IBA3T6lHM2pT/HRjWk/r9uyx0Q4 # atx0mkLVYS9y55/oTlKLE00h792S+maadAdy3VgTweiwoEOXD785wv3h+fwH/wTQ # tC9lhAxhMO4p+OP9888Wxkbl6BqRWXud54RTzqp2Vr+yen1Q1A6umyMB7Xq0snIY # G5B1Acc4UgJlPQ/ZiMkqgxQNFCWQvz0G9oLgSPD8Ky0AkX22PcDOboPuNT4RceWP # X0UVZUsX9IUgs7QF41HiQSwEeOOHGyrfQdmSslATrbmH/18M5QrsTM5JINjct9G4 # 2xqN8VF9Z8WOiGMjNbvlpcEmmysYl5QyhrEDoFnQTU7bFrD3JX0fIfu1sbLWeBqX # wbp4Z8yACTtphK2VbzOvi4vc0RCmRNzvYQQ2PjZ7NaTXE4Gu3vggAJ+rtzUTAfJo # tvOSqcMgNwLZa1Y+ET/lb0VyjrYwFuHtg0QWyQjP5350LTpv086pyVUh4A3w/Os5 # hTGFZgFe5bCyMnpY09M0yPdHaQ/56oYUsSIcyKyVAgMBAAGjggFJMIIBRTAdBgNV # HQ4EFgQUt7A4cdtYQ5oJjE1ZqrSonp41RFIwHwYDVR0jBBgwFoAUn6cVXQBeYl2D # 9OXSZacbUzUZ6XIwXwYDVR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3Nv # ZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUy # MDIwMTAoMSkuY3JsMGwGCCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDov # L3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1l # LVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUB # Af8EDDAKBggrBgEFBQcDCDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQAD # ggIBAM3cZ7NFUHRMsLKzjl7rJPIkv7oJ+s9kkut0hZif9WSt60SzYGULp1zmdPqc # +w8eHTkhqX0GKCp2TTqSzBXBhwHOm8+p6hUxNlDewGMZUos952aTXblAT3OKBnfV # BLQyUavrSjuJGZAW30cNY3rjVDUlGD+VygQHySaDaviJQbK6/6fQvUUFoqIk3ldG # fjnAtnebsVlqh6WWamVc5AZdpWR1jSzN/oxKYqc1BG4SxxlPtcfrAdBz/cU4bxVX # qAAf02NZscvJNpRnOALf5kVo2HupJXCsk9TzP5PNW2sTS3TmwhIQmPxr0E0UqOoj # UrBJUOhbITAxcnSa/IMluL1HXRtLQZI+xs2eRtuPOUsKUW71/1YeqsYCLHLvu82c # eDVQQvP7GHEEkp2kEjiofbjYErBo2iCEaxxeX4Z9HvAgA4MsQkbn6e4EFQf13sP+ # Kn3XgMIvJbqLJeFcQja+SUeOXu5cfkxe0GzTNojdyIwzaHlhOflVRZNrxee3B+yZ # wd3JHDIvv71uSI/SIzzt9cU2GyHQVqxBSrRtKW6W8Vw7zpVvoVsIv3ljxg+7NiGS # lXX1s7zbBNDMUj9OnzOlHK/3mrOU8YEuRf6RwakW5UCeGamy5MiKu2YuyKiGBCv4 # OGhPstNe7ALkEOh8BX12t4ntuYu+gw9L6yCPY0jWYaQtzAP9MIIHcTCCBVmgAwIB # AgITMwAAABXF52ueAptJmQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UE # BhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAc # BgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0 # IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1 # WhcNMzAwOTMwMTgzMjI1WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O # 1YLT/e6cBwfSqWxOdcjKNVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZn # hUYjDLWNE893MsAQGOhgfWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t # 1w/YJlN8OWECesSq/XJprx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxq # D89d9P6OU8/W7IVWTe/dvI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmP # frVUj9z6BVWYbWg7mka97aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSW # rAFKu75xqRdbZ2De+JKRHh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv # 231fgLrbqn427DZM9ituqBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zb # r17C89XYcz1DTsEzOUyOArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYcten # IPDC+hIK12NvDMk2ZItboKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQc # xWv2XFJRXRLbJbqvUAV6bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17a # j54WcmnGrnu3tz5q4i6tAgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQAB # MCMGCSsGAQQBgjcVAgQWBBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQU # n6cVXQBeYl2D9OXSZacbUzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEw # QTA/BggrBgEFBQcCARYzaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9E # b2NzL1JlcG9zaXRvcnkuaHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQB # gjcUAgQMHgoAUwB1AGIAQwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/ # MB8GA1UdIwQYMBaAFNX2VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJ # oEeGRWh0dHA6Ly9jcmwubWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01p # Y1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYB # BQUHMAKGPmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9v # Q2VyQXV0XzIwMTAtMDYtMjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3h # LB9nATEkW+Geckv8qW/qXBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x # 5MKP+2zRoZQYIu7pZmc6U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74p # y27YP0h1AdkY3m2CDPVtI1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1A # oL8ZthISEV09J+BAljis9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbC # HcNhcy4sa3tuPywJeBTpkbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB # 9s7GdP32THJvEKt1MMU0sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNt # yo4JvbMBV0lUZNlz138eW0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3 # rsjoiV5PndLQTHa1V1QJsWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcV # v7TOPqUxUYS8vwLBgqJ7Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A24 # 5oyZ1uEi6vAnQj0llOZ0dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lw # Y1NNje6CbaUFEMFxBmoQtB1VM1izoXBm8qGCAtcwggJAAgEBMIIBAKGB2KSB1TCB # 0jELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1Jl # ZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMk # TWljcm9zb2Z0IElyZWxhbmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1U # aGFsZXMgVFNTIEVTTjpGQzQxLTRCRDQtRDIyMDElMCMGA1UEAxMcTWljcm9zb2Z0 # IFRpbWUtU3RhbXAgU2VydmljZaIjCgEBMAcGBSsOAwIaAxUAx2IeGHhk58MQkzzS # WknGcLjfgTqggYMwgYCkfjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGlu # Z3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBv # cmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAN # BgkqhkiG9w0BAQUFAAIFAOhS2iEwIhgPMjAyMzA3MDgwMjM4MjVaGA8yMDIzMDcw # OTAyMzgyNVowdzA9BgorBgEEAYRZCgQBMS8wLTAKAgUA6FLaIQIBADAKAgEAAgIp # owIB/zAHAgEAAgIRKTAKAgUA6FQroQIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgor # BgEEAYRZCgMCoAowCAIBAAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUA # A4GBACdlQT8AlIy5FP+IIYba5jCtmF91SYcq9HIrf5kgJEaFKe0UkL2NTvn/Gr3+ # d0nAI34Vh9Vu3RTkeaf87sQL0IgZIPJoHw+7j7oM9CPgIqwOZAGqOKT+1wIYpWqB # KSEXy81eh70kqnHBxQcZ7jTtPSMeaK+mZ6lRrzT9RU8GAXjmMYIEDTCCBAkCAQEw # gZMwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcT # B1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UE # AxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAG59gANZVRPvAMA # AQAAAbkwDQYJYIZIAWUDBAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0B # CRABBDAvBgkqhkiG9w0BCQQxIgQgga+HpuN1fPuIu1BY4ubH7ifQ9VL8eLshf6s3 # qOfOzHcwgfoGCyqGSIb3DQEJEAIvMYHqMIHnMIHkMIG9BCBk60bO8W85uTAfJVEO # 3vX2aLaQFcgcGpdwsOoi+foP9DCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1w # IFBDQSAyMDEwAhMzAAABufYADWVUT7wDAAEAAAG5MCIEIF9eCe+3ZFU1QBI6fLVY # 9rPNaLLefrDNHnnysn7FWSQHMA0GCSqGSIb3DQEBCwUABIICAKDrShhlueyDvsEv # Fv8vO6/szJc+bXQPSkFkpTt9OlPnTtrBFawPo7Ldh8chaOsIXBolaFz2A6M63qEM # fEJPArICo8JYy2PKOj1by15Jm+YdDi483tfLVXhChIbJkIWT6DK8JkIgwHYCwG93 # bHWsl3jg7v/qS3I7wT1ERLYU1dTlxAkBOy7qe1CHSx5aPVfpGJSfnJmcMQxMpPUq # h6BD7nGeviLfdu+Lw5NFnIrDyhZFq7AAd6XmQlbb778bvoh9ITF2l76EJkhbQ9Ow # HO+BjOAJ0xQDHRwXspTRjGvAuFQmwsna8X3evOnwe0Tasmsvm2Dxa1vbofb/0Hkc # 5Df0YouiBR0I37jhNKQUsx0FAE/n5GN4rM5TgHdon/gE+CeJu4nUlzhGCjzkwAu3 # 9tei3EYc0HpkIxtb7F+3tjYX8NcVQjxbBbZ0ki24/evqgOdn9PcAvnzDwYLOWghS # 4LqpYT1sCnN6dCJmxjsudclfaojnVWuGVb+RzUwaxn35p2Vc2rcT2p6enliYdUkd # DyCOo/BUiNZQhGTLMReQaf8IcvVq3Jo7/HwfCOr52yoDTxX6Lkc8aM2O1RKdZn9r # enxZoQ/tmA6ZthE3cvrup7fKianMPzOvLim84GhhvGCE7WsDhpppFJDhgJIdh4s0 # kOEi5rlnAa8aHOpS9M3rCA0rev9z # SIG # End signature block |