Obs/bin/ObsDep/content/Powershell/Roles/Common/PhysicalMachineHelpers.psm1
|
# -------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All Rights Reserved. # Microsoft Corporation (or based on where you live, one of its affiliates) licenses this sample code for your internal testing purposes only. # Microsoft provides the following sample code AS IS without warranty of any kind. The sample code arenot supported under any Microsoft standard support program or services. # Microsoft further disclaims all implied warranties including, without limitation, any implied warranties of merchantability or of fitness for a particular purpose. # The entire risk arising out of the use or performance of the sample code remains with you. # In no event shall Microsoft, its authors, or anyone else involved in the creation, production, or delivery of the code be liable for any damages whatsoever # (including, without limitation, damages for loss of business profits, business interruption, loss of business information, or other pecuniary loss) # arising out of the use of or inability to use the sample code, even if Microsoft has been advised of the possibility of such damages. # --------------------------------------------------------------- Import-Module -Name "$PSScriptRoot\..\Common\RoleHelpers.psm1" Import-Module -Name "$PSScriptRoot\..\..\Common\NetworkHelpers.psm1" Import-Module -Name "$PSScriptRoot\..\..\Common\StorageHelpers.psm1" Import-Module -Name "$PSScriptRoot\..\..\Common\ClusterHelpers.psm1" <# .Synopsis Function to shut down all provisioning machines from any role context .Parameter OOBManagementModulePath A relative path to the out-of-band management dll -- during deployment it is available locally but in ECE service it is located in an unpacked package. .Parameter Parameters This object is based on the customer manifest. It contains the private information of the Key Vault role, as well as public information of all other roles. It is passed down by the deployment engine. .Example Stop-DeployingMachines -Parameters $Parameters This will shut down all hosts that are in provisioning state #> function Stop-DeployingMachines { [CmdletBinding()] param ( [Parameter(Mandatory=$false)] [string] $OOBManagementModulePath = "$PSScriptRoot\..\..\OOBManagement\bin\Microsoft.AzureStack.OOBManagement.dll", [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) # After deployment, the source of this module changes Import-Module -Name $OOBManagementModulePath Import-Module -Name "$PSScriptRoot\..\..\OOBManagement\bin\Newtonsoft.Json.dll" # Shut down all machines to avoid IP conflicts $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration $bareMetalCredential = Get-BareMetalCredential -Parameters $Parameters $OEMRole = $Parameters.Roles["OEM"].PublicConfiguration $OEMModel = $OEMRole.PublicInfo.UpdatePackageManifest.UpdateInfo.Model if ([String]::IsNullOrEmpty($Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name)) { Trace-Execution "Target to all physical nodes since node list not set in execution context" [Array]$provisioningNodes = $physicalMachinesRole.Nodes.Node } else { Trace-Execution "Get node list from execution context" # Determine whether the context of the operation is a cluster scale out if ($Parameters.Context.ExecutionContext.Roles.Role.RoleName -ieq "Cluster") { $nodes = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.PhysicalNodes.PhysicalNode.Name } else { $nodes = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name } [Array]$provisioningNodes = $physicalMachinesRole.Nodes.Node | Where-Object { $nodes -contains $_.Name } } # If any machines are not accessible during this call, the caller may fail. It is important to mark all provisioning nodes as failed when this happens foreach ($node in $provisioningNodes) { $bmcIP = $node.BmcIPAddress $nodeName = $node.Name $nodeInstance = $node.NodeInstance $oobProtocol = $node.OOBProtocol if ($OEMModel -match "Hyper-V") { $trustedHosts = (Get-Item -Path WSMan:\localhost\Client\TrustedHosts).Value if (($trustedHosts -ne '*') -and ($bmcIP -notin $trustedHosts.Split(','))) { Set-Item -Path WSMan:\localhost\Client\TrustedHosts -Value $bmcIP -Concatenate -Force } Invoke-Command -ComputerName $bmcIP -Credential $bareMetalCredential -ScriptBlock {Stop-VM -VMName $using:nodeName -TurnOff -Force -ErrorAction Stop } -ErrorAction Stop Set-Item -Path WSMan:\localhost\Client\TrustedHosts -Value $trustedHosts -Force } else { Trace-Execution "Shut down $nodeName ($bmcIP)." Stop-IpmiDevice -TargetAddress $bmcIP -Credential $bareMetalCredential -NodeInstance $nodeInstance -OOBProtocol $oobProtocol -Verbose -Wait } } } <# .Synopsis Function to start-up all provisioning machines from any role context .Parameter OOBManagementModulePath A relative path to the out-of-band management dll -- during deployment it is available locally but in ECE service it is located in an unpacked package. .Parameter Parameters This object is based on the customer manifest. It contains the private information of the Key Vault role, as well as public information of all other roles. It is passed down by the deployment engine. .Example Start-DeployingMachines -Parameters $Parameters This will startup all hosts that are in provisioning state #> function Start-DeployingMachines { [CmdletBinding()] param ( [Parameter(Mandatory=$false)] [string] $OOBManagementModulePath = "$PSScriptRoot\..\..\OOBManagement\bin\Microsoft.AzureStack.OOBManagement.dll", [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters ) # After deployment, the source of this module changes Import-Module -Name $OOBManagementModulePath Import-Module -Name "$PSScriptRoot\..\..\OOBManagement\bin\Newtonsoft.Json.dll" $physicalMachinesRole = $Parameters.Roles["BareMetal"].PublicConfiguration $bareMetalCredential = Get-BareMetalCredential -Parameters $Parameters $OEMRole = $Parameters.Roles["OEM"].PublicConfiguration $OEMModel = $OEMRole.PublicInfo.UpdatePackageManifest.UpdateInfo.Model if ([String]::IsNullOrEmpty($Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name)) { Trace-Execution "Target to all physical nodes since node list not set in execution context" [Array]$provisioningNodes = $physicalMachinesRole.Nodes.Node } else { Trace-Execution "Get node list from execution context" # Determine whether the context of the operation is a cluster scale out if ($Parameters.Context.ExecutionContext.Roles.Role.RoleName -ieq "Cluster") { $nodes = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.PhysicalNodes.PhysicalNode.Name } else { $nodes = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name } [Array]$provisioningNodes = $physicalMachinesRole.Nodes.Node | Where-Object { $nodes -contains $_.Name } } # If any machines are not accessible during this call, the caller may fail. It is important to mark all provisioning nodes as failed when this happens foreach ($node in $provisioningNodes) { $bmcIP = $node.BmcIPAddress $nodeName = $node.Name $nodeInstance = $node.NodeInstance $oobProtocol = $node.OOBProtocol # TODO: For now this function will not support virtual hosts. # Due to time constraints and TZL requirements to get # SED support and clean up in CI, we will comment this # part of handling Virtual environments. MUST FIX soon. if ($OEMModel -match "Hyper-V") { Trace-Warning "Virtual environments are not supported in this version." Trace-Warning "This function is FOR NOW only designed to be called for SED cleanup on physical machines" Trace-Warning "Virtual environments will be handled here with the coming version." <# $trustedHosts = (Get-Item -Path WSMan:\localhost\Client\TrustedHosts).Value if (($trustedHosts -ne '*') -and ($bmcIP -notin $trustedHosts.Split(','))) { Set-Item -Path WSMan:\localhost\Client\TrustedHosts -Value $bmcIP -Concatenate -Force } Invoke-Command -ComputerName $bmcIP -Credential $bareMetalCredential -ScriptBlock {Stop-VM -VMName $using:nodeName -TurnOff -Force -ErrorAction Stop } -ErrorAction Stop Set-Item -Path WSMan:\localhost\Client\TrustedHosts -Value $trustedHosts -Force #> } else { Trace-Execution "Starting physical node: $nodeName ($bmcIP)." Start-IpmiDevice -TargetAddress $bmcIP -Credential $bareMetalCredential -NodeInstance $NodeInstance -OOBProtocol $oobProtocol -Verbose -Wait } } } <# .Synopsis Function to add a DHCP MAC-based reservation for a given machine. .Parameter NetworkName The network name within the infrastructure network where the IP should be reserved. .Parameter NodeName The node name to be associated with the reservation. .Parameter IPv4Address An IPv4 Address from the speficied network scope that the node will have reserved. .Parameter MacAddress The MAC address to associate with the reservation -- this can be in either dash or dashless format, but not comma format. .Parameter RemoteDHCPServerName The remote server that will act as the DHCP server to configure. .Parameter RemoteDHCPServerNameCredentials Credentials used to add the reservation only used on a remote computer when specified. .Example Add-DeployingMachineNetworkReservation -NetworkName 's-cluster-HostNic' -NodeName 'foo' -IPv4Address '10.0.0.1' -MACAddress '7C-FE-90-AF-1A-00' This will add a reservation for node foo with MAC 7C-FE-90-AF-1A-00 using the local credentials on the local DHCP server #> function Add-DeployingMachineNetworkReservation { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $NetworkName, [Parameter(Mandatory=$true)] [String] $NodeName, [Parameter(Mandatory=$true)] [String] $IPv4Address, [Parameter(Mandatory=$true)] [String] $MacAddress, [Parameter(Mandatory=$false)] [String] $RemoteDHCPServerName, [Parameter(Mandatory=$false)] [PSCredential] $RemoteDHCPServerNameCredentials ) $scriptBlock = { $scope = Get-DhcpServerv4Scope | Where-Object { $_.Name -eq $using:NetworkName } $reservation = $scope | Get-DhcpServerv4Reservation | Where-Object { $_.ClientId.Replace(':','').Replace('-','') -eq ($using:MacAddress).Replace(':','').Replace('-','') } foreach ($entry in $reservation) { # Always clear existing reservations that apply to this client ID Remove-DhcpServerv4Reservation -ClientId $entry.ClientId -ScopeId $entry.ScopeId } $scope | Add-DhcpServerv4Reservation -Name $using:NodeName -IPAddress $using:IPv4Address -ClientId $using:MacAddress -Description $using:NodeName } if ($remoteDHCPServerName) { $session = New-PSSession -ComputerName $RemoteDHCPServerName -Credential $RemoteDHCPServerNameCredentials } else { $session = New-PSSession } try { Trace-Execution "Adding DHCP reservation in '$NetworkName' scope: $NodeName - $IPv4Address - $MacAddress." Invoke-Command -Session $session -ScriptBlock $scriptBlock } catch { Trace-Error "Failed with error: $_" throw } finally { Remove-PSSession -Session $session -ErrorAction Ignore } } <# .Synopsis Function to add a DHCP MAC-based reservation for a given machine. .Parameter NetworkName The network name within the infrastructure network where the IP should be reserved. .Parameter NodeName The node name to be associated with the reservation. .Parameter IPv4Address An IPv4 Address from the speficied network scope that the node will have reserved. .Parameter RemoteDHCPServerName The remote server that will act as the DHCP server to configure. .Parameter RemoteDHCPServerNameCredentials Credentials used to add the reservation only used on a remote computer when specified. .Example Remove-MachineNetworkReservation -NetworkName 's-cluster-HostNic' -NodeName 'foo' -IPv4Address '10.0.0.1' -RemoteDHCPServerName 'Machine' -RemoteDHCPServerNameCredentials $cred This will remove a reservation and its leases for node foo with IP address specified #> function Remove-MachineNetworkReservation { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [String] $NetworkName, [Parameter(Mandatory=$true)] [String] $IPv4Address, [Parameter(Mandatory=$true)] [String] $RemoteDHCPServerName, [Parameter(Mandatory=$true)] [PSCredential] $RemoteDHCPServerNameCredentials ) $scriptBlock = { $scope = Get-DhcpServerv4Scope | Where-Object { $_.Name -eq $using:NetworkName } $scope | Get-DhcpServerv4Reservation | Where-Object { $_.IPAddress -eq ($using:IPv4Address) } | Remove-DhcpServerv4Reservation # Remove the DHCP lease for the machine that is being removed as well Get-DhcpServerv4Lease -ScopeId $scope.ScopeId | Where-Object { $_.IPAddress -eq $using:IPv4Address } | Remove-DhcpServerv4Lease } try { $session = New-PSSession -ComputerName $RemoteDHCPServerName -Credential $RemoteDHCPServerNameCredentials Trace-Execution "Adding DHCP reservation in '$NetworkName' scope: $NodeName - $IPv4Address ." Invoke-Command -Session $session -ScriptBlock $scriptBlock } catch { Trace-Error "Failed with error: $_" throw } finally { $session | Remove-PSSession -ErrorAction Ignore } } <# .Synopsis Function to force a machine to boot in to PXE using BMC controls only. .Parameter OOBManagementModulePath A relative path to the out-of-band management dll -- during deployment it is available locally but in ECE service it is located in an unpacked package. .Parameter PhysicalNode Information about the node to start in to PXE. .Parameter BMCCredential Credentials used to interact with the BMC controller. .Example Start-PXEBoot -NodeName $Name -BmcIPAddress $BmcIPAddress -BMCCredential $cred This will force the machine to boot in to PXE #> function Start-PXEBoot { [CmdletBinding()] param ( [Parameter(Mandatory=$false)] [string] $OOBManagementModulePath="$PSScriptRoot\..\..\OOBManagement\bin\Microsoft.AzureStack.OOBManagement.dll", [Parameter(Mandatory=$true)] [string] $NodeName, [Parameter(Mandatory=$true)] [string] $BmcIPAddress, [Parameter(Mandatory=$true)] [PSCredential] $BMCCredential, [Parameter(Mandatory=$true)] [string] $NodeInstance, [Parameter(Mandatory=$true)] [string] $OOBProtocol ) # After deployment, the source of this module changes Import-Module -Name $OOBManagementModulePath Import-Module -Name "$PSScriptRoot\..\..\OOBManagement\bin\Newtonsoft.Json.dll" $logText = "Initiate PXE boot for $NodeName (BMC: $BmcIPAddress). `r`n" $logText += "Shutdown $NodeName (BMC: $BmcIPAddress). `r`n" # Normally the following line should do nothing, because the machine is expected to have been powered off in an earlier step, unless this is a deployment # restart, in which case the machine may be running. $logText += (Stop-IpmiDevice -TargetAddress $BmcIPAddress -Credential $BMCCredential -NodeInstance $NodeInstance -OOBProtocol $OOBProtocol -Wait -Verbose 4>&1) $logText += "`n Set PXE boot $NodeName (BMC: $BmcIPAddress). `r`n" # Some machines will set the next boot device correctly only if the machine is currently powered off. $logText += (Set-IpmiDeviceOneTimePxeBoot -TargetAddress $BmcIPAddress -Credential $BMCCredential -NodeInstance $NodeInstance -OOBProtocol $OOBProtocol -Verbose 4>&1) $logText += "`n Start $NodeName (BMC: $BmcIPAddress). `r`n" $logText += (Start-IpmiDevice -TargetAddress $BmcIPAddress -Credential $BMCCredential -NodeInstance $NodeInstance -OOBProtocol $OOBProtocol -Wait -Verbose 4>&1) $logText += "`n Set PXE boot $NodeName (BMC: $BmcIPAddress OEMWorkaround). `r`n" # Some machines (Dell R630) will set the next boot device correctly only if the machine is currently powered on and gets power cycled (not reset, but # specifically power cycled) after the request to set one-time PXE boot. $logText += (Set-IpmiDeviceOneTimePxeBoot -TargetAddress $BmcIPAddress -Credential $BMCCredential -NodeInstance $NodeInstance -OOBProtocol $OOBProtocol -Verbose 4>&1) $logText += "`n Restart $NodeName (BMC: $BmcIPAddress OEMWorkaround). `r`n" $logText += (Restart-IpmiDevice -TargetAddress $BmcIPAddress -Credential $BMCCredential -NodeInstance $NodeInstance -OOBProtocol $OOBProtocol -Verbose 4>&1) $retObj = @{LogText = $logText} return $retObj } <# .Synopsis Function to force a machine to reboot out of band from any context that can access BMC. .Parameter OOBManagementModulePath A relative path to the out-of-band management dll -- during deployment it is available locally but in ECE service it is located in an unpacked package. .Parameter PhysicalNode Information about the node to restart. .Parameter BMCCredential Credentials used to interact with the BMC controller. .Example Restart-Node -PhysicalNode $node -BMCCredential $cred This will force the machine to reboot #> function Restart-Node { [CmdletBinding()] param ( [Parameter(Mandatory=$false)] [string] $OOBManagementModulePath="$PSScriptRoot\..\..\OOBManagement\bin\Microsoft.AzureStack.OOBManagement.dll", [Parameter(Mandatory=$true)] [System.Xml.XmlElement] $PhysicalNode, [Parameter(Mandatory=$true)] [PSCredential] $BMCCredential, [Parameter(Mandatory=$true)] [string] $NodeInstance, [Parameter(Mandatory=$true)] [string] $OOBProtocol ) # After deployment, the source of this module changes Import-Module -Name $OOBManagementModulePath Import-Module -Name "$PSScriptRoot\..\..\OOBManagement\bin\Newtonsoft.Json.dll" $nodeName = $PhysicalNode.Name $bmcIP = $PhysicalNode.BmcIPAddress Trace-Execution "Initiate IPMI-based restart for $nodeName (BMC: $bmcIP)." Restart-IpmiDevice -TargetAddress $bmcIP -Credential $BMCCredential -NodeInstance $NodeInstance -OOBProtocol $OOBProtocol -Verbose -Wait } <# .Synopsis Function to get product information of a physical node. .Parameter OOBManagementModulePath A relative path to the out-of-band management dll -- during deployment it is available locally but in ECE service it is located in an unpacked package. .Parameter PhysicalNode Information about the node. .Parameter BMCCredential Credentials used to interact with the BMC controller. .Example Restart-Node -PhysicalNode $node -BMCCredential $cred #> function Get-ProductInfo { [CmdletBinding()] param ( [Parameter(Mandatory = $false)] [string] $OOBManagementModulePath = "$PSScriptRoot\..\..\OOBManagement\bin\Microsoft.AzureStack.OOBManagement.dll", [Parameter(Mandatory=$true)] [System.Xml.XmlElement] $PhysicalMachinesRole, [Parameter(Mandatory=$true)] [PSCredential] $BMCCredential, [Parameter(Mandatory=$true)] [string] $OEMModel ) # After deployment, the source of this module changes Import-Module -Name $OOBManagementModulePath Import-Module -Name "$PSScriptRoot\..\..\OOBManagement\bin\Newtonsoft.Json.dll" if ([String]::IsNullOrEmpty($Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name)) { Trace-Execution "Target to all physical nodes since node list not set in execution context" [Array]$provisioningNodes = $PhysicalMachinesRole.Nodes.Node } else { Trace-Execution "Get node list from execution context" # Determine whether the context of the operation is a cluster scale out if ($Parameters.Context.ExecutionContext.Roles.Role.RoleName -ieq "Cluster") { $nodes = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.PhysicalNodes.PhysicalNode.Name } else { $nodes = $Parameters.Context.ExecutionContext.Roles.Role.Nodes.Node.Name } [Array]$provisioningNodes = $PhysicalMachinesRole.Nodes.Node | Where-Object { $nodes -contains $_.Name } } foreach ($node in $provisioningNodes) { $bmcIP = $node.BmcIPAddress $nodeName = $node.Name $nodeInstance = $node.NodeInstance $oobProtocol = $node.OOBProtocol try { Trace-Execution "Initiate IPMI-based cmd to get Fru log for $nodeName (BMC: $bmcIP)." $fru = Get-IpmiDeviceFruLogs -TargetAddress $bmcIP -Credential $BMCCredential -NodeInstance $nodeInstance -OOBProtocol $oobProtocol if ($fru) { return @{"Model" = $fru.ProductInfo.ProductName "Vendor" = $fru.ProductInfo.ManufacturerName "Serial" = $fru.ProductInfo.SerialNumber } } } catch { Trace-Execution "Fail to Get-IpmiDeviceFruLogs for $nodeName (BMC: $bmcIP): $_" } } return $null } <# .Synopsis Function to wait for ping, CIM, recent OS installation (with a deployment artifact) and WinRM to be available on a machine .Parameter StartTime The start time of the operation, used to check that OS boot time was strictly after the wait period. .Parameter StopTime The stop time for the wait operation after which the operation is considered failed. .Parameter PhysicalNodeArray A list of physical machine nodes to wait. .Parameter RemoteCIMCredential The credential to use to connect to CIM and WinRM. .Parameter DeploymentID A unique identifier meant to signify the deployment that the machine is associated with. .Parameter IgnoreOldOSCheckResult Whether to consider it a failure if the OS discovered on the machine is from before the waiting period began. #> function Wait-RemoteCimInitializedOneNodeBareMetal { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [DateTime] $StartTime, [Parameter(Mandatory=$true)] [DateTime] $StopTime, [Parameter(Mandatory=$true)] [System.Xml.XmlElement[]] $PhysicalNodeArray, [Parameter(Mandatory=$true)] [PSCredential] $RemoteCIMCredential, [Parameter(Mandatory=$true)] [Guid] $DeploymentID, [Parameter(Mandatory=$true)] [bool] $IgnoreOldOSCheckResults ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop $remainingNodes = $PhysicalNodeArray $failedNodeNames = @() do { $nodes = $remainingNodes foreach ($node in $nodes) { $nodeName = $node.Name $nodeIP = $node.IPv4Address # Default to HostNIC if we cant find any IPv4Addresses if (-not $nodeIP) { $hostNICNetworkName = Get-NetworkNameForCluster -ClusterName "s-cluster" -NetworkName "HostNIC" $nodeIP = ($node.NICs.NIC | Where-Object NetworkId -eq $hostNICNetworkName | ForEach-Object IPv4Address)[0].Split('/')[0] } if (Test-IPConnection $nodeIP) { $cimSession = $null $session = $null try { $errorCountBefore = $global:error.Count #try the node name first because in some scenario IP is not trusted $cimSession = New-CimSession -ComputerName $nodeName -Credential $RemoteCIMCredential -ErrorAction SilentlyContinue if ($null -eq $cimSession) { $cimSession = New-CimSession -ComputerName $nodeIP -Credential $RemoteCIMCredential -ErrorAction SilentlyContinue } $os = $null if ($cimSession) { $os = Get-CimInstance win32_operatingsystem -CimSession $cimSession -ErrorAction SilentlyContinue } $errorCountAfter = $global:error.Count $numberOfNewErrors = $errorCountAfter - $errorCountBefore if ($numberOfNewErrors -gt 0) { $global:error.RemoveRange(0, $numberOfNewErrors) } if ($os) { $osLastBootUpTime = $os.LastBootUpTime $osInstallTime = $os.InstallDate #try the node name first because in some scenario IP is not trusted $session = New-PSSession -ComputerName $nodeName -Credential $RemoteCIMCredential -ErrorAction SilentlyContinue if ($null -eq $session) { $session = New-PSSession -ComputerName $nodeIP -Credential $RemoteCIMCredential -ErrorAction SilentlyContinue } } else { Trace-Execution "$nodeName could not query an OS." } } catch { if ($session) { Trace-Execution "Caught exception after session to new OS was created: $_" Remove-PSSession $session -ErrorAction SilentlyContinue $session = $null } $global:error.RemoveAt(0) } if ($session) { $isNewDeployment = Invoke-Command -Session $session -ScriptBlock { Test-Path "$env:SystemDrive\CloudBuilderTemp\$($using:DeploymentID).txt" } if ($isNewDeployment) { Trace-Execution "$nodeName has finished OS deployment. Boot time reported by the node - $($osLastBootUpTime.ToString())." $failedNodeNames = $failedNodeNames | Where-Object Name -ne $nodeName } else { Trace-Warning "$nodeName has booted up to an old OS installed on $($osInstallTime.ToString())." if (-not $IgnoreOldOSCheckResult) { $failedNodeNames += $nodeName } } $remainingNodes = $remainingNodes | Where-Object Name -ne $node.Name } elseif ($cimSession) { Remove-CimSession $cimSession } else { Trace-Execution "$nodeName could not establish CIM session using IP $nodeIP or name." } } else { Trace-Execution "$nodeName did not respond to ping with IP $nodeIP ." } } if (-not $remainingNodes) { break } Start-Sleep -Seconds 15 } until ([DateTime]::Now -gt $StopTime) $remainingNodeNames = @($remainingNodes | ForEach-Object Name) $totalBmdWaitTimeMinutes = [int]($StopTime - $StartTime).TotalMinutes if ($failedNodeNames -or $remainingNodeNames) { Trace-Error "Bare metal deployment failed to complete in $totalBmdWaitTimeMinutes minutes - $(($failedNodeNames + $remainingNodeNames) -join ',')." } else { Trace-Execution "Bare metal deployment has completed on the target node." } } function Wait-ForISOImageBareMetal { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [DateTime] $StartTime, [Parameter(Mandatory=$true)] [DateTime] $StopTime, [Parameter(Mandatory=$true)] [Array] $PhysicalNodeArray, [Parameter(Mandatory=$true)] [PSCredential] $RemoteCIMCredential, [Parameter(Mandatory=$true)] [Guid] $DeploymentID, [Parameter(Mandatory=$true)] [bool] $IgnoreOldOSCheckResults ) $ErrorActionPreference = [System.Management.Automation.ActionPreference]::Stop [Array]$remainingNodes = $PhysicalNodeArray | Sort-Object Name $failedNodeNames = @() $wait = $true while ($wait) { if ([DateTime]::Now -lt $StopTime) { foreach ($node in $remainingNodes) { $nodeName = $node.Name $nodeIP = $node.IPv4Address if (Test-IPConnection $nodeIP) { Trace-Execution "$nodeName is responding to ping" $cimSession = $null $session = $null try { $errorCountBefore = $global:error.Count # Try the node name first because in some scenario IP is not trusted $cimSession = New-CimSession -ComputerName $nodeName -Credential $RemoteCIMCredential -ErrorAction SilentlyContinue if ($null -eq $cimSession) { $cimSession = New-CimSession -ComputerName $nodeIP -Credential $RemoteCIMCredential -ErrorAction SilentlyContinue } $os = $null if ($null -ne $cimSession) { Trace-Execution "$nodeName CIM was session created - getting Win32_OperatingSystem data" $os = Get-CimInstance Win32_OperatingSystem -CimSession $cimSession -ErrorAction SilentlyContinue } $errorCountAfter = $global:error.Count $numberOfNewErrors = $errorCountAfter - $errorCountBefore if ($numberOfNewErrors -gt 0) { $global:error.RemoveRange(0, $numberOfNewErrors) } if ($null -ne $os) { $osLastBootUpTime = $os.LastBootUpTime $osInstallTime = $os.InstallDate Trace-Execution "$nodeName attempting to create PS session as '$($RemoteCIMCredential.UserName)'" # Try the node name first because in some scenario IP is not trusted try { $session = Microsoft.PowerShell.Core\New-PSSession -ComputerName $nodeName -Credential $RemoteCIMCredential if ($null -eq $session) { $session = Microsoft.PowerShell.Core\New-PSSession -ComputerName $nodeIP -Credential $RemoteCIMCredential } } catch { Trace-Execution "$nodeName failed to created PS session with error: $($PSItem.Exception.Message)" } } else { Trace-Execution "$nodeName could not query Win32_OperatingSystem" } } catch { if ($null -ne $session) { Trace-Execution "Caught exception after session to new OS was created: $($PSItem.Exception.Message)" Remove-PSSession $session -ErrorAction SilentlyContinue $session = $null } $global:error.RemoveAt(0) } if ($null -ne $session) { Trace-Execution "$nodeName PS session was created - looking for $DeploymentID.txt file on this node" Trace-Execution "Waiting for 5 min after successful network connection so that setupcomplete can converge on the physical machine." Start-Sleep -Seconds 300 $isNewDeployment = Invoke-Command -Session $session -ScriptBlock { Test-Path -Path "$env:SystemDrive\CloudBuilderTemp\$($using:DeploymentID).txt" } if ($true -eq $isNewDeployment) { Trace-Execution "$nodeName has finished OS deployment. Boot time reported by the node - $($osLastBootUpTime.ToString())" $failedNodeNames = $failedNodeNames | Where-Object Name -ne $nodeName } else { Trace-Warning "$nodeName has booted up to an old OS installed on $($osInstallTime.ToString())" if (-not $IgnoreOldOSCheckResult) { $failedNodeNames += $nodeName } } Trace-Execution "$nodeName has completed bare metal deployment" $remainingNodes = $remainingNodes | Where-Object Name -ne $nodeName } else { Trace-Execution "$nodeName could not establish Powershell session using IP $nodeIP or name" Trace-Execution "$nodeName attempt to map network drive to C`$" try { $fileFound = $false $netShare = New-PSDrive -Name $nodeName -PSProvider FileSystem -Root "\\$nodeName\C`$" -Credential $RemoteCIMCredential if ($null -ne $netShare) { $fileFound = Test-Path -Path "$($nodeName):\CloudBuilderTemp\$($DeploymentID).txt" } } catch { Trace-Execution "$nodeName failed to map network drive with error: $($PSItem.Exception.Message)" } if ($true -eq $fileFound) { Trace-Execution "$nodeName has completed bare metal deployment" $remainingNodes = $remainingNodes | Where-Object Name -ne $nodeName } } if ($null -ne $cimSession) { Remove-CimSession $cimSession } else { Trace-Execution "$nodeName could not establish CIM session using IP $nodeIP or name" } if ($null -ne $netShare) { Remove-PSDrive -Name $nodeName -ErrorAction SilentlyContinue $netShare = $null } } else { Trace-Execution "$nodeName did not respond to ping with IP $nodeIP" } } if ($remainingNodes.Count -eq 0) { Trace-Execution "All nodes have completed bare metal deployment" $wait = $false } else { Trace-Execution "Still waiting for $($remainingNodes.Count) node(s) to complete" Start-Sleep -Seconds 60 } } else { Trace-Execution "Timed out waiting for all nodes to complete bare metal deployment" Trace-Execution "$($remainingNodes.Count) node(s) did not respond in time" $wait = $false } } if ($remainingNodes.Count -eq 0) { $remainingNodeNames = @() } else { $remainingNodeNames = @($remainingNodes | ForEach-Object Name) } $totalBmdWaitTimeMinutes = [int]($StopTime - $StartTime).TotalMinutes if (($failedNodeNames.Count + $remainingNodeNames.Count) -gt 0) { [string]$badNodes = ($failedNodeNames + $remainingNodeNames) -join ',' Trace-Error "Bare metal deployment failed to complete in $totalBmdWaitTimeMinutes minutes - $badNodes." } else { Trace-Execution "Bare metal deployment has completed on the target nodes." } } <# .SYNOPSIS While a host is being restarted and brought back online, this function is used to select a different host on the same cluster to perform remote PowerShell operations on. #> function Get-HostNameForRemoting { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [Parameter(Mandatory=$true)] [string] $HostBeingRebooted ) $ErrorActionPreference = "Stop" $clusterName = Get-NodeRefClusterId -Parameters $Parameters -RoleName "BareMetal" -NodeName $HostBeingRebooted $availableHosts = Get-ActiveClusterNodes -Parameters $Parameters -ClusterName $clusterName $availableHosts = $availableHosts | Where-Object { $_ -ne $HostBeingRebooted } if ($availableHosts.Count -eq 0) { Trace-Error "There are no available hosts on which to perform remote operations." } else { Trace-Execution "Selected host $($availableHosts[0])." return $availableHosts[0] } } <# .SYNOPSIS This function is used to do cluster aware update based on different Cau Plugins. #> function Invoke-CauRunHelper { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [System.Collections.Hashtable] $cauParams ) $ErrorActionPreference = "Stop" Trace-Execution "Start invoke caurun" Invoke-CauRun @cauParams } Export-ModuleMember -Function Start-PXEBoot Export-ModuleMember -Function Restart-Node Export-ModuleMember -Function Stop-DeployingMachines Export-ModuleMember -Function Start-DeployingMachines Export-ModuleMember -Function Add-DeployingMachineNetworkReservation Export-ModuleMember -Function Remove-MachineNetworkReservation Export-ModuleMember -Function Wait-RemoteCimInitializedOneNodeBareMetal Export-ModuleMember -Function Wait-ForISOImageBareMetal Export-ModuleMember -Function Get-HostNameForRemoting Export-ModuleMember -Function Get-ProductInfo Export-ModuleMember -Function Invoke-CauRunHelper # SIG # Begin signature block # MIInwgYJKoZIhvcNAQcCoIInszCCJ68CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCD0NXoJNEboxNGW # ZU2YnPu9GkeV561/p/S/GGxHF9S7X6CCDXYwggX0MIID3KADAgECAhMzAAADTrU8 # esGEb+srAAAAAANOMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMwMzE2MTg0MzI5WhcNMjQwMzE0MTg0MzI5WjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDdCKiNI6IBFWuvJUmf6WdOJqZmIwYs5G7AJD5UbcL6tsC+EBPDbr36pFGo1bsU # p53nRyFYnncoMg8FK0d8jLlw0lgexDDr7gicf2zOBFWqfv/nSLwzJFNP5W03DF/1 # 1oZ12rSFqGlm+O46cRjTDFBpMRCZZGddZlRBjivby0eI1VgTD1TvAdfBYQe82fhm # WQkYR/lWmAK+vW/1+bO7jHaxXTNCxLIBW07F8PBjUcwFxxyfbe2mHB4h1L4U0Ofa # +HX/aREQ7SqYZz59sXM2ySOfvYyIjnqSO80NGBaz5DvzIG88J0+BNhOu2jl6Dfcq # jYQs1H/PMSQIK6E7lXDXSpXzAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUnMc7Zn/ukKBsBiWkwdNfsN5pdwAw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMDUxNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAD21v9pHoLdBSNlFAjmk # mx4XxOZAPsVxxXbDyQv1+kGDe9XpgBnT1lXnx7JDpFMKBwAyIwdInmvhK9pGBa31 # TyeL3p7R2s0L8SABPPRJHAEk4NHpBXxHjm4TKjezAbSqqbgsy10Y7KApy+9UrKa2 # kGmsuASsk95PVm5vem7OmTs42vm0BJUU+JPQLg8Y/sdj3TtSfLYYZAaJwTAIgi7d # hzn5hatLo7Dhz+4T+MrFd+6LUa2U3zr97QwzDthx+RP9/RZnur4inzSQsG5DCVIM # pA1l2NWEA3KAca0tI2l6hQNYsaKL1kefdfHCrPxEry8onJjyGGv9YKoLv6AOO7Oh # JEmbQlz/xksYG2N/JSOJ+QqYpGTEuYFYVWain7He6jgb41JbpOGKDdE/b+V2q/gX # UgFe2gdwTpCDsvh8SMRoq1/BNXcr7iTAU38Vgr83iVtPYmFhZOVM0ULp/kKTVoir # IpP2KCxT4OekOctt8grYnhJ16QMjmMv5o53hjNFXOxigkQWYzUO+6w50g0FAeFa8 # 5ugCCB6lXEk21FFB1FdIHpjSQf+LP/W2OV/HfhC3uTPgKbRtXo83TZYEudooyZ/A # Vu08sibZ3MkGOJORLERNwKm2G7oqdOv4Qj8Z0JrGgMzj46NFKAxkLSpE5oHQYP1H # tPx1lPfD7iNSbJsP6LiUHXH1MIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGaIwghmeAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAANOtTx6wYRv6ysAAAAAA04wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIDyDubBbNjKcac35c9njzJLp # b+TlCMDwT8eiEKZZxaAfMEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAKh3JsvXyj55qSrzrDOSAb9GDXQggZm/1+tsnZvqf3tV/bRdJtaSEF/JJ # /GeIh5Kvn0xMxOYApanSiMee2VEGE0pTk5iEQezWI+bRKyHCdgPW4uBhKCh552GJ # AyXqVUpwAFxOgP0AVe9ICp5znaEMVcvUzbJ8VAATzWBSLAZxdusN2WKWD4oQrI1c # k+75COZEFU4rmEoiuTmtNz+VBjRXLrvuuZ7uDOgERT1CAjfk8Ahh/+Y0G22gu/2U # PFUEsw/7FdXN0RUjvzeK/2y3IMGKHM54M5qXk89k64rfEkm3HZqXiQLDrT2HmRfc # popOEkN20QSA5IkGAacdmiX+MsoToaGCFywwghcoBgorBgEEAYI3AwMBMYIXGDCC # FxQGCSqGSIb3DQEHAqCCFwUwghcBAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq # hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCCo56HtgzXUmH4rufz0FPGyCYzxzVFXJ+uWN9VZefr1dAIGZMmJekYf # GBMyMDIzMDgwMzA4MjExNi42MjZaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO # OjJBRDQtNEI5Mi1GQTAxMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT # ZXJ2aWNloIIRezCCBycwggUPoAMCAQICEzMAAAGxypBD7gvwA6sAAQAAAbEwDQYJ # KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjIw # OTIwMjAyMTU5WhcNMjMxMjE0MjAyMTU5WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl # cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjoyQUQ0LTRC # OTItRkEwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIaiqz7V7BvH7IOMPEeDM2Uw # CpM8LxAUPeJ7Uvu9q0RiDBdBgshC/SDre3/YJBqGpn27a7XWOMviiBUfMNff51Nx # KFoSX62Gpq36YLRZk2hN1wigrCO656z5pVTjJp3Q8jdYAJX3ruJea3ccfTgxAgT3 # Uv/sP4w0+yZAYa2JZalV3MBgIFi3VwKFA4ClQcr+V4SpGzqz8faqabmYypuJ35Zn # 8G/201pAN2jDEOu7QaDC0rGyDdwSTVmXcHM46EFV6N2F69nwfj2DZh74gnA1DB7N # FcZn+4v1kqQWn7AzBJ+lmOxvKrURlV/u19Mw1YP+zVQyzKn5/4r/vuYSRj/thZr+ # FmZAUtTAacLzouBENuaSBuOY1k330eMp8nndSNUsUjj/nn7gcdFqzdQNudJb+Xxm # Rwi9LwjA0/8PlOsKTZ8Xw6EEWPVLfNojSuWpZMTaMzz/wzSPp5J02kpYmkdl50lw # yGRLO5X7iWINKmoXySdQmRdiGMTkvRStXKxIoEm/EJxCaI+k4S3+BWKWC07EV5T3 # UG7wbFb4LfvgbbaKM58HytAyjDnO9fEi0vrp8JFTtGhdtwhEEkraMtGVt+CvnG0Z # lH4mvpPRPuJbqE509e6CqmHwzTuUZPFMFWvJn4fPv0d32Ws9jv2YYmE/0WR1fULs # +TxxpWgn1z0PAOsxSZRPAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQU9Jtnke8NrYSK # 9fFnoVE0pr0OOZMwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD # VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j # cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG # CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw # MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD # CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBANjnN5JqpeVShIrQ # IaAQnNVOv1cDEmCkD6oQufX9NGOX28Jw/gdkGtMJyagA0lVbumwQla5LPhBm5LjI # UW/5aYhzSlZ7lxeDykw57wp2AqoMAJm7bXcXtJt/HyaRlN35hAhBV+DmGnBIRcE5 # C2bSFFY3asD50KUSCPmKl/0NFadPeoNqbj5ZUna8VAfMSDsdxeyxjs8r/9Vpqy8l # gIVBqRrXtFt6n1+GFpJ+2AjPspfPO7Y+Y/ozv5dTEYum5eDLDdD1thQmHkW8s0BB # DbIOT3d+dWdPETkf50fM/nALkMEdvYo2gyiJrOSG0a9Z2S/6mbJBUrgrkgPp2HjL # kycR4Nhwl67ehAhWxJGKD2gRk88T2KKXLiRHAoYTZVpHbgkYLspBLJs9C77ZkuxX # uvIOGaId7EJCBOVRMJygtx8FXpoSu3jWEdau0WBMXxhVAzEHTu7UKW3Dw+KGgW7R # Rlhrt589SK8lrPSvPM6PPnqEFf6PUsTVO0bOkzKnC3TOgui4JhlWliigtEtg1SlP # MxcdMuc9uYdWSe1/2YWmr9ZrV1RuvpSSKvJLSYDlOf6aJrpnX7YKLMRoyKdzTkcv # Xw1JZfikJeGJjfRs2cT2JIbiNEGK4i5srQbVCvgCvdYVEVZXVW1Iz/LJLK9XbIkM # MjmECJEsa07oadKcO4ed9vY6YYBGMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ # mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh # dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 # WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB # BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK # NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg # fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp # rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d # vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 # 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR # Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu # qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO # ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb # oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 # bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t # AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW # BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb # UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku # aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA # QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 # VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu # bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw # LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt # MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q # XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 # U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt # I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis # 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp # kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 # sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e # W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ # sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 # Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 # dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ # tB1VM1izoXBm8qGCAtcwggJAAgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh # bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjoy # QUQ0LTRCOTItRkEwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUA7WSxvqQDbA7vyy69Tn0wP5BGxyuggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF # AOh1WVgwIhgPMjAyMzA4MDMwNjM4MTZaGA8yMDIzMDgwNDA2MzgxNlowdzA9Bgor # BgEEAYRZCgQBMS8wLTAKAgUA6HVZWAIBADAKAgEAAgICAAIB/zAHAgEAAgIUgzAK # AgUA6Haq2AIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB # AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAGcXgXW302MhVOay # aiGH+r324MvYMuBI1aPvSuKS/jajyJBITyUykpiklbspPzfzDjbmL1WdRbOlNYmI # UNdZyigqzOtxgwozcMTAzY4oRmvVw18+YvSjn3h6RfpTtHdUF3CQM+N/fFhL7KzH # pHiI/j8KtZTvZhyhM6ECpGX5eatsMYIEDTCCBAkCAQEwgZMwfDELMAkGA1UEBhMC # VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV # BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp # bWUtU3RhbXAgUENBIDIwMTACEzMAAAGxypBD7gvwA6sAAQAAAbEwDQYJYIZIAWUD # BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B # CQQxIgQgxBmPHvl+jbOE/ktHKGhzdPN0m+TTzLHbCbGvqXy6zVkwgfoGCyqGSIb3 # DQEJEAIvMYHqMIHnMIHkMIG9BCCD7Q2LFFvfqeDoy9gpu35t6dYerrDO0cMTlOIo # mzTPbDCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u # MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp # b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB # scqQQ+4L8AOrAAEAAAGxMCIEIGX1ph5ViWF8/JYiY4ZGgxxhjfmvwkkkVnC1JI7N # Su/OMA0GCSqGSIb3DQEBCwUABIICAAXDc7ofrJ79EC1qu6/rjfIuaLxltai+Ytb8 # uV3q5y4C5hIxplixL8HbhLHIKj5pCBZ907KXIQogaGZipTpEQLydKOTByDREj6yq # S9rcAIOvmQsfO/OdhY/YeYfgow3CY0qKsaIso9V0roPiWRCT8d9BwRHm4TDFUC9A # NQjMMd5dBT8MuWBw3cbtg0di+94fg7i5n4rfw+00qGduDPvIt/58BC9dLpgMiyVo # Bi1mQBpZw0tGDx6QOsInW5NiW9hGOKBKeHSXmNuJBtrH0XAqaMBgvrA1+fBYqUGI # vDfNPYKvJY//vVXTSG45pxBO25jvkCsHG/deREU1THjIJdjjUg3j/8C69NVd4VFc # nwisq28nBRd8nom9qYw7vlN+VmMFUHRDw9WGYu8hchLW7xbpbb3XtPN/+CzdcbUC # 752YVw+HgU0B+iQp/uk4dn1qYU5vYiGJLWmFKw6Da2OMAPKoo41XNsRdgDXCDO7a # Xyc+LZF3rn9Lt1HbWHaQNYTRaQiaoWvqjJqPdF+1jUTi3leFAznK4Bic5cy3SOrA # ZpqB36aSAH1TYlpjkF5VyO76KfaROLZft8IcEk6lpuvFQbBIKGiu+bV4LBwuDdid # SgS3en/17dpn7Pq0+fjPOKZ9w6nAj4bdV2oaSpJlRUu3uf6aLSKpovkmfeZ80mLV # TTJyLCsq # SIG # End signature block |