Obs/bin/ObsDep/content/Powershell/ObservabilityConfig.psm1
|
<###################################################
# # # Copyright (c) Microsoft. All rights reserved. # # # ##################################################> class ObservabilityConfig { static RegisterObservabilityEventSource([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] foreach ($eventSource in $eventSources) { $eventSourceRootName = $eventSource.EventSourceRootName $eventSourceLogName = $eventSource.EventSourceLogName $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes $eventSourcePublisher = $eventSource.EventSourcePublisher Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog" try { # Register eventsource on hosts Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" $manifestDllPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.dll" $manifestPath = "$observabilityNugetPath\lib\net472\$Using:eventSourceRootName.man" # Grant read only access to manifest files Trace-Execution "Granting read only access to $manifestPath" icacls $manifestPath /grant Everyone:R Trace-Execution "Granting read only access to $manifestDllPath" icacls $manifestDllPath /grant Everyone:R $publisherList = wevtutil ep if ($publisherList.Contains($Using:eventSourcePublisher)) { Trace-Execution "Publisher $Using:eventSourcePublisher already exists. Uninstalling." wevtutil uninstall-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath" Trace-Execution "Successfully uninstalled publisher $Using:eventSourcePublisher." } # Register the EventSource with Windows Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath" wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath" Trace-Execution "wevtutil setting log size to $Using:eventSourceLogSizeInBytes" wevtutil set-log $Using:eventSourceLogName /MaxSize:"$Using:eventSourceLogSizeInBytes" } } catch { Trace-Execution "Registering Observability EventSource failed with error: $_" throw $_ } Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded." } } static CreateObservabilityVolume([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityVolumeLabel $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveSizeInBytes $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) $PhysicalDriveLetter = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.PhysicalDriveLetter $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName $volumeLocation = "${PhysicalDriveLetter}:" $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName Write-ObservabilityVolumeCreationStartTelemetry ` -ComputerNames ($hostIps -join " ") ` -VolumeFilePath $volumePath ` -VolumeAccessPath $driveAccessPath ` -VolumeLabel $volumeLabel ` -VolumeSize $volumeSize $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) # Add Observability Volume to each Host $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $VolumePath, [string] [parameter(Mandatory=$true)] $AccessPath, [string] [parameter(Mandatory=$true)] $VolumeLocation, [string] [Parameter(Mandatory=$true)] $VolumeLabel, [string] [Parameter(Mandatory=$true)] $VolumeSize ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" if (Test-Path -Path $AccessPath) { Write-ObservabilityLog "Folder $AccessPath already present. Skipping Observability volume creation." } else { Add-ObservabilityVolumeWithRetry -Path $VolumePath -AccessPath $AccessPath -VolumeLabel $VolumeLabel -Size $VolumeSize -StaticSize Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $VolumePath Add-MountVolumeScheduledTask -Path $VolumePath } } $argList = @($volumePath, $driveAccessPath, $volumeLocation, $volumeLabel, $volumeSize) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp Write-ObservabilityLog "Observability Volume Setup Succeeded" } static SetUptimeScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] Write-InfoLog "Obtaining local admin credential..." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Write-InfoLog "Starting Uptime scheduled task Setup" Write-InfoLog "Hosts: $hostNames" $scriptBlock = { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityUptimeHelpers.psm1" $hostName = $env:COMPUTERNAME Write-InfoLog "Starting uptime scheduled task on host $hostName." try { Set-UptimeTaskWithRetry Write-InfoLog "Uptime scheduled task on host $hostName succeeded." } catch { Write-ErrorLog "Uptime scheduled task on host $hostName failed with exception $_ ." throw $_ } } Invoke-Command -ComputerName $hostIps -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp | Out-Null Write-InfoLog "Uptime scheduled task completed." } static SetObservabilityEventTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters, $eventName, $eventScriptBlock) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name Write-InfoLog "Starting $eventName events task Setup" Write-InfoLog "Hosts: $hostNames" $exceptionMessage = "" $scriptBlock = $eventScriptBlock $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] foreach ($hostIp in $hostIps) { Write-InfoLog "Starting $eventName events task on host $hostIp." $result = Invoke-Command -ComputerName $hostIp -ScriptBlock $scriptBlock -Credential $localAdminCredential -Authentication Credssp if($result -eq $true) { Write-InfoLog "$eventName events task on host $hostIp succeeded." } else { Write-ErrorLog "$eventName events task on host $hostIp failed with exception $result." $exceptionMessage += "${hostIp}: $result`n" } } if($exceptionMessage) { throw $exceptionMessage } } static SetCensusEventScheduledTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" $scriptBlock = { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" Set-CensusTaskWithRetry } [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "CensusEvent", $scriptBlock) Write-InfoLog "Census events scheduled task completed." } static SetRegistrationEventOneTimeTask([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" $scriptBlock = { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityCensusHelpers.psm1" Set-RegistrationEventsTaskWithRetry } [ObservabilityConfig]::SetObservabilityEventTask($Parameters, "RegistrationEvents", $scriptBlock) Write-InfoLog "Registration events one-time task completed." } static CreateVolumeFoldersAndPruner([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilitySubFolderConfigFileName $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.Host.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) Write-ObservabilityLog "Starting Observability Volume Folder and Pruner Setup" Write-ObservabilityLog "Hosts: $hostNames" Write-ObservabilityLog "Volume folder mount access path: $driveAccessPath" Write-ObservabilityLog "Observability subfolder folder cleanup threshold: $folderCleanupThresholdPercent" Write-ObservabilityLog "Observability subfolder folder free space threshold: $folderFreeSpaceThresholdPercent" Write-ObservabilityLog "Observability volume purge frequency in minutes: $purgeFolderFrequencyInMinutes" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) # Add Observability Volume to each Host $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AccessPath, [string] [Parameter(Mandatory=$true)] $FolderCleanupThresholdPercent, [string] [Parameter(Mandatory=$true)] $FolderFreeSpaceThresholdPercent, [string] [Parameter(Mandatory=$true)] $PurgeFolderFrequencyInMinutes, [string] [Parameter(Mandatory=$true)] $SubFolderConfigFileName ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" New-VolumeFoldersAndPrunerWithRetry ` -AccessPath $AccessPath ` -CleanupThresholdPercent $FolderCleanupThresholdPercent ` -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent ` -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes ` -SubFolderConfigFileName $SubFolderConfigFileName } $argList = @( $driveAccessPath, $folderCleanupThresholdPercent, $folderFreeSpaceThresholdPercent, $purgeFolderFrequencyInMinutes, $subFolderConfigFileName ) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp Write-ObservabilityLog "Observability subfolder and pruner Setup Succeeded" } # Set registry key HKLM\Software\Microsoft\AzureStack DeviceType to AzureEdge static SetDeviceTypeRegistryKey([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "SetDeviceTypeRegistryKey start." $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { Write-Verbose "Setting HKLM\Software\Microsoft\AzureStack DeviceType registry key to AzureEdge." $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\' if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name 'DeviceType' -PropertyType 'String' -Value "AzureEdge" -Force Write-Verbose "Finished DeviceType Registry key setup on $($env:COMPUTERNAME)" } Write-ObservabilityLog "Succeeded in setting AzureStack DeviceType Registry key." } catch { Write-ObservabilityErrorLog "[Error] DeviceType registry key set up failed with an error : $_ " throw $_ } } # Set up AzureStack environment to enable Autologger and UTC Telemetry via MA # Reg key set up needs to happend upfornt during host configuration static SetUpUtcExporterFeature([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $utcExporterPath = "$($env:SystemRoot)\System32\UtcExporters" # Create UtcExporters folder in system32 if (-not (Test-Path -Path $utcExporterPath)) { New-Item -Path "$($env:SystemRoot)\System32" -ItemType Directory -Name 'UtcExporters' -Force } $GenevaNameSpace = "AEOprdTely" if(Test-IsCIEnv) { $GenevaNameSpace = "AEOppeTely" } Write-ObservabilityLog "Set up the required regkey to enable utc exporter Feature" New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack' -Name 'AllowExporters' -PropertyType 'DWORD' -Value 1 -Force New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters' -Force New-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Force New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'DllPath' -PropertyType 'String' -Value "$($utcExporterPath)\UtcGenevaExporter.dll" -Force New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\exporters\GenevaExporter' -Name 'GenevaNamespace' -PropertyType 'String' -Value $GenevaNameSpace -Force New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Diagnostics\DiagTrack\TestHooks' -Name 'SkipSignatureMitigation' -PropertyType 'DWORD' -Value 1 -Force # Update UTC Service binary Write-ObservabilityLog "Updating UTC service dll" Stop-Service "diagtrack" -Force Start-Sleep -Seconds 10 Write-ObservabilityLog "Copy UTC Exporter dll to $($utcExporterPath)" Copy-Item -Path "$observabilityNugetPath\lib\net472\UtcGenevaExporter.dll" -Destination $utcExporterPath -Force Start-Service "diagtrack" Write-ObservabilityLog "Finished UTC Exporter setup on $($env:COMPUTERNAME)" } Write-ObservabilityLog "Set up AzureStack environment to enable Autologger and UTC Telemetry via MA succeeded." } catch { Write-ObservabilityErrorLog "[Error] UTC Exporter set up failed with an error : $_ " throw $_ } } # Install VC Runtime on all hosts static InstallVcRuntime([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] Write-ObservabilityLog "Starting installation of X64 VC Redistributable." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "[Starting] installation of VC Redistributable on host $($env:COMPUTERNAME)" $vcRedistFilePath = "$observabilityNugetPath\content\VS17\VC_redist.x64.exe" if(Test-Path -Path $vcRedistFilePath) { Start-Process -File $vcRedistFilePath -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow } else { $errMsg = "[ERROR] VC Redistributable [$vcRedistilePath] not found on host $($env:COMPUTERNAME)" Write-ObservabilityErrorLog $errMsg throw $errMsg } Write-ObservabilityLog "[Finished] installation of VC Redistributable on host $($env:COMPUTERNAME)" } } catch { Write-ObservabilityErrorLog "[Failed] installation of VC Redistributable: $_ " throw $_ } Write-ObservabilityLog "Finished installation of X64 VC Redistributable." } static CreateObservabilityVolumeOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Starting observability volume setup on NC VMs" $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name $volumeFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveFileName $volumeLabel = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityVolumeLabel $volumeSize = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveSizeInBytes $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) $folderCleanupThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaCleanupThresholdInPercent $folderFreeSpaceThresholdPercent = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.FolderQuotaFreeSpaceThresholdInPercent $purgeFolderFrequencyInMinutes = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.PurgeFolderFrequencyInMinutes $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $VolumePath, [string] [parameter(Mandatory=$true)] $HostComputerName, [string] [parameter(Mandatory=$true)] $AccessPath, [string] [parameter(Mandatory=$true)] $FolderCleanupThresholdPercent, [string] [parameter(Mandatory=$true)] $FolderFreeSpaceThresholdPercent, [string] [Parameter(Mandatory=$true)] $PurgeFolderFrequencyInMinutes, [string] [parameter(Mandatory=$true)] $SubFolderConfigFileName, [PSCredential] [parameter(Mandatory=$true)] $Credential ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $remoteVolumePath = "\\$HostComputerName\$VolumePath" $remoteVolumePath = $remoteVolumePath -replace ":", "$" Add-VolumeAccessPath -AccessPath $AccessPath -VolumePath $remoteVolumePath Add-MountVolumeScheduledTask -Path $remoteVolumePath -Credential $Credential New-VolumeFoldersAndPrunerWithRetry ` -AccessPath $AccessPath ` -CleanupThresholdPercent $FolderCleanupThresholdPercent ` -FreeSpaceThresholdPercent $FolderFreeSpaceThresholdPercent ` -PurgeFolderFrequencyInMinutes $PurgeFolderFrequencyInMinutes ` -SubFolderConfigFileName $SubFolderConfigFileName Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName } if(-not($ncVMs -is [System.Array])) # Only 1 VM exists { $ncVMs = @($ncVMs) } foreach($ncVM in $ncVMs) { $hostComputerName = $env:COMPUTERNAME Write-ObservabilityLog "Starting observability volume onboarding for $ncVM." try { Get-VM $ncVM | Out-Null # Test that VM is accessbile } catch { Write-ObservabilityLog "$ncVM not found on $hostComputerName. No op." continue } $vmHardDrivePath = (Get-VMHardDiskDrive $ncVM).Path if($vmHardDrivePath -is [System.Array]) { $vmHardDrivePath = $vmHardDrivePath[0] } $volumeLocation = Split-Path $vmHardDrivePath -Parent $volumePath = Join-Path -Path $volumeLocation -ChildPath $volumeFileName if(Test-Path $driveAccessPath) { Write-Observability "Folder $driveAccessPath already created. Skipping Observability Volume creation." } else { Write-ObservabilityLog "Adding observability volume at volume path $volumePath" Add-ObservabilityVolumeWithRetry -Path $volumePath -AccessPath $driveAccessPath -VolumeLabel $volumeLabel -Size $volumeSize -StaticSize if((Get-DiskImage $volumePath).Attached) { Dismount-VHD $volumePath } $argList = @( $volumePath, $hostComputerName, $driveAccessPath, $folderCleanupThresholdPercent, $folderFreeSpaceThresholdPercent, $purgeFolderFrequencyInMinutes, $subFolderConfigFileName, $domainUserCredential ) Invoke-Command ` -ComputerName $ncVM ` -ScriptBlock $scriptBlock ` -Credential $domainUserCredential ` -Authentication Credssp ` -ArgumentList $argList Write-ObservabilityLog "Finished observability volume onboarding for $ncVM." } } } static RegisterObservabilityEventSourceOnNC([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $eventSources = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EventSources.EventSource $ncVMs = $Parameters.Roles["NC"].PublicConfiguration.Nodes.Node.Name $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) foreach ($eventSource in $eventSources) { $eventSourceRootName = $eventSource.EventSourceRootName $eventSourceLogName = $eventSource.EventSourceLogName $eventSourceLogSizeInBytes = $eventSource.EventSourceLogSizeInBytes Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog" $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $EventSourceRootName, [string] [parameter(Mandatory=$true)] $EventSourceLogName, [string] [parameter(Mandatory=$true)] $EventSourceLogSizeInBytes ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" $manifestDllPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.dll" $manifestPath = "$observabilityNugetPath\lib\net472\$EventSourceRootName.man" # Grant read only access to manifest files Trace-Execution "Granting read only access to $manifestPath" icacls $manifestPath /grant Everyone:R Trace-Execution "Granting read only access to $manifestDllPath" icacls $manifestDllPath /grant Everyone:R # Register the EventSource with Windows Trace-Execution "wevtutil installing manifest $manifestPath with resourceFilePath and messageFilePath $manifestDllPath" wevtutil install-manifest $manifestPath /resourceFilePath:"$manifestDllPath" /messageFilePath:"$manifestDllPath" Trace-Execution "wevtutil setting log size to $EventSourceLogSizeInBytes" wevtutil set-log $EventSourceLogName /MaxSize:"$EventSourceLogSizeInBytes" } try { # Register eventsource on hosts Invoke-Command ` -ComputerName $ncVMs ` -ScriptBlock $scriptBlock ` -Credential $domainUserCredential ` -Authentication Credssp ` -ArgumentList @($eventSourceRootName, $eventSourceLogName, $eventSourceLogSizeInBytes) } catch { Trace-Execution "Registering Observability EventSource failed with error: $_" throw $_ } Trace-Execution "Registering Observability EventSource $eventSourceLogName with Windows EventLog suceeded." } } static SetFolderQuotas ([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $subFolderConfigFileName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilitySubFolderConfigFileName $driveAccessPath = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.Volumes.NC.ObservabilityDriveAccessPath.Path $driveAccessPath = $Global:ExecutionContext.InvokeCommand.ExpandString($driveAccessPath) Write-ObservabilityLog "Starting observability volume fsrm quota setup on $hostNames" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AccessPath, [string] [parameter(Mandatory=$true)] $SubFolderConfigFileName ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Set-FolderQuotas -AccessPath $AccessPath -SubFolderConfigFileName $SubFolderConfigFileName } try { Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock $scriptBlock ` -ArgumentList @($driveAccessPath, $subFolderConfigFileName) } catch { Write-ObservabilityErrorLog "[Failed] setup of FSRM Quotas: $_ " throw $_ } } static [PSCredential] GetLocalCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Obtaining local admin credential..." $localAdminCredential = $null try { $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $localAdmin = $securityInfo.LocalUsers.User | Where-Object Role -eq $Parameters.Configuration.Role.PrivateInfo.Accounts.BuiltInAdminAccountID $localAdminCredential = $Parameters.GetCredential($localAdmin.Credential) } catch { Write-ObservabilityErrorLog "Failed to obtain local admin credentials: $_ " throw $_ } Write-ObservabilityLog "Local admin credential obtained." if ($localAdminCredential.UserName -eq "removed") { Write-ObservabilityLog "The local admin credential obtained has been removed. Using domain credential instead." return [ObservabilityConfig]::GetDomainCredential($Parameters) } return $localAdminCredential } static [PSCredential] GetDomainCredential([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Obtaining domain user credentials..." $domainUserCredential = $null try { $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $physicalMachinesRole = $Parameters.Configuration.Role $domainName = $Parameters.Roles.Domain.PublicConfiguration.PublicInfo.DomainConfiguration.Fqdn $domainUser = $securityInfo.DomainUsers.User | Where-Object Role -EQ $physicalMachinesRole.PrivateInfo.Accounts.DomainUserAccountID $domainUserCredential = $Parameters.GetCredential($domainUser.Credential) # Making sure that the credential is domain qualified, PowerShell Direct require credential to be domain qualified if($($domainUserCredential.GetNetworkCredential().Domain) -eq "") { $domainUserCredential = New-Object -TypeName PSCredential -ArgumentList "$domainName\$($domainUserCredential.UserName)", $($domainUserCredential.Password) } Write-ObservabilityLog "Domain user credentials obtained." } catch { Write-ObservabilityErrorLog "Failed to obtain domain user credentials: $_ " throw $_ } return $domainUserCredential } static InstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $exceptionMessage = "" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] try { Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation start." $deploymentLaunchType = Get-DeploymentLaunchype if ($deploymentLaunchType -eq "CloudDeployment") { Write-ObservabilityLog "$env:COMPUTERNAME This stamp was launched via CloudDeployment. Skipping Bootstrap Observability installation." return } $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $cloudId = $cloudRole.PublicInfo.CloudId $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare $proxyUrl = [ObservabilityConfig]::GetProxyUrl($Parameters) $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AgentMsiPath, [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $CloudId, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$true)] $StampId ) $errorMessage = "" $setHostNICIPAddressLogFile = Join-Path -Path $env:systemdrive -ChildPath $LogFileRelativePath Start-Transcript -Append -Path $setHostNICIPAddressLogFile try { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $ResourceName = Get-ArcResourceName Write-ObservabilityLog "$env:COMPUTERNAME Going to install Arc-for-server agent..." Install-ArcForServerAgent ` -AgentMsiPath $AgentMsiPath ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -ResourceGroupName $ResourceGroupName ` -EnvironmentName $EnvironmentName ` -Region $Region ` -ResourceName $ResourceName ` -ProxyUrl $using:proxyUrl | Out-Null Write-ObservabilityLog "$env:COMPUTERNAME Going to download Observability Extension..." Install-ArcForServerExtensions ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -AccountId $AccountId ` -ResourceGroupName $ResourceGroupName ` -ResourceName $ResourceName ` -EnvironmentName $EnvironmentName ` -Region $Region | Out-Null $hostNameHash = Get-Sha256Hash -ClearString (hostname) $nodeId = "$($StampId)-$hostNameHash" Write-BootstrapNodeIdAndHardwareIdHashTelemetry -BootstrapNodeId $nodeId } catch{ $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } finally { Write-ArcForServerInstallationStopTelemetry ` -ComputerName $env:COMPUTERNAME ` -Message "Bootstrap Observability installation end" ` -ExceptionDetails $errorMessage } } $argList = @( $arcForServerMsiFilePath, $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.TenantId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $cloudId, $registrationParams.EnvironmentName, $registrationParams.Region, $registrationParams.ResourceName ) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp } catch { $errMsg = "Bootstrap Observability installation failed with $_" $exceptionMessage += $errMsg Write-ObservabilityErrorLog $errMsg throw } Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability installation end." } static InstallRemoteSupportArcExtension([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $exceptionMessage = "" $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] try { Write-ObservabilityLog "$env:COMPUTERNAME Remote Support Arc installation start." $deploymentLaunchType = Get-DeploymentLaunchype if ($deploymentLaunchType -eq "CloudDeployment") { Write-ObservabilityLog "$env:COMPUTERNAME This stamp was launched via CloudDeployment. Skipping RemoteSupport Arc installation." return } $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$true)] $StampId ) $errorMessage = "" $setHostNICIPAddressLogFile = Join-Path -Path $env:systemdrive -ChildPath $LogFileRelativePath Start-Transcript -Append -Path $setHostNICIPAddressLogFile try { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityConstants.psm1" $ResourceName = Get-ArcResourceName Write-ObservabilityLog "$env:COMPUTERNAME Going to download Remote Support extension..." Install-ArcForServerExtensions ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -AccountId $AccountId ` -ResourceGroupName $ResourceGroupName ` -ResourceName $ResourceName ` -EnvironmentName $EnvironmentName ` -Region $Region ` -Type $ObservabilityConfigConstants.RemoteSupportExtensionType | Out-Null # Bring extension out of listener mode if it is StartArcRemoteSupportAgent } catch { $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } finally { Write-ArcForServerInstallationStopTelemetry ` -ComputerName $env:COMPUTERNAME ` -Message "Remote Support Arc extension installation end" ` -ExceptionDetails $errorMessage } } $argList = @( $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.TenantId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $registrationParams.EnvironmentName, $registrationParams.Region, $registrationParams.ResourceName ) Invoke-Command ` -ComputerName $hostIps ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $localAdminCredential ` -Authentication Credssp } catch { $errMsg = "Remote Support Arc Extension installation failed with $_" $exceptionMessage += $errMsg Write-ObservabilityErrorLog $errMsg throw } Write-ObservabilityLog "$env:COMPUTERNAME Remote Support Arc Extension installation end." } static UninstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation start." $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare Write-ObservabilityLog "$env:COMPUTERNAME Arc For Server MSI Path: $arcForServerMsiFilePath" $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $AgentMsiPath, [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $AccountId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $CloudId ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" # Import MiscConstants to remove GMAScenario reg key $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent" $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking $gmaScenarioRegKeyPath = $MiscConstants.GMAScenarioRegKey.Path $gmaScenarioRegKeyName = $MiscConstants.GMAScenarioRegKey.Name if (Test-RegKeyExists -Path $gmaScenarioRegKeyPath -Name $gmaScenarioRegKeyName) { Remove-ItemProperty ` -Path $gmaScenarioRegKeyPath ` -Name $gmaScenarioRegKeyName ` -Force Write-ObservabilityLog "Removed registry key $gmaScenarioRegKeyName at path $gmaScenarioRegKeyPath" } Uninstall-ArcForServerExtensions ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -AccountId $AccountId ` -ResourceGroupName $ResourceGroupName ` -CloudId $CloudId $result = Uninstall-ArcForServerAgent ` -AgentMsiPath $AgentMsiPath ` -AccessToken $AccessToken return $result } # Install Arc for server agent on each Host $exceptionMessage = "" foreach($hostIp in $hostIps) { $argList = @($arcForServerMsiFilePath, $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.AccountId, $registrationParams.ResourceGroupName, $registrationParams.CloudId) $result = Invoke-Command ` -ComputerName $hostIp ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $domainUserCredential ` -Authentication Credssp if ($result -eq $true) { Write-ObservabilityLog "Arc for server agent setup on host $hostIp succeeded." } else { Write-ObservabilityLog "Arc for server agent setup on host $hostIp failed with exception $result." $exceptionMessage += "${hostIp}: $result`n" } } if($exceptionMessage) { throw $exceptionMessage } Write-ObservabilityLog "$env:COMPUTERNAME Bootstrap Observability uninstallation end." } static SetupArcAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" try { Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent update start." $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $arcForServerMsiFilePath = $cloudRole.PublicInfo.DefaultInfraStorageLocations.DefaultLocalShare $errorMessage = "" Write-ObservabilityLog "$env:COMPUTERNAME Going to update Arc-for-server agent..." Update-ArcForServerAgent -AgentMsiPath $arcForServerMsiFilePath | Out-Null } catch{ $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent update end." } static ConnectArcAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $exceptionMessage = "" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } try { Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent connect start." $proxyUrl = [ObservabilityConfig]::GetProxyUrl($Parameters) $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) $scriptBlock = { param ( [string] [Parameter(Mandatory=$true)] $AccessToken, [string] [Parameter(Mandatory=$true)] $SubscriptionId, [string] [Parameter(Mandatory=$true)] $TenantId, [string] [Parameter(Mandatory=$true)] $ResourceGroupName, [string] [Parameter(Mandatory=$true)] $EnvironmentName, [string] [Parameter(Mandatory=$true)] $Region, [string] [Parameter(Mandatory=$true)] $StampId ) $errorMessage = "" try { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $ResourceName = Get-ArcResourceName Write-ObservabilityLog "$env:COMPUTERNAME Going to connect Arc-for-server agent..." Connect-ArcForServerAgent ` -AccessToken $AccessToken ` -SubscriptionId $SubscriptionId ` -TenantId $TenantId ` -ResourceGroupName $ResourceGroupName ` -EnvironmentName $EnvironmentName ` -Region $Region ` -ResourceName $ResourceName ` -ProxyUrl $using:proxyUrl | Out-Null } catch{ $errorMessage = $PSItem.ToString() Write-ObservabilityErrorLog $errorMessage throw $PSItem.Exception.Message } } $argList = @( $registrationParams.ArmAccessToken, $registrationParams.SubscriptionId, $registrationParams.TenantId, $registrationParams.ResourceGroupName, $registrationParams.EnvironmentName, $registrationParams.Region, $registrationParams.ResourceName ) $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Invoke-Command ` -Credential $localAdminCredential ` -Authentication Credssp ` -ComputerName $hostNames ` -ScriptBlock $scriptBlock ` -ArgumentList $argList } catch { $exceptionMessage += "ArcAgent connect failed with $_" Write-ObservabilityErrorLog $exceptionMessage throw } Write-ObservabilityLog "$env:COMPUTERNAME Arc Agent connect end." } <# .SYNOPSIS Evaluation function to identify what type of GMA configuration action to use. Following are the cases involved during evaluation : case 1 : If the interface is invoked in case of Deployment then run configuration on all nodes. case 2 : If the interface is invoked in case of AddNode or RepairNode then run configuration on only that node. .EXAMPLE [GMATenantJson]::EvaluateGMAConfigurationType($Parameters) .PARAMETER Parameters The ECE role parameters for this interface. #> static [CloudEngine.Actions.ConditionalActionDescription] EvaluateListenerModeConfigurationType([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $ErrorActionPreference = "Stop" $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" # Evalute for ArcA deployment if (Test-IsArcADeployment -Parameters $Parameters) { Write-ObservabilityLog "EvaluateListenerModeConfigurationType: Skip executing SetWatchdogToListenerMode for ArcA deployment" -Verbose return [CloudEngine.Actions.ConditionalActionDescription]::CreateNegative() } ## For classes in PS, we need to use the call stack to get the current executing function. $functionName = $(Get-PSCallStack)[0].FunctionName try { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $cloudRolePath = "ObservabilityConfig" $listenerModeOnAllNodes = "SetListenerModeOnAllNodes" $listenerModeOnOneNode = "SetListenerModeOnOneNode" $onAllNodesAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnAllNodes) $onOneNodeAction = [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction($cloudRolePath, $listenerModeOnOneNode) $nodeName = Get-ExecutionContextNodeName -Parameters $Parameters if($null -ne $nodeName) { # In add node or repair node case, ExecutionContextNodeName should have exactly one value. Trace-Execution "Execution context node was determined. Listener Mode will be set on $nodeName only." return $onOneNodeAction } # In regular deployment case, ExecutionContextNodeName will be null. Trace-Execution "No exceution context node was determined. Listener Mode will set on all the nodes." return $onAllNodesAction } catch { Trace-Error "$functionName :Failed to evaluate ListenerMode configuration type on node $($env:COMPUTERNAME) due to following error : $_" throw } } static StopArcExtensionObservabilityAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $hostName = Get-ExecutionContextNodeName -Parameters $Parameters # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIp = $allHosts[$hostName] Invoke-Command ` -ComputerName $hostIp ` -Credential $domainUserCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $ObsAgentServiceName = "AzureStack Observability Agent" $ObsAgentServiceDisplayName = "AzureStack Arc Extension Observability Agent" $stopDelayInSeconds = 10 $service = Get-Service | Where-Object {$_.Name -eq $ObsAgentServiceName -and $_.DisplayName -eq $ObsAgentServiceDisplayName } if($service) { if ($service.Status -ne "Stopped") { Write-ObservabilityLog "Stopping service $ObsAgentServiceName with display name $ObsAgentServiceDisplayName on host $($env:COMPUTERNAME)" $service | Stop-Service Write-ObservabilityLog "Sleeping $stopDelayInSeconds seconds after stopping $ObsAgentServiceName before unregistering." Start-Sleep -Seconds $stopDelayInSeconds } Write-ObservabilityLog "Unregistering service $ObsAgentServiceName." sc.exe delete $ObsAgentServiceName Write-ObservabilityLog "Successfully unregistered service $ObsAgentServiceDisplayName" } else { Write-ObservabilityLog "Service $ObsAgentServiceName not found. Noop." } } } static StopRemSupAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { # Making this no-op since it's going in the same payload as InstallRemoteSupportArcExtension # Will remove the action plan for this in another payload } static SetWatchdogToListenerMode([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $domainUserCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $hostName = Get-ExecutionContextNodeName -Parameters $Parameters # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIp = $allHosts[$hostName] Invoke-Command ` -ComputerName $hostIp ` -Credential $domainUserCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability' Write-ObservabilityLog "Setting $registryPath WatchdogListenerMode registry key to 1 on $($env:COMPUTERNAME)." if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name 'WatchdogListenerMode' -PropertyType 'DWord' -Value 1 -Force Write-ObservabilityLog "Finished setting WatchdogListenerMode Registry key to DWord 1 on $($env:COMPUTERNAME)" $timeLimitInSeconds = 600 $retryPeriod = 10 $elapsed = 0 $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue while($maHostProcess -or $maCoreProcess) { if($maHostProcess) { $pids = $maHostProcess.Id | Out-String Write-ObservabilityLog "MonAgentHost Active Process Ids: `n $pids" } if($maCoreProcess) { $pids = $maCoreProcess.Id | Out-String Write-ObservabilityLog "MonAgentCore Active Process Ids: `n $pids" } if($elapsed -gt $timeLimitInSeconds) { $errMsg = "WatchdogListenerMode has failed to stop Monitoring agent processes after $timeLimitInSeconds." Write-ObservabilityErrorLog $errMsg throw $errMsg } Write-ObservabilityLog "Monitoring Agent processes have not yet stopped. Retrying in $retryPeriod seconds." Start-Sleep -Seconds $retryPeriod $elapsed += $retryPeriod $maHostProcess = Get-Process "MonAgentHost" -ErrorAction SilentlyContinue $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue } Write-ObservabilityLog "MonitoringAgent has been successfully stopped after Watchdog has been set to listener mode." } } static SetGMATenantJsonRegistryKeys([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "SetGMATenantJsonRegistryKeys start" $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIps = $allHosts[$hostNames] $gmaNugetName = "Microsoft.AzureStack.Observability.GenevaMonitoringAgent" $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $gmaNugetName) -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking ## GMA cache folder path $gmaCacheFolderName = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.GMACacheFolderName $gmaCacheFolderPath = Join-Path -Path $env:SystemDrive -ChildPath $gmaCacheFolderName ## Determine GcsEnvironment $gcsEnvironment = "Prod" ## default environment ## Check if the reg key created for CI exists or not, if yes then change the GCSEnvironment to point to PPE. if(Test-IsCIEnv) { $gcsEnvironment = "Ppe" } $envInfoFilePath = "$GmaPackageContentPath\EnvironmentInfo.json" $tenantInfoContent = Get-Content $envInfoFilePath -Raw | ConvertFrom-Json $envInfo = $tenantInfoContent.$GcsEnvironment # Settings from EnvironmentInfo.json $gcsEndpoint = $envInfo.EndPoint $gcsAccount = $envInfo.Account $genevaConfigVersion = $envInfo.ConfigVersion # Settings from ECE $assemblyVersion = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version $registrationSubscriptionId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationSubscriptionId $registrationResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceGroupName $stampId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationResourceName $gcsRegionName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationRegion $clusterName = $Parameters.Roles["Cluster"].PublicConfiguration.Clusters.Node.Name $cloudId = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.CloudId $deviceArmResourceUri = "/Subscriptions/$registrationSubscriptionId/resourceGroups/$registrationResourceGroupName/providers/Microsoft.AzureStackHCI/clusters/$stampId" $osBuild = Get-OSBuildVersion $registrationArcResourceGroupName = $registrationResourceGroupName $arcResourceGroupName = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.RegistrationArcServerResourceGroupName if (![String]::IsNullOrEmpty($arcResourceGroupName)) { Trace-Execution "Using RegistrationArcServerResourceGroupName $arcResourceGroupName for ArcServer resource in Tenant Json" $registrationArcResourceGroupName = $arcResourceGroupName } $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Invoke-Command ` -ComputerName $hostIps ` -Credential $localAdminCredential ` -Authentication Credssp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $gmaPackageContentPath = Join-Path $(Get-ASArtifactPath -NugetName $Using:gmaNugetName) -ChildPath "content" Import-Module "$gmaPackageContentPath\GMATenantJsonHelper.psm1" -DisableNameChecking # Will unset GMA scenario registry path at bootstrap uninstall Set-GMAScenarioRegistryKeyToBootstrap | Out-Null # For Bootstrap Scenario, nodeid is stampid-sha256(hostname) $hostNameHash = Get-Sha256Hash -ClearString (hostname) $nodeId = "$($Using:stampId)-$hostNameHash" $arcAgentResourceId = Get-ArcResourceId -SubscriptionId $Using:registrationSubscriptionId -ResourceGroupName $Using:registrationArcResourceGroupName -CloudId $Using:cloudId $configTypes = @("Telemetry", "Diagnostics", "Health", "Metrics", "Security") foreach($configType in $configTypes) { $envInfo = $Using:envInfo $gcsNameSpace = $envInfo.Namespaces.$configType if ($configType -eq "Metrics") { $gcsNameSpace = Get-MetricsNamespaceRegionMapping -Region $using:gcsRegionName -MetricsNameSpace $envInfo.Namespaces.$configType $envInfo #Set environment variables required for 3P metrics Set-EnvironmentVariablesForMetrics ` -GcsEnvironment $using:gcsEnvironment ` -EnvInfoFilePath $using:envInfoFilePath ` -AssemblyBuildVersion $Using:assemblyVersion ` -ClusterName $Using:clusterName ` -HciResourceUri $Using:deviceArmResourceUri } # cacheLocalPath will be overwritten by Extension Install script because cache location for bootstrap is not known until install time $cacheLocalPath = Join-Path -Path $Using:gmaCacheFolderPath -ChildPath $($configType + "Cache") Set-TenantConfigRegistryKeys ` -ConfigType $configType ` -Version "1.0" ` -GcsAuthIdType "AuthMSIToken" ` -GcsEnvironment $Using:gcsEndpoint ` -GcsGenevaAccount $Using:gcsAccount ` -GcsNamespace $gcsNameSpace ` -GcsRegion $Using:gcsRegionName ` -GenevaConfigVersion $Using:genevaConfigVersion ` -LocalPath $cacheLocalPath ` -DisableUpdate "true" ` -DisableCustomImds "true" ` -MONITORING_AEO_REGION $Using:gcsRegionName ` -MONITORING_AEO_DEVICE_ARM_RESOURCE_URI $Using:deviceArmResourceUri ` -MONITORING_AEO_STAMPID $Using:stampId ` -MONITORING_AEO_CLUSTER_NAME $Using:clusterName ` -MONITORING_AEO_OSBUILD $Using:osBuild ` -MONITORING_AEO_ASSEMBLYBUILD $Using:assemblyVersion ` -MONITORING_AEO_NODEID $nodeId ` -MONITORING_AEO_NODE_ARC_RESOURCE_URI $arcAgentResourceId ` -MONITORING_AEO_CLUSTER_NODE_NAME "%COMPUTERNAME%" } $hostname = hostname Write-ObservabilityLog "SetGMATenantJsonRegistryKeys on $hostname succeeded." } } static GenerateTelemetryFromCachedFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files start." $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $domainCredential = [ObservabilityConfig]::GetDomainCredential($Parameters) $cloudId = $cloudRole.PublicInfo.CloudId # Get host names using execution context $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } $scriptBlock = { param ( [string] [parameter(Mandatory=$true)] $CorrelationId ) $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" $logOrchestratorNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.LogOrchestrator" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $result = Invoke-CachedTelemetryFilesParsing ` -LogOrchestratorNugetPath $logOrchestratorNugetPath ` -CorrelationId $CorrelationId return $result } # Install Arc for server agent on each Host $exceptionMessage = "" foreach($hostName in $hostNames) { $argList = @($cloudId) $result = Invoke-Command ` -ComputerName $hostName ` -ScriptBlock $scriptBlock ` -ArgumentList $argList ` -Credential $domainCredential ` -Authentication Credssp if ($result -eq $true) { Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName succeeded." } else { Write-ObservabilityLog "Generate Telemetry from Cached files on host $hostName failed with exception $result." $exceptionMessage += "${hostName}: $result`n" } } if($exceptionMessage) { throw $exceptionMessage } Write-ObservabilityLog "$env:COMPUTERNAME Generate Telemetry from Cached files end." } static SyncDiagnosticLevel([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Trace-Execution "Getting parameters for Set-AzStackHCI" -Verbose $computerName = Get-Cluster | Select-Object -expand Name $resourceId = Get-AzureStackHCI | Select-Object -expand AzureResourceUri $streamingData = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.StreamingDataClient Trace-Execution "Streaming Data = $streamingData" -Verbose $episodicDataUpload = $Parameters.Roles["ObservabilityConfig"].PublicConfiguration.PublicInfo.EpisodicDataUpload Trace-Execution "Episodic Data Upload = $episodicDataUpload" -Verbose if ($streamingData) { if ($episodicDataUpload) { $diagnosticLevel = "Enhanced" } else { $diagnosticLevel = "Basic" } } else { $diagnosticLevel = "Off" } Trace-Execution "Diagnostic Level will be set to $diagnosticLevel" -Verbose Trace-Execution "Calling GetRegistrationParameters" -Verbose $registrationParams = [ObservabilityConfig]::GetRegistrationParameters($Parameters) Trace-Execution "Calling Set-AzStackHCI. Parameters = ComputerName $computerName, ResourceId $resourceId, DiagnosticLevel $diagnosticLevel, AccountId $($registrationParams.AccountId)" -Verbose Set-AzStackHCI -ComputerName $computerName ` -ResourceId $resourceId ` -DiagnosticLevel $diagnosticLevel ` -ArmAccessToken $registrationParams.ArmAccessToken ` -AccountId $registrationParams.AccountId ` -Confirm:$false ` -Verbose $retryAttempt = 0 $retrySleepTimeInSeconds = 10 $Retries = 5 $success = $false while(-not($success) -and ($retryAttempt -lt $Retries)) { $retryAttempt = $retryAttempt + 1 Trace-Execution "Attempt $retryAttempt of $Retries" -Verbose try { Trace-Execution "Calling Sync-AzureStackHCI" -Verbose Sync-AzureStackHCI -Verbose Trace-Execution "Calling Get-AzureStackHCI" -Verbose $portalDiagnosticLevel = Get-AzureStackHCI | Select-Object -expand DiagnosticLevel if ($portalDiagnosticLevel -eq $diagnosticLevel) { Trace-Execution "Portal diagnostic level was successfully set to $portalDiagnosticLevel" -Verbose $success = $true } else { throw "The Diagnostic Level does not match. Portal was not set to $diagnosticLevel, instead is $portalDiagnosticLevel" } } catch { if ($retryAttempt -lt $Retries) { $exceptionMessage = $_.Exception.Message Trace-Execution "Failure during syncing diagnostic level: '$exceptionMessage'. Retrying." -Verbose } else { Trace-Error "Syncing Diagnostic Level failed with error: $_" } Start-Sleep -Seconds $retrySleepTimeInSeconds } } Trace-Execution "Syncing Diagnostic Level successful. Set to: $diagnosticLevel" -Verbose } static UpdateGMATenantJsonNodeId([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $hostName = Get-ExecutionContextNodeName -Parameters $Parameters # Get Host IPs for all hosts $allHosts = Get-NetworkMgmtIPv4FromECEForAllHosts -Parameters $Parameters # Filter host IPs for only hosts from execution context. This is to cover ScaleOut scenario $hostIp = $allHosts[$hostName] $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $assemblyBuild = "" try { Write-ObservabilityLog "Getting Assembly Build using GetStampVersion" Import-Module EceClient $ececlient = Create-ECEClusterServiceClient $assemblyBuild = $ececlient.GetStampVersion().GetAwaiter().GetResult() if ($assemblyBuild -eq "99.9999.9.10") { Write-ObservabilityLog "Getting Assembly Build using GetStampVersion returned $assemblyBuild, which is a possibly erroneous override. Using Assembly version in ECE parameters." $assemblyBuild = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version } } catch { Write-ObservabilityLog "Getting Assembly Build using GetStampVersion failed. Using Assembly version in ECE parameters" $assemblyBuild = $Parameters.Roles["Cloud"].PublicConfiguration.PublicInfo.Version } Write-ObservabilityLog "Assembly Build value of $assemblyBuild obtained." $localAdminCredential = [ObservabilityConfig]::GetLocalCredential($Parameters) Invoke-Command ` -Credential $localAdminCredential ` -Authentication Credssp ` -ComputerName $hostIp ` -ScriptBlock { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $success = $false $extError = "" # Get NodeId $azureStackHciDetails = Get-AzureStackHCI $StampId = $azureStackHciDetails.AzureResourceName $nodeId = $StampId + "-" + (Get-ClusterNode(hostname)).Id $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability\TenantJson' $nodeIdKey = "MONITORING_AEO_NODEID" $assemblyBuildKey = "MONITORING_AEO_ASSEMBLYBUILD" Write-ObservabilityLog "$($env:COMPUTERNAME): Setting keys $nodeIdKey to $nodeId and $assemblyBuildKey to $using:assemblyBuild at registry path $registryPath." if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name $nodeIdKey -PropertyType 'String' -Value $nodeId -Force New-ItemProperty -Path $registryPath -Name $assemblyBuildKey -PropertyType 'String' -Value $using:assemblyBuild -Force Write-ObservabilityLog "Finished setting registry keys $nodeIdKey and $assemblyBuildKey on $($env:COMPUTERNAME)" $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*TelemetryAndDiagnostics*\*\" -ErrorAction SilentlyContinue).FullName foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found Observability TelemetryAndDiagnostics extension at $extRootPath." if($success) { Write-ObservabilityLog "NodeId already updated successfully. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'scripts\ExtensionHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $logFile = Get-HandlerLogFile $gmaPackageContentPath = Get-GmaPackageContentPath Import-Module (Join-Path -Path "$gmaPackageContentPath" -ChildPath 'GMATenantJsonHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $cacheDirectories = New-CacheDirectories -LogFile $logFile $gmaCacheFolderPath = $cacheDirectories.GMACache $jsonDropLocation = Join-Path -Path $gmaCacheFolderPath -ChildPath "JsonDropLocation" foreach ($configType in $global:MiscConstants.ConfigTypes.Values) { $jsonConfigFileName = "AEO" + $configType + ".json" $tenantJsonFilePath = Join-Path -Path $jsonDropLocation -ChildPath $jsonConfigFileName Write-ObservabilityLog "Creating file $tenantJsonFilePath." Set-TenantConfigJsonFile -ConfigType $configType -FilePath $tenantJsonFilePath -LogFile $logFile Write-ObservabilityLog "Creation of file $tenantJsonFilePath succeeded." } $success = $true } catch { Write-ObservabilityErrorLog "Updating NodeId in tenant jsons failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Updating NodeId in tenant jsons failed with error $extError" } else { $errMsg = "Could not update NodeId in tenant jsons because no TelemetryAndDiagnostics Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } } static DeleteObservabilityAgents([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $obsServices = @( "AzureStack Observability Agent", "AzureStack Observability FDA", "AzureStack Observability GenevaMonitoringAgent", "AzureStack Observability RemoteSupportAgent" ) $stopDelayInSeconds = 10 $agentManifestPath = "C:\Agents\AgentManifests" $obsServices | ForEach-Object { $service = Get-Service -Name $_ -ErrorAction SilentlyContinue | Where-Object {$_.DisplayName -notlike "*Arc Extension*"} if ($service) { try { Write-ObservabilityLog "Stopping service $($service.DisplayName)." $service | Stop-Service Write-ObservabilityLog "Successfully stopped service $($service.DisplayName). Sleeping for $stopDelayInSeconds before unregistering service." Start-Sleep -Seconds $stopDelayInSeconds Write-ObservabilityLog "Unregistering service $($service.DisplayName)." sc.exe delete $service.Name Write-ObservabilityLog "Stopping and unregistering of service $($service.DisplayName) succeeded." $manifestPath = "$agentManifestPath\$($service.DisplayName).json" if (Test-Path -Path $manifestPath) { Write-ObservabilityLog "Deleting agent manifest at $manifestPath." Remove-Item $manifestPath -Force Write-ObservabilityLog "Deletion of agent manifest at $manifestPath succeeded." } } catch { Write-ObservabilityErrorLog "Stopping and unregistering of service $($service.DisplayName) failed with error $_" } } } } static SetListenerModeOff([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $registryPath = 'HKLM:\SOFTWARE\Microsoft\AzureStack\Observability' Write-ObservabilityLog "Setting $registryPath WatchdogListenerMode registry key to 0 on $($env:COMPUTERNAME)." if (!(Test-Path -Path $registryPath)) { New-Item -Path $registryPath -Force } New-ItemProperty -Path $registryPath -Name 'WatchdogListenerMode' -PropertyType 'DWord' -Value 0 -Force Write-ObservabilityLog "Finished setting WatchdogListenerMode Registry key to DWord 0 on $($env:COMPUTERNAME)" $timeLimitInSeconds = 600 $retryPeriod = 10 $elapsed = 0 $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue while($null -eq $maCoreProcess) { if($elapsed -gt $timeLimitInSeconds) { $errMsg = "Watchdog has failed to start Monitoring agent processes after $timeLimitInSeconds." Write-ObservabilityErrorLog $errMsg throw $errMsg } Write-ObservabilityLog "Monitoring Agent processes have not yet started. Retrying in $retryPeriod seconds." Start-Sleep -Seconds $retryPeriod $elapsed += $retryPeriod $maCoreProcess = Get-Process "MonAgentCore" -ErrorAction SilentlyContinue } Write-ObservabilityLog "MonitoringAgent has been successfully started. Watchdog listener mode has been set to off." } static StartArcExtensionObsAgent([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $success = $false $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*TelemetryAndDiagnostics*\*\" -ErrorAction SilentlyContinue).FullName $extError = "" foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found Observability TelemetryAndDiagnostics extension at $extRootPath." if ($success) { Write-ObservabilityLog "TelemetryAndDiagnostics Arc Extension Observability Agent already started. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'scripts\ExtensionHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $logFile = Get-HandlerLogFile ## Misc constants is defined here: https://msazure.visualstudio.com/One/_git/ASZ-Observability-MonitoringAgent?path=/src/GenevaMonitoringAgent/DeploymentScripts/GMATenantJsonHelper.psm1 ## Register Obs Agent as Windows service $obsAgent = $global:MiscConstants.ObsServiceDetails.ObsAgent $binaryFilePath = Join-Path -Path $global:ObsArtifactsPaths.ObservabilityAgent ` -ChildPath $obsAgent.BinaryFileName Write-ObservabilityLog "Registering $($obsAgent.DisplayName) service at path $binaryFilePath" Register-ServiceForObservability ` -ServiceName $obsAgent.Name ` -ServiceDisplayName $obsAgent.DisplayName ` -ServiceBinaryFilePath $binaryFilePath ` -LogFile $logFile ## Start Observability Agent. Write-ObservabilityLog "Starting $($obsAgent.DisplayName) service" Start-ServiceForObservability ` -ServiceName $obsAgent.Name ` -LogFile $logFile Write-ObservabilityLog "Observability Agent $($obsAgent.DisplayName) successfully registered and started" $success = $true } catch { Write-ObservabilityErrorLog "Starting TelemetryAndDiagnostics Arc Extension Observability Agent failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Starting TelemetryAndDiagnostics Arc Extension Observability Agent failed with error $extError" } else { $errMsg = "Could not start Arc Extension Observability Agent because no Arc Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } static RegenerateGMATenantJsonFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $success = $false $paths = (Get-ChildItem "C:\Packages\Plugins\Microsoft.AzureStack.Observability*TelemetryAndDiagnostics*\*\" -ErrorAction SilentlyContinue).FullName $extError = "" foreach ($extRootPath in $paths) { Write-ObservabilityLog "Found Observability TelemetryAndDiagnostics extension at $extRootPath." if($success) { Write-ObservabilityLog "GMA Tenant Json files already updated successfully. Ignoring extension at $extRootPath." } else { try { Import-Module (Join-Path -Path $extRootPath -ChildPath 'scripts\ExtensionHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $logFile = Get-HandlerLogFile $gmaPackageContentPath = Get-GmaPackageContentPath Import-Module (Join-Path -Path "$gmaPackageContentPath" -ChildPath 'GMATenantJsonHelper.psm1') ` -DisableNameChecking ` -Verbose:$false $cacheDirectories = New-CacheDirectories -LogFile $logFile $gmaCacheFolderPath = $cacheDirectories.GMACache $jsonDropLocation = Join-Path -Path $gmaCacheFolderPath -ChildPath "JsonDropLocation" foreach ($configType in $global:MiscConstants.ConfigTypes.Values) { $jsonConfigFileName = "AEO" + $configType + ".json" $tenantJsonFilePath = Join-Path -Path $jsonDropLocation -ChildPath $jsonConfigFileName Write-ObservabilityLog "Creating file $tenantJsonFilePath." Set-TenantConfigJsonFile -ConfigType $configType -FilePath $tenantJsonFilePath -LogFile $logFile Write-ObservabilityLog "Creation of file $tenantJsonFilePath succeeded." } $success = $true } catch { Write-ObservabilityErrorLog "Regenerating tenant jsons failed with error $_" $extError = $_ } } } if (-not $success) { if ($extError) { throw "Regenerating GMA tenant jsons failed with error $extError" } else { $errMsg = "Could not regenerate GMA tenant jsons because no TelemetryAndDiagnostics Extension was found at C:\Packages\Plugins." Write-ObservabilityErrorLog $errMsg throw $errMsg } } } static RestoreMAWatchdogRegKeyAndService([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $functionName = "RestoreMAWatchdogRegKeyAndService" Write-ObservabilityLog "Starting $functionName." $HCITelemetryRegKey = @{ Path = 'HKLM:\SYSTEM\Software\Microsoft\MAWatchdogService\HCITelemetry' Name = 'AllowTelemetry' PropertyType = 'String' Value = 'True' } $regKey = $(Get-ItemProperty -Path $HCITelemetryRegKey.Path -Name $HCITelemetryRegKey.Name -ErrorAction SilentlyContinue) if($null -ne $regKey) { Write-ObservabilityLog "$functionName : Registry key $($HCITelemetryRegKey.Name) at $($HCITelemetryRegKey.Path) already exists. Noop." } else { Write-ObservabilityLog "$functionName : Registry key $($HCITelemetryRegKey.Name) at $($HCITelemetryRegKey.Path) does not exist. Restoring." $watchdogServiceName = "WatchdogAgent" if (-not (Test-Path -Path $HCITelemetryRegKey.Path)) { New-Item -Path $HCITelemetryRegKey.Path -Force } New-ItemProperty ` -Path $HCITelemetryRegKey.Path ` -Name $HCITelemetryRegKey.Name ` -PropertyType $HCITelemetryRegKey.PropertyType ` -Value $HCITelemetryRegKey.Value ` -Force | Out-Null Write-ObservabilityLog "$functionName : Created registry key $($HCITelemetryRegKey.Name) at $($HCITelemetryRegKey.Path) with value $($HCITelemetryRegKey.Value)." $watchdogAgent = Get-Service $watchdogServiceName -ErrorAction SilentlyContinue if($watchdogAgent) { Write-ObservabilityLog "$functionName : Restarting $watchdogServiceName windows service." $watchdogAgent | Restart-Service -ErrorAction SilentlyContinue } } } static [Hashtable] GetRegistrationParameters([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { Import-Module $PSScriptRoot\Roles\Common\RoleHelpers.psm1 $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" $cloudDeploymentNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Solution.Deploy.CloudDeployment" Import-Module Az.Accounts -Force $hostNames = Get-ExecutionContextNodeName -Parameters $Parameters if($null -eq $hostNames) { $hostNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name } Trace-Execution "Getting registration parameters from ECE role parameters" $nodeNames = $Parameters.Roles["BareMetal"].PublicConfiguration.Nodes.Node.Name $cloudRole = $Parameters.Roles["Cloud"].PublicConfiguration $securityInfo = $cloudRole.PublicInfo.SecurityInfo $registrationParams = @{ EnvironmentName = $cloudRole.PublicInfo.RegistrationCloudName SubscriptionId = $cloudRole.PublicInfo.RegistrationSubscriptionId TenantId = $cloudRole.PublicInfo.RegistrationTenantId Region = $cloudRole.PublicInfo.RegistrationRegion ResourceGroupName = $cloudRole.PublicInfo.RegistrationResourceGroupName ResourceName = $cloudRole.PublicInfo.RegistrationResourceName CloudId = $cloudRole.PublicInfo.CloudId EnableAzureArcServer = [System.Convert]::ToBoolean($cloudRole.PublicInfo.EnableAzureArcServer) ComputerName = $nodeNames | Select-Object -First 1 } if (![String]::IsNullOrEmpty($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName)) { Trace-Execution "Using RegistrationArcServerResourceGroupName $($cloudRole.PublicInfo.RegistrationArcServerResourceGroupName) for ArcServer registration" $registrationParams.ResourceGroupName = $cloudRole.PublicInfo.RegistrationArcServerResourceGroupName } $registrationParameterSet = $cloudRole.PublicInfo.RegistrationParameterSet if ($registrationParameterSet -eq "DefaultSet") { Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using user token cache" $registrationTokenCacheUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationTokenCacheID $registrationTokenCacheCred = $Parameters.GetCredential($registrationTokenCacheUser.Credential) $clientId = $cloudRole.PublicInfo.RegistrationClientId Trace-Execution "Using clientId $clientId to get access token" Trace-Execution "Going to use RegistrationHelpers $cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1" Import-Module "$cloudDeploymentNugetPath\content\Setup\Common\RegistrationHelpers.psm1" $armAccessToken = Get-AccessToken -AzureEnvironment $registrationParams.EnvironmentName -TenantId $registrationParams.TenantId -TokenCacheCred $registrationTokenCacheCred -ClientId $clientId Trace-Execution "Access tokens length using token cache is $($armAccessToken.AccessToken.Length)" $registrationParams += @{ AccountId = $registrationTokenCacheCred.UserName ArmAccessToken = $armAccessToken.AccessToken } } else { Trace-Execution "RegistrationParameterSet $registrationParameterSet, getting access tokens using service principal" $registrationSPUser = $securityInfo.AADUsers.User | ? Role -EQ $Parameters.Configuration.Role.PrivateInfo.Accounts.RegistrationSPID $registrationSPCred = $Parameters.GetCredential($registrationSPUser.Credential) Login-AzAccount -Environment $registrationParams.EnvironmentName -Credential $registrationSPCred -Tenant $registrationParams.TenantId -ServicePrincipal $armAccessToken = Get-AzAccessToken -Verbose Trace-Execution "Access token length using service principal is $($armAccessToken.Token.Length)" $registrationParams += @{ AccountId = $registrationSPCred.UserName ArmAccessToken = $armAccessToken.Token } } if($registrationParams.ArmAccessToken.Length -eq 0) { throw "GetRegistrationParameters failed to retrieve AccessToken" } return $registrationParams } static [string] GetProxyUrl([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" Write-ObservabilityLog "Checking if proxy settings exist" $proxySettings = Get-ASProxySettings -Parameters $Parameters $proxyUrl = $proxySettings.HTTP if ($proxyUrl) { if (-not (($proxyUrl -cmatch "http://") -or ($proxyUrl -cmatch "https://"))) { $proxyUrl = "http://$proxyUrl" } } if ($proxyUrl) { Write-ObservabilityLog "Using proxy url $proxyUrl for Arc Agent connection." } return $proxyUrl } <# .SYNOPSIS Evaluation function to determine run SetupObservability or skip it for ArcA deployment ArcA currently does not support Arc4Server, skip Observability feature for ArcA deployment .EXAMPLE [GMATenantJson]::EvaluateSetupObservability($Parameters) .PARAMETER Parameters The ECE role parameters for this interface. #> static [CloudEngine.Actions.ConditionalActionDescription] EvaluateSetupObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $ErrorActionPreference = "Stop" return [ObservabilityConfig]::EvaluateObservabilityAction($Parameters, "SetupObservability") } <# .SYNOPSIS Evaluation function to identify run InstallBootstrapObservability or skip it for ArcA deployment ArcA currently does not support Arc4Server, skip Observability feature for ArcA deployment .EXAMPLE [GMATenantJson]::EvaluateInstallBootstrapObservability($Parameters) .PARAMETER Parameters The ECE role parameters for this interface. #> static [CloudEngine.Actions.ConditionalActionDescription] EvaluateInstallBootstrapObservability([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $ErrorActionPreference = "Stop" return [ObservabilityConfig]::EvaluateObservabilityAction($Parameters, "InstallBootstrapObservability") } <# .SYNOPSIS Evaluation function to identify run PostRegistrationObservabilitySetup or skip it for ArcA deployment ArcA currently does not support Arc4Server, skip Observability feature for ArcA deployment .EXAMPLE [GMATenantJson]::EvaluatePostRegistrationObservabilitySetup($Parameters) .PARAMETER Parameters The ECE role parameters for this interface. #> static [CloudEngine.Actions.ConditionalActionDescription] EvaluatePostRegistrationObservabilitySetup([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $ErrorActionPreference = "Stop" return [ObservabilityConfig]::EvaluateObservabilityAction($Parameters, "PostRegistrationObservabilitySetup") } <# .SYNOPSIS Evaluation function to identify run EvaluateSendTelemetryFromCachedFiles or skip it for ArcA deployment ArcA currently does not support Arc4Server, skip Observability feature for ArcA deployment .EXAMPLE [GMATenantJson]::EvaluateSendTelemetryFromCachedFiles($Parameters) .PARAMETER Parameters The ECE role parameters for this interface. #> static [CloudEngine.Actions.ConditionalActionDescription] EvaluateSendTelemetryFromCachedFiles([CloudEngine.Configurations.EceInterfaceParameters] $Parameters) { $ErrorActionPreference = "Stop" return [ObservabilityConfig]::EvaluateObservabilityAction($Parameters, "SendTelemetryFromCachedFiles") } static [CloudEngine.Actions.ConditionalActionDescription] EvaluateObservabilityAction([CloudEngine.Configurations.EceInterfaceParameters] $Parameters, [string] $actionName) { $ErrorActionPreference = "Stop" $observabilityNugetPath = Get-ASArtifactPath -NugetName "Microsoft.AzureStack.Observability.ObservabilityDeployment" Import-Module "$observabilityNugetPath\content\Powershell\ObservabilityHelpers.psm1" if (-not (Test-IsArcADeployment -Parameters $Parameters)) { Write-ObservabilityLog "EvaluateObservabilityAction: Execute $actionName" return [CloudEngine.Actions.ConditionalActionDescription]::CreateWithDefinedAction("ObservabilityConfig", $actionName) } Write-ObservabilityLog "EvaluateObservabilityAction: Skip executing $actionName for ArcA deployment" -Verbose return [CloudEngine.Actions.ConditionalActionDescription]::CreateNegative() } } # SIG # Begin signature block # MIInwQYJKoZIhvcNAQcCoIInsjCCJ64CAQExDzANBglghkgBZQMEAgEFADB5Bgor # BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG # KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCBm10a/KCdcWyF9 # Av2T6ikMUwenISWD2ZCzpUYyeE2nv6CCDXYwggX0MIID3KADAgECAhMzAAADrzBA # DkyjTQVBAAAAAAOvMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD # VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p # bmcgUENBIDIwMTEwHhcNMjMxMTE2MTkwOTAwWhcNMjQxMTE0MTkwOTAwWjB0MQsw # CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u # ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy # b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB # AQDOS8s1ra6f0YGtg0OhEaQa/t3Q+q1MEHhWJhqQVuO5amYXQpy8MDPNoJYk+FWA # hePP5LxwcSge5aen+f5Q6WNPd6EDxGzotvVpNi5ve0H97S3F7C/axDfKxyNh21MG # 0W8Sb0vxi/vorcLHOL9i+t2D6yvvDzLlEefUCbQV/zGCBjXGlYJcUj6RAzXyeNAN # xSpKXAGd7Fh+ocGHPPphcD9LQTOJgG7Y7aYztHqBLJiQQ4eAgZNU4ac6+8LnEGAL # go1ydC5BJEuJQjYKbNTy959HrKSu7LO3Ws0w8jw6pYdC1IMpdTkk2puTgY2PDNzB # tLM4evG7FYer3WX+8t1UMYNTAgMBAAGjggFzMIIBbzAfBgNVHSUEGDAWBgorBgEE # AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQURxxxNPIEPGSO8kqz+bgCAQWGXsEw # RQYDVR0RBD4wPKQ6MDgxHjAcBgNVBAsTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEW # MBQGA1UEBRMNMjMwMDEyKzUwMTgyNjAfBgNVHSMEGDAWgBRIbmTlUAXTgqoXNzci # tW2oynUClTBUBgNVHR8ETTBLMEmgR6BFhkNodHRwOi8vd3d3Lm1pY3Jvc29mdC5j # b20vcGtpb3BzL2NybC9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3JsMGEG # CCsGAQUFBwEBBFUwUzBRBggrBgEFBQcwAoZFaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNDb2RTaWdQQ0EyMDExXzIwMTEtMDctMDguY3J0 # MAwGA1UdEwEB/wQCMAAwDQYJKoZIhvcNAQELBQADggIBAISxFt/zR2frTFPB45Yd # mhZpB2nNJoOoi+qlgcTlnO4QwlYN1w/vYwbDy/oFJolD5r6FMJd0RGcgEM8q9TgQ # 2OC7gQEmhweVJ7yuKJlQBH7P7Pg5RiqgV3cSonJ+OM4kFHbP3gPLiyzssSQdRuPY # 1mIWoGg9i7Y4ZC8ST7WhpSyc0pns2XsUe1XsIjaUcGu7zd7gg97eCUiLRdVklPmp # XobH9CEAWakRUGNICYN2AgjhRTC4j3KJfqMkU04R6Toyh4/Toswm1uoDcGr5laYn # TfcX3u5WnJqJLhuPe8Uj9kGAOcyo0O1mNwDa+LhFEzB6CB32+wfJMumfr6degvLT # e8x55urQLeTjimBQgS49BSUkhFN7ois3cZyNpnrMca5AZaC7pLI72vuqSsSlLalG # OcZmPHZGYJqZ0BacN274OZ80Q8B11iNokns9Od348bMb5Z4fihxaBWebl8kWEi2O # PvQImOAeq3nt7UWJBzJYLAGEpfasaA3ZQgIcEXdD+uwo6ymMzDY6UamFOfYqYWXk # ntxDGu7ngD2ugKUuccYKJJRiiz+LAUcj90BVcSHRLQop9N8zoALr/1sJuwPrVAtx # HNEgSW+AKBqIxYWM4Ev32l6agSUAezLMbq5f3d8x9qzT031jMDT+sUAoCw0M5wVt # CUQcqINPuYjbS1WgJyZIiEkBMIIHejCCBWKgAwIBAgIKYQ6Q0gAAAAAAAzANBgkq # hkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 # IDIwMTEwHhcNMTEwNzA4MjA1OTA5WhcNMjYwNzA4MjEwOTA5WjB+MQswCQYDVQQG # EwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwG # A1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSgwJgYDVQQDEx9NaWNyb3NvZnQg # Q29kZSBTaWduaW5nIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC # CgKCAgEAq/D6chAcLq3YbqqCEE00uvK2WCGfQhsqa+laUKq4BjgaBEm6f8MMHt03 # a8YS2AvwOMKZBrDIOdUBFDFC04kNeWSHfpRgJGyvnkmc6Whe0t+bU7IKLMOv2akr # rnoJr9eWWcpgGgXpZnboMlImEi/nqwhQz7NEt13YxC4Ddato88tt8zpcoRb0Rrrg # OGSsbmQ1eKagYw8t00CT+OPeBw3VXHmlSSnnDb6gE3e+lD3v++MrWhAfTVYoonpy # 4BI6t0le2O3tQ5GD2Xuye4Yb2T6xjF3oiU+EGvKhL1nkkDstrjNYxbc+/jLTswM9 # sbKvkjh+0p2ALPVOVpEhNSXDOW5kf1O6nA+tGSOEy/S6A4aN91/w0FK/jJSHvMAh # dCVfGCi2zCcoOCWYOUo2z3yxkq4cI6epZuxhH2rhKEmdX4jiJV3TIUs+UsS1Vz8k # A/DRelsv1SPjcF0PUUZ3s/gA4bysAoJf28AVs70b1FVL5zmhD+kjSbwYuER8ReTB # w3J64HLnJN+/RpnF78IcV9uDjexNSTCnq47f7Fufr/zdsGbiwZeBe+3W7UvnSSmn # Eyimp31ngOaKYnhfsi+E11ecXL93KCjx7W3DKI8sj0A3T8HhhUSJxAlMxdSlQy90 # lfdu+HggWCwTXWCVmj5PM4TasIgX3p5O9JawvEagbJjS4NaIjAsCAwEAAaOCAe0w # ggHpMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBRIbmTlUAXTgqoXNzcitW2o # ynUClTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYD # VR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBa # BgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2Ny # bC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsG # AQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29t # L3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MIGfBgNV # HSAEgZcwgZQwgZEGCSsGAQQBgjcuAzCBgzA/BggrBgEFBQcCARYzaHR0cDovL3d3 # dy5taWNyb3NvZnQuY29tL3BraW9wcy9kb2NzL3ByaW1hcnljcHMuaHRtMEAGCCsG # AQUFBwICMDQeMiAdAEwAZQBnAGEAbABfAHAAbwBsAGkAYwB5AF8AcwB0AGEAdABl # AG0AZQBuAHQALiAdMA0GCSqGSIb3DQEBCwUAA4ICAQBn8oalmOBUeRou09h0ZyKb # C5YR4WOSmUKWfdJ5DJDBZV8uLD74w3LRbYP+vj/oCso7v0epo/Np22O/IjWll11l # hJB9i0ZQVdgMknzSGksc8zxCi1LQsP1r4z4HLimb5j0bpdS1HXeUOeLpZMlEPXh6 # I/MTfaaQdION9MsmAkYqwooQu6SpBQyb7Wj6aC6VoCo/KmtYSWMfCWluWpiW5IP0 # wI/zRive/DvQvTXvbiWu5a8n7dDd8w6vmSiXmE0OPQvyCInWH8MyGOLwxS3OW560 # STkKxgrCxq2u5bLZ2xWIUUVYODJxJxp/sfQn+N4sOiBpmLJZiWhub6e3dMNABQam # ASooPoI/E01mC8CzTfXhj38cbxV9Rad25UAqZaPDXVJihsMdYzaXht/a8/jyFqGa # J+HNpZfQ7l1jQeNbB5yHPgZ3BtEGsXUfFL5hYbXw3MYbBL7fQccOKO7eZS/sl/ah # XJbYANahRr1Z85elCUtIEJmAH9AAKcWxm6U/RXceNcbSoqKfenoi+kiVH6v7RyOA # 9Z74v2u3S5fi63V4GuzqN5l5GEv/1rMjaHXmr/r8i+sLgOppO6/8MO0ETI7f33Vt # Y5E90Z1WTk+/gFcioXgRMiF670EKsT/7qMykXcGhiJtXcVZOSEXAQsmbdlsKgEhr # /Xmfwb1tbWrJUnMTDXpQzTGCGaEwghmdAgEBMIGVMH4xCzAJBgNVBAYTAlVTMRMw # EQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVN # aWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNp # Z25pbmcgUENBIDIwMTECEzMAAAOvMEAOTKNNBUEAAAAAA68wDQYJYIZIAWUDBAIB # BQCgga4wGQYJKoZIhvcNAQkDMQwGCisGAQQBgjcCAQQwHAYKKwYBBAGCNwIBCzEO # MAwGCisGAQQBgjcCARUwLwYJKoZIhvcNAQkEMSIEIKgYwJLfpWqDTecekmkA73qq # DnoZHiU788inEL5CNzV/MEIGCisGAQQBgjcCAQwxNDAyoBSAEgBNAGkAYwByAG8A # cwBvAGYAdKEagBhodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20wDQYJKoZIhvcNAQEB # BQAEggEAAJZBgCFTU76SzYhHGdxvatsegBt4PNkjPMzTkSih0IAaU4R+kG583yXk # M5KZOfumnE0Z45KrvB4OsZGY7xJ9uZII5vtV7+SDuKb04s4XDSOG86kyRClxPYDP # gpPs46tp7+rAUo44J31TAc5u81uxRSNijIyA0xZ6TxULAyjyfzDnAog8ByFW5r/5 # FiTd93FmY1YwYQzbCDTuZbqOxf+DtL3DJQDatpLykNwBLVow0slJMCzWzKTjbNY/ # 0ec51JQrtLn2GjMdJra4JhKKtjgg/AIuiadwc2wVWF9lNebqKogzr8wiuPwAjWmT # ND6w+ZhQHKH1rQIJM4ir36Z6EEBFP6GCFyswghcnBgorBgEEAYI3AwMBMYIXFzCC # FxMGCSqGSIb3DQEHAqCCFwQwghcAAgEDMQ8wDQYJYIZIAWUDBAIBBQAwggFZBgsq # hkiG9w0BCRABBKCCAUgEggFEMIIBQAIBAQYKKwYBBAGEWQoDATAxMA0GCWCGSAFl # AwQCAQUABCDTvxse1qBiZHLOyvyBL5pKwQPiOnrtL+MfHc9FhinC7QIGZpaCaRcg # GBMyMDI0MDcxNjE2MjcyNS44ODFaMASAAgH0oIHYpIHVMIHSMQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMS0wKwYDVQQLEyRNaWNyb3NvZnQgSXJl # bGFuZCBPcGVyYXRpb25zIExpbWl0ZWQxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNO # Ojg2REYtNEJCQy05MzM1MSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBT # ZXJ2aWNloIIRejCCBycwggUPoAMCAQICEzMAAAHdXVcdldStqhsAAQAAAd0wDQYJ # KoZIhvcNAQELBQAwfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTAwHhcNMjMx # MDEyMTkwNzA5WhcNMjUwMTEwMTkwNzA5WjCB0jELMAkGA1UEBhMCVVMxEzARBgNV # BAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jv # c29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxhbmQgT3Bl # cmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4NkRGLTRC # QkMtOTMzNTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCC # AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKhOA5RE6i53nHURH4lnfKLp # +9JvipuTtctairCxMUSrPSy5CWK2DtriQP+T52HXbN2g7AktQ1pQZbTDGFzK6d03 # vYYNrCPuJK+PRsP2FPVDjBXy5mrLRFzIHHLaiAaobE5vFJuoxZ0ZWdKMCs8acjhH # UmfaY+79/CR7uN+B4+xjJqwvdpU/mp0mAq3earyH+AKmv6lkrQN8zgrcbCgHwsqv # vqT6lEFqYpi7uKn7MAYbSeLe0pMdatV5EW6NVnXMYOTRKuGPfyfBKdShualLo88k # G7qa2mbA5l77+X06JAesMkoyYr4/9CgDFjHUpcHSODujlFBKMi168zRdLerdpW0b # BX9EDux2zBMMaEK8NyxawCEuAq7++7ktFAbl3hUKtuzYC1FUZuUl2Bq6U17S4CKs # qR3itLT9qNcb2pAJ4jrIDdll5Tgoqef5gpv+YcvBM834bXFNwytd3ujDD24P9Dd8 # xfVJvumjsBQQkK5T/qy3HrQJ8ud1nHSvtFVi5Sa/ubGuYEpS8gF6GDWN5/KbveFk # dsoTVIPo8pkWhjPs0Q7nA5+uBxQB4zljEjKz5WW7BA4wpmFm24fhBmRjV4Nbp+n7 # 8cgAjvDSfTlA6DYBcv2kx1JH2dIhaRnSeOXePT6hMF0Il598LMu0rw35ViUWcAQk # UNUTxRnqGFxz5w+ZusMDAgMBAAGjggFJMIIBRTAdBgNVHQ4EFgQUbqL1toyPUdpF # yyHSDKWj0I4lw/EwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYD # VR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9j # cmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwG # CCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQu # Y29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIw # MjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADAWBgNVHSUBAf8EDDAKBggrBgEFBQcD # CDAOBgNVHQ8BAf8EBAMCB4AwDQYJKoZIhvcNAQELBQADggIBAC5U2bINLgXIHWbM # cqVuf9jkUT/K8zyLBvu5h8JrqYR2z/eaO2yo1Ooc9Shyvxbe9GZDu7kkUzxSyJ1I # ZksZZw6FDq6yZNT3PEjAEnREpRBL8S+mbXg+O4VLS0LSmb8XIZiLsaqZ0fDEcv3H # eA+/y/qKnCQWkXghpaEMwGMQzRkhGwcGdXr1zGpQ7HTxvfu57xFxZX1MkKnWFENJ # 6urd+4teUgXj0ngIOx//l3XMK3Ht8T2+zvGJNAF+5/5qBk7nr079zICbFXvxtidN # N5eoXdW+9rAIkS+UGD19AZdBrtt6dZ+OdAquBiDkYQ5kVfUMKS31yHQOGgmFxuCO # zTpWHalrqpdIllsy8KNsj5U9sONiWAd9PNlyEHHbQZDmi9/BNlOYyTt0YehLbDov # mZUNazk79Od/A917mqCdTqrExwBGUPbMP+/vdYUqaJspupBnUtjOf/76DAhVy8e/ # e6zR98PkplmliO2brL3Q3rD6+ZCVdrGM9Rm6hUDBBkvYh+YjmGdcQ5HB6WT9Rec8 # +qDHmbhLhX4Zdaard5/OXeLbgx2f7L4QQQj3KgqjqDOWInVhNE1gYtTWLHe4882d # /k7Lui0K1g8EZrKD7maOrsJLKPKlegceJ9FCqY1sDUKUhRa0EHUW+ZkKLlohKrS7 # FwjdrINWkPBgbQznCjdE2m47QjTbMIIHcTCCBVmgAwIBAgITMwAAABXF52ueAptJ # mQAAAAAAFTANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT # Cldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29m # dCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJvb3QgQ2VydGlmaWNh # dGUgQXV0aG9yaXR5IDIwMTAwHhcNMjEwOTMwMTgyMjI1WhcNMzAwOTMwMTgzMjI1 # WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCAiIwDQYJKoZIhvcNAQEB # BQADggIPADCCAgoCggIBAOThpkzntHIhC3miy9ckeb0O1YLT/e6cBwfSqWxOdcjK # NVf2AX9sSuDivbk+F2Az/1xPx2b3lVNxWuJ+Slr+uDZnhUYjDLWNE893MsAQGOhg # fWpSg0S3po5GawcU88V29YZQ3MFEyHFcUTE3oAo4bo3t1w/YJlN8OWECesSq/XJp # rx2rrPY2vjUmZNqYO7oaezOtgFt+jBAcnVL+tuhiJdxqD89d9P6OU8/W7IVWTe/d # vI2k45GPsjksUZzpcGkNyjYtcI4xyDUoveO0hyTD4MmPfrVUj9z6BVWYbWg7mka9 # 7aSueik3rMvrg0XnRm7KMtXAhjBcTyziYrLNueKNiOSWrAFKu75xqRdbZ2De+JKR # Hh09/SDPc31BmkZ1zcRfNN0Sidb9pSB9fvzZnkXftnIv231fgLrbqn427DZM9itu # qBJR6L8FA6PRc6ZNN3SUHDSCD/AQ8rdHGO2n6Jl8P0zbr17C89XYcz1DTsEzOUyO # ArxCaC4Q6oRRRuLRvWoYWmEBc8pnol7XKHYC4jMYctenIPDC+hIK12NvDMk2ZItb # oKaDIV1fMHSRlJTYuVD5C4lh8zYGNRiER9vcG9H9stQcxWv2XFJRXRLbJbqvUAV6 # bMURHXLvjflSxIUXk8A8FdsaN8cIFRg/eKtFtvUeh17aj54WcmnGrnu3tz5q4i6t # AgMBAAGjggHdMIIB2TASBgkrBgEEAYI3FQEEBQIDAQABMCMGCSsGAQQBgjcVAgQW # BBQqp1L+ZMSavoKRPEY1Kc8Q/y8E7jAdBgNVHQ4EFgQUn6cVXQBeYl2D9OXSZacb # UzUZ6XIwXAYDVR0gBFUwUzBRBgwrBgEEAYI3TIN9AQEwQTA/BggrBgEFBQcCARYz # aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9Eb2NzL1JlcG9zaXRvcnku # aHRtMBMGA1UdJQQMMAoGCCsGAQUFBwMIMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIA # QwBBMAsGA1UdDwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNX2 # VsuP6KJcYmjRPZSQW9fOmhjEMFYGA1UdHwRPME0wS6BJoEeGRWh0dHA6Ly9jcmwu # bWljcm9zb2Z0LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dF8yMDEw # LTA2LTIzLmNybDBaBggrBgEFBQcBAQROMEwwSgYIKwYBBQUHMAKGPmh0dHA6Ly93 # d3cubWljcm9zb2Z0LmNvbS9wa2kvY2VydHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYt # MjMuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCdVX38Kq3hLB9nATEkW+Geckv8qW/q # XBS2Pk5HZHixBpOXPTEztTnXwnE2P9pkbHzQdTltuw8x5MKP+2zRoZQYIu7pZmc6 # U03dmLq2HnjYNi6cqYJWAAOwBb6J6Gngugnue99qb74py27YP0h1AdkY3m2CDPVt # I1TkeFN1JFe53Z/zjj3G82jfZfakVqr3lbYoVSfQJL1AoL8ZthISEV09J+BAljis # 9/kpicO8F7BUhUKz/AyeixmJ5/ALaoHCgRlCGVJ1ijbCHcNhcy4sa3tuPywJeBTp # kbKpW99Jo3QMvOyRgNI95ko+ZjtPu4b6MhrZlvSP9pEB9s7GdP32THJvEKt1MMU0 # sHrYUP4KWN1APMdUbZ1jdEgssU5HLcEUBHG/ZPkkvnNtyo4JvbMBV0lUZNlz138e # W0QBjloZkWsNn6Qo3GcZKCS6OEuabvshVGtqRRFHqfG3rsjoiV5PndLQTHa1V1QJ # sWkBRH58oWFsc/4Ku+xBZj1p/cvBQUl+fpO+y/g75LcVv7TOPqUxUYS8vwLBgqJ7 # Fx0ViY1w/ue10CgaiQuPNtq6TPmb/wrpNPgkNWcr4A245oyZ1uEi6vAnQj0llOZ0 # dFtq0Z4+7X6gMTN9vMvpe784cETRkPHIqzqKOghif9lwY1NNje6CbaUFEMFxBmoQ # tB1VM1izoXBm8qGCAtYwggI/AgEBMIIBAKGB2KSB1TCB0jELMAkGA1UEBhMCVVMx # EzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoT # FU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEtMCsGA1UECxMkTWljcm9zb2Z0IElyZWxh # bmQgT3BlcmF0aW9ucyBMaW1pdGVkMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4 # NkRGLTRCQkMtOTMzNTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy # dmljZaIjCgEBMAcGBSsOAwIaAxUANiNHGWXbNaDPxnyiDbEOciSjFhCggYMwgYCk # fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH # UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD # Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF # AOpBAOUwIhgPMjAyNDA3MTYyMjIzMzNaGA8yMDI0MDcxNzIyMjMzM1owdjA8Bgor # BgEEAYRZCgQBMS4wLDAKAgUA6kEA5QIBADAJAgEAAgEIAgH/MAcCAQACAhEkMAoC # BQDqQlJlAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEA # AgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAAwjrG3Mn9A3HZ2jh # 2wUEQobXRpHnG2DI06f16hE261euRm63LVwLzzDWHNbhSXGhb0eGyKXTowF/bxBM # BP/W1m32mixR7vFGevDQW9gZZO0otzGXtKAKukM1vfSXdpQbtN8DA3eCyaUR7vFj # ASI1wzIl6Jo2Xs+CphrbmnTwlPIxggQNMIIECQIBATCBkzB8MQswCQYDVQQGEwJV # UzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UE # ChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGlt # ZS1TdGFtcCBQQ0EgMjAxMAITMwAAAd1dVx2V1K2qGwABAAAB3TANBglghkgBZQME # AgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJ # BDEiBCB2EwJRbMBdfmGybLPxK161Nml3JHQdeCeLrpGE8qNbfzCB+gYLKoZIhvcN # AQkQAi8xgeowgecwgeQwgb0EIGH/Di2aZaxPeJmce0fRWTftQI3TaVHFj5GI43rA # MWNmMIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24x # EDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlv # bjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAHd # XVcdldStqhsAAQAAAd0wIgQg9QMUaq3dYqJdr/VTY54MOZuaznkFbDnhiR7Xfmg2 # EIUwDQYJKoZIhvcNAQELBQAEggIASzr03SuRXCyslGMR7muip/amGBK8KkBtRyxs # SYLvfHJy20qzaxFXFTkBTOahbJ//exff133AlvUw5INBbOopPkw05SG291GSB+EL # EKGS8bsKm7QBSXirBEH/xx9fLKeUdHCOd5aBGa2Y3ZE5iyaft4KfowQpEkvuhxDX # FxVE9KzCrSFinMElueBxQztN2hlFIuL2W/SUa6oXqBic1UTNvGAvXYVBZSUcUl1d # SWEYaZ55auM2DXbL6H56ekRT4yUduG2dcEDwIO9UXA+jYQX8ert64Chm058c1VsA # 6Jk3ObFOhLgREbrPO+qkhFNRzdk9Vi9ft0qG1M0QpevpPcPyw4OvCqcZrZczYuXU # tYjyMigEI0Rm/L1+Ks/CV0GilY78wBrnAR6FwxJl1WB8i9GiJ4+wKb/op8zRKx9K # qIUFr7s1x5qDHfMU0bu0nlTqKuouVxxRNmvM/ETO4jXywrUV5jINOEs/bvGoD/G4 # Bk96UyWfZFKrCU+b/MZ6q47woWEVX9BrIS89jl7WRJk5pwp3uJjBt4r9SjYUgGNC # 3x0WFVdMAY8AOXngi6tOXgF4M7ZBHGCN0xf4ME4z1jeeNf67Qsm4S+vtk0OVqUMu # ZwO1tl+IeZq1woztsUiicu+P6OdgW2I4fK7KSDgwyvYHTBCvrXhpc+YgWk2qrAGD # ZTR5yqg= # SIG # End signature block |