Framework/Configurations/ContinuousAssurance/RunbookScanAgent.ps1
function ConvertStringToBoolean($strToConvert) { switch ($strToConvert) { "true" {return $true} "false" {return $false} } return $false #adding this to prevent error all path doesn't return value" } function SanitizeStorage(){ $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG -Name $StorageAccountName $currentContext = New-AzureStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $keys[0].Value -Protocol Https $Container = Get-AzureStorageContainer -Name "azsdk-controls-state" -Context $currentContext -ErrorAction SilentlyContinue if($null -eq $Container) { New-AzureStorageContainer -Name "azsdk-controls-state" -Permission Off -Context $currentContext } } function RunAzSDKScan() { if(-not [string]::IsNullOrWhiteSpace($AltOMSWorkspaceId) -and -not [string]::IsNullOrWhiteSpace($AltOMSWorkspaceSharedKey)) { Set-AzSKOMSSettings -OMSWorkspaceID $OMSWorkspaceId -OMSSharedKey $OMSWorkspaceSharedKey -AltOMSWorkspaceId $AltOMSWorkspaceId -AltOMSSharedKey $AltOMSWorkspaceSharedKey -Source "CC" } else { Set-AzSKOMSSettings -OMSWorkspaceID $OMSWorkspaceId -OMSSharedKey $OMSWorkspaceSharedKey -Source "CC" } if(-not [string]::IsNullOrWhiteSpace($WebhookUrl)) { if(-not [string]::IsNullOrWhiteSpace($WebhookAuthZHeaderName) -and -not [string]::IsNullOrWhiteSpace($WebhookAuthZHeaderValue)) { Set-AzSKWebhookSettings -WebhookUrl $WebhookUrl -AuthZHeaderName $WebhookAuthZHeaderName -AuthZHeaderValue $WebhookAuthZHeaderValue -Source "CC" } else { Set-AzSKWebhookSettings -WebhookUrl $WebhookUrl -Source "CC" } } if(-not [string]::IsNullOrWhiteSpace($AltOMSWorkspaceId) -and -not [string]::IsNullOrWhiteSpace($AltOMSWorkspaceSharedKey)) { Set-AzSKOMSSettings -AltOMSWorkspaceId $AltOMSWorkspaceId -AltOMSSharedKey $AltOMSWorkspaceSharedKey -Source "CC" } if(-not [string]::IsNullOrWhiteSpace($WebhookUrl)) { if(-not [string]::IsNullOrWhiteSpace($WebhookAuthZHeaderName) -and -not [string]::IsNullOrWhiteSpace($WebhookAuthZHeaderValue)) { Set-AzSKWebhookSettings -WebhookUrl $WebhookUrl -AuthZHeaderName $WebhookAuthZHeaderName -AuthZHeaderValue $WebhookAuthZHeaderValue -Source "CC" } else { Set-AzSKWebhookSettings -WebhookUrl $WebhookUrl -Source "CC" } } #set values in azsdksettings.json $EnableAADAuthForOnlinePolicyStore = ConvertStringToBoolean($EnableAADAuthForOnlinePolicyStore) if ($EnableAADAuthForOnlinePolicyStore) { Set-AzSKPolicySettings -OnlinePolicyStoreUrl $OnlinePolicyStoreUrl -EnableAADAuthForOnlinePolicyStore } else { Set-AzSKPolicySettings -OnlinePolicyStoreUrl $OnlinePolicyStoreUrl } Set-AzSKPrivacyNoticeResponse -AcceptPrivacyNotice "yes" # skipping the EULA and privacy prompt PublishEvent -EventName "CA Scan Started" -Properties @{ "ResourceGroupNames" = $ResourceGroupNames; ` "OnlinePolicyStoreUrl" = $OnlinePolicyStoreUrl; ` "OMSWorkspaceId" = $OMSWorkspaceId; } CheckForSubscriptionsSnapshotData #Check if the preview code is enabled #get the current storagecontext $existingStorage = Find-AzureRmResource -ResourceGroupNameEquals $StorageAccountRG -ResourceNameContains "azsdk" -ResourceType "Microsoft.Storage/storageAccounts" if(($existingStorage|Measure-Object).Count -gt 1) { $existingStorage = $existingStorage[0] Write-Output ("Multiple storage accounts found in resource group. Using Storage Account: $($existingStorage.ResourceName) for storing logs") } #Create output files in storage $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG -Name $existingStorage.ResourceName $centralStorageContext = New-AzureStorageContext -StorageAccountName $existingStorage.ResourceName -StorageAccountKey $keys[0].Value -Protocol Https if($Global:IsPreviewMode) { try { $Global:activeScanObjects | ForEach-Object { $activeScanObject = $_; if($activeScanObject.Status -ne "COM") { $subId = $activeScanObject.SubscriptionId; Select-AzureRmSubscription -SubscriptionId $subId | Out-Null "Started scan for the subscription: $subId" PersistSubscriptionSnapshot -SubscriptionID $subId -Status "INP" -StorageContext $centralStorageContext RunAzSDKScanForASub -SubscriptionID $subId -LoggingOption $activeScanObject.LoggingOption -StorageContext $centralStorageContext PersistSubscriptionSnapshot -SubscriptionID $subId -Status "COM" -StorageContext $centralStorageContext "Completed scan for the subscription: $subId" } } } finally{ Select-AzureRmSubscription -SubscriptionId $RunAsConnection.SubscriptionID | Out-Null } } else { RunAzSDKScanForASub -SubscriptionID $RunAsConnection.SubscriptionID -LoggingOption "CentralSub" -StorageContext $centralStorageContext } } function RunAzSDKScanForASub { param ( $SubscriptionID, $LoggingOption, $StorageContext ) $svtResultPath = [string]::Empty $subscriptionResultPath = [string]::Empty $parentFolderPath = [string]::Empty #------------------------------------Subscription scan---------------------------------------------------------------- "Running command 'Get-AzSKSubscriptionSecurityStatus'" $subScanTimer = [System.Diagnostics.Stopwatch]::StartNew(); PublishEvent -EventName "CA Scan Subscription Started" $subscriptionResultPath = Get-AzSKSubscriptionSecurityStatus -SubscriptionId $SubscriptionID -ExcludeTags "OwnerAccess" #---------------------------Check subscription scan status-------------------------------------------------------------- if ([string]::IsNullOrWhiteSpace($subscriptionResultPath)) { PublishEvent -EventName "CA Scan Subscription Error" -Metrics @{"TimeTakenInMs" = $subScanTimer.ElapsedMilliseconds; "SuccessCount" = 0} "Subscription scan failed." } else { PublishEvent -EventName "CA Scan Subscription Completed" -Metrics @{"TimeTakenInMs" = $subScanTimer.ElapsedMilliseconds; "SuccessCount" = 1} "Subscription scan succeeded." $parentFolderPath = (Get-Item $subscriptionResultPath).parent.FullName } #-------------------------------------Resources Scan------------------------------------------------------------------ "Running command 'Get-AzSKAzureServicesSecurityStatus'" $serviceScanTimer = [System.Diagnostics.Stopwatch]::StartNew(); PublishEvent -EventName "CA Scan Services Started" $svtResultPath = Get-AzSKAzureServicesSecurityStatus -SubscriptionId $SubscriptionID -ResourceGroupNames $ResourceGroupNames -ExcludeTags "OwnerAccess" -UsePartialCommits #---------------------------Check resources scan status-------------------------------------------------------------- if ([string]::IsNullOrWhiteSpace($svtResultPath)) { "Azure resources scan failed." PublishEvent -EventName "CA Scan Services Error" -Metrics @{"TimeTakenInMs" = $serviceScanTimer.ElapsedMilliseconds; "SuccessCount" = 0} } else { "Azure resources scan succeeded." $parentFolderPath = (Get-Item $svtResultPath).parent.FullName PublishEvent -EventName "CA Scan Services Completed" -Metrics @{"TimeTakenInMs" = $serviceScanTimer.ElapsedMilliseconds; "SuccessCount" = 1} } #----------------------------------------Export reports to storage--------------------------------------------------- if (![string]::IsNullOrWhiteSpace($subscriptionResultPath) -or ![string]::IsNullOrWhiteSpace($svtResultPath)) { #Check if storage account exists if($Global:IsPreviewMode) { if($LoggingOption -ne "CentralSub") { $existingStorage = Find-AzureRmResource -ResourceGroupNameEquals $StorageAccountRG -ResourceNameContains "azsdk" -ResourceType "Microsoft.Storage/storageAccounts" if(($existingStorage|Measure-Object).Count -gt 1) { $existingStorage = $existingStorage[0] Write-Output ("Multiple storage accounts found in resource group. Using Storage Account: $($existingStorage.ResourceName) for storing logs") } #Create output files in storage $containerName = "azsdkexecutionlogs" $archiveFilePath = "$parentFolderPath\AutomationLogs_" + $(Get-Date -format "yyyyMMdd_HHmmss") + ".zip" $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG -Name $existingStorage.ResourceName $localStorageContext = New-AzureStorageContext -StorageAccountName $existingStorage.ResourceName -StorageAccountKey $keys[0].Value -Protocol Https try { Get-AzureStorageContainer -Name $containerName -Context $localStorageContext -ErrorAction Stop | Out-Null } catch { New-AzureStorageContainer -Name $containerName -Context $localStorageContext | Out-Null } PersistToStorageAccount -StorageContext $localStorageContext -SubscriptionResultPath $subscriptionResultPath -SvtResultPath $svtResultPath -SubscriptionId $SubscriptionID PurgeOlderScanReports -StorageContext $localStorageContext } else { PersistToStorageAccount -StorageContext $StorageContext -SubscriptionResultPath $subscriptionResultPath -SvtResultPath $svtResultPath -SubscriptionId $SubscriptionID PurgeOlderScanReports -StorageContext $StorageContext } } else { PersistToStorageAccount -StorageContext $StorageContext -SubscriptionResultPath $subscriptionResultPath -SvtResultPath $svtResultPath -SubscriptionId $SubscriptionID PurgeOlderScanReports -StorageContext $StorageContext } #clean-up of logs in automation sandbox if (![string]::IsNullOrWhiteSpace($svtResultPath)) { Remove-Item -Path $svtResultPath -Recurse -ErrorAction Ignore } if (![string]::IsNullOrWhiteSpace($subscriptionResultPath)) { Remove-Item -Path $subscriptionResultPath -Recurse -ErrorAction Ignore } if (![string]::IsNullOrWhiteSpace($archiveFilePath)) { Remove-Item -Path $archiveFilePath -Recurse -ErrorAction Ignore } } } function PersistToStorageAccount { param( $StorageContext, $SubscriptionResultPath, $SvtResultPath, $SubscriptionId ) if (![string]::IsNullOrWhiteSpace($SubscriptionResultPath) -or ![string]::IsNullOrWhiteSpace($SvtResultPath)) { #Check if the passed storagecontext is null. This would be in the case of default scenario i.e non preview mode $containerName = "azsdkexecutionlogs" $timeStamp=(Get-Date -format "yyyyMMdd_HHmmss") $archiveFilePath = "$parentFolderPath\AutomationLogs_" + $timeStamp + ".zip" $storageLocation="$SubscriptionId/AutomationLogs_" + $timestamp + ".zip" try { Get-AzureStorageContainer -Name $containerName -Context $StorageContext -ErrorAction Stop | Out-Null } catch { New-AzureStorageContainer -Name $containerName -Context $StorageContext | Out-Null } #Persist the files to the storage account using the passed storage context try { if (![string]::IsNullOrWhiteSpace($SvtResultPath)) { Compress-Archive -Path $SvtResultPath -CompressionLevel Optimal -DestinationPath $archiveFilePath -Update } if (![string]::IsNullOrWhiteSpace($SubscriptionResultPath)) { Compress-Archive -Path $SubscriptionResultPath -CompressionLevel Optimal -DestinationPath $archiveFilePath -Update } Set-AzureStorageBlobContent -File $archiveFilePath -Container $containerName -Context $StorageContext -Blob $storageLocation -ErrorAction Stop | Out-Null "Exported reports to storage $StorageAccountName" PublishEvent -EventName "CA Scan Reports Persisted" -Properties @{"StorageAccountName" = $StorageAccountName; "ArchiveFilePath" = $archiveFilePath } -Metrics @{"SuccessCount" = 1} } catch { "Could not export reports to storage $StorageAccountName" PublishEvent -EventName "CA Scan Reports Persist Error" -Properties @{"ErrorRecord" = ($_ | Out-String); "StorageAccountName" = $StorageAccountName; "ArchiveFilePath" = $archiveFilePath } -Metrics @{"SuccessCount" = 0} throw $_.Exception } } } function PurgeOlderScanReports { param( $StorageContext ) $containerName = "azsdkexecutionlogs" $NotBefore = [DateTime]::Now.AddDays(-30); $OldLogCount = (Get-AzureStorageBlob -Container $containerName -Context $StorageContext | Where-Object { $_.LastModified -lt $NotBefore} | Measure-Object).Count Get-AzureStorageBlob -Container $containerName -Context $StorageContext | Where-Object { $_.LastModified -lt $NotBefore} | Remove-AzureStorageBlob -Force -ErrorAction SilentlyContinue if($OldLogCount -gt 0) { #deleted successfully all the old reports Write-Output ("Removed all the scan logs/reports older than date: $($NotBefore.ToShortDateString()) from storage account: [$StorageAccountName]") } } function CheckForSubscriptionsSnapshotData() { try { $CAScanDataBlobName = "CAScanObjects.json" $CAActiveScanSnapshotBlobName = "ActiveScanSnapshot.json" $destinationFolderPath = $env:temp + "\AzSDKTemp\" if(-not (Test-Path -Path $destinationFolderPath)) { mkdir -Path $destinationFolderPath -Force } $keys = Get-AzureRmStorageAccountKey -ResourceGroupName $StorageAccountRG -Name $StorageAccountName $currentContext = New-AzureStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $keys[0].Value -Protocol Https #Fetch if there is any existing active scan snapshot $CAScanSourceDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAScanDataBlobName -Context $currentContext -ErrorAction SilentlyContinue if($null -eq $CAScanSourceDataBlobObject) { $Global:IsPreviewMode = $false; return; } $CAScanDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $currentContext -ErrorAction SilentlyContinue if($null -ne $CAScanDataBlobObject) { $CAActiveSnapshotBlobContentObject = Get-AzureStorageBlobContent -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $currentContext -Destination $destinationFolderPath -Force $Global:activeScanObjects = [array](Get-ChildItem -Path "$destinationFolderPath\$CAActiveScanSnapshotBlobName" -Force | Get-Content | ConvertFrom-Json) } else { #Fetch the CA Scan objects $CAScanDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAScanDataBlobName -Context $currentContext -ErrorAction Stop | Out-Null $CAScanDataBlobContentObject = Get-AzureStorageBlobContent -Container $CentralScanContainerName -Blob $CAScanDataBlobName -Context $currentContext -Destination $destinationFolderPath -Force $CAScanDataBlobContent = Get-ChildItem -Path "$destinationFolderPath\$CAScanDataBlobName" -Force | Get-Content | ConvertFrom-Json #create the active snapshot from the ca scan objects $Global:activeScanObjects = @(); if(($CAScanDataBlobContent | Measure-Object).Count -gt 0) { $CAScanDataBlobContent | ForEach-Object { $CAScanDataInstance = $_; $out = "" | Select SubscriptionId, Status, LoggingOption, CreatedTime, StartedTime, CompletedTime $out.SubscriptionId = $CAScanDataInstance.SubscriptionId $out.Status = "NA"; $out.LoggingOption = $CAScanDataInstance.LoggingOption; $out.CreatedTime = [DateTime]::UtcNow.ToString(); $out.StartedTime = [DateTime]::MinValue.ToString(); $out.CompletedTime = [DateTime]::MinValue.ToString(); $Global:activeScanObjects += $out; } $snapshotFilePath = "$destinationFolderPath\$CAActiveScanSnapshotBlobName" $Global:activeScanObjects | ConvertTo-Json -Depth 10 | Out-File $snapshotFilePath Set-AzureStorageBlobContent -File $snapshotFilePath -Container $CentralScanContainerName -BlobType Block -Context $currentContext -Force } } if(($Global:activeScanObjects | Measure-Object).Count -gt 0) { $Global:IsPreviewMode = $true; } } catch { PublishEvent -EventName "CA Scan Error-PreviewSnapshotComputation" -Properties @{ "ErrorRecord" = ($_ | Out-String) } -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds; "SuccessCount" = 0} $Global:IsPreviewMode = $false; } } function PersistSubscriptionSnapshot { param( $SubscriptionID, $Status, $StorageContext ) try { $CAActiveScanSnapshotBlobName = "ActiveScanSnapshot.json" $destinationFolderPath = $env:temp + "\AzSDKTemp\" if(-not (Test-Path -Path $destinationFolderPath)) { mkdir -Path $destinationFolderPath -Force } #Fetch if there is any existing active scan snapshot $CAScanDataBlobObject = Get-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $StorageContext -ErrorAction SilentlyContinue if($null -ne $CAScanDataBlobObject) { $CAActiveSnapshotBlobContentObject = Get-AzureStorageBlobContent -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $StorageContext -Destination $destinationFolderPath -Force $activeScanObjects = [array](Get-ChildItem -Path "$destinationFolderPath\$CAActiveScanSnapshotBlobName" -Force | Get-Content | ConvertFrom-Json) $matchedSubId = $activeScanObjects | Where-Object {$_.SubscriptionId -eq $SubscriptionID} if(($matchedSubId | Measure-Object).Count -gt 0) { $matchedSubId[0].SubscriptionId = $SubscriptionID $matchedSubId[0].Status = $Status; if($Status -eq "COM") { $matchedSubId[0].CompletedTime = [DateTime]::UtcNow.ToString(); } elseif($Status -eq "INP") { $matchedSubId[0].StartedTime = [DateTime]::UtcNow.ToString(); } } if(($activeScanObjects | Where-Object { $_.Status -ne "COM"} | Measure-Object).Count -eq 0) { Remove-AzureStorageBlob -Container $CentralScanContainerName -Blob $CAActiveScanSnapshotBlobName -Context $StorageContext -Force } else { $tempFilePath = "$destinationFolderPath\$CAActiveScanSnapshotBlobName" $activeScanObjects | ConvertTo-Json -Depth 10 | Out-File $tempFilePath Set-AzureStorageBlobContent -File $tempFilePath -Container $CentralScanContainerName -BlobType Block -Context $StorageContext -Force } } } catch { PublishEvent -EventName "CA Scan Error-PreviewSnapshotPersist" -Properties @{ "ErrorRecord" = ($_ | Out-String) } -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds; "SuccessCount" = 0} $Global:IsPreviewMode = $false; } } try { #start timer $scanAgentTimer = [System.Diagnostics.Stopwatch]::StartNew(); #config start $ResourceGroupNames = Get-AutomationVariable -Name "AppResourceGroupNames" $OMSWorkspaceId = Get-AutomationVariable -Name "OMSWorkspaceId" $OMSWorkspaceSharedKey = Get-AutomationVariable -Name "OMSSharedKey" $AltOMSWorkspaceId = Get-AutomationVariable -Name "AltOMSWorkspaceId" -ErrorAction SilentlyContinue $AltOMSWorkspaceSharedKey = Get-AutomationVariable -Name "AltOMSSharedKey" -ErrorAction SilentlyContinue $WebhookUrl = Get-AutomationVariable -Name "WebhookUrl" -ErrorAction SilentlyContinue $WebhookAuthZHeaderName = Get-AutomationVariable -Name "WebhookAuthZHeaderName" -ErrorAction SilentlyContinue $WebhookAuthZHeaderValue = Get-AutomationVariable -Name "WebhookAuthZHeaderValue" -ErrorAction SilentlyContinue $StorageAccountName = Get-AutomationVariable -Name "ReportsStorageAccountName" $AzSDKModuleName = "AzSDKPreview" $StorageAccountRG = "AzSDKRG" $RunbookName = "Continuous_Assurance_Runbook" $CAHelperScheduleName = "CA_Helper_Schedule" $CentralScanContainerName = "azsdk-scan-objects" ##config end #Set subscription id $SubscriptionID = $RunAsConnection.SubscriptionID $Global:IsPreviewMode = $false; $Global:activeScanObjects = @(); Select-AzureRmSubscription -SubscriptionId $SubscriptionID; if($Global:FoundExistingJob) { return; } $isAzSDKAvailable = (Get-AzureRmAutomationModule -ResourceGroupName $AutomationAccountRG ` -AutomationAccountName $AutomationAccountName ` -Name $AzSDKModuleName -ErrorAction SilentlyContinue | ` Where-Object {$_.ProvisioningState -eq "Succeeded" -or $_.ProvisioningState -eq "Created"} | ` Measure-Object).Count -gt 0 if ($isAzSDKAvailable) { Import-Module $AzSDKModuleName } #check if AzureRM is available (for scenario where AzSDK is available but AzureRM extraction might have failed) $isAzureRMAvailable = (Get-AzureRmAutomationModule -ResourceGroupName $AutomationAccountRG ` -AutomationAccountName $AutomationAccountName ` -Name AzureRM -ErrorAction SilentlyContinue | ` Where-Object {$_.ProvisioningState -eq "Succeeded" -or $_.ProvisioningState -eq "Created"} | ` Measure-Object).Count -gt 0 #return if modules are not ready if ((Get-Command -Name "Get-AzSKAzureServicesSecurityStatus" -ErrorAction SilentlyContinue|Measure-Object).Count -eq 0 -or !$isAzureRMAvailable) { "AzSDK module not available. Skipping AzSDK scan. Will retry in the next run." PublishEvent -EventName "CA Job Skipped" -Properties @{"SubscriptionId" = $RunAsConnection.SubscriptionID} -Metrics @{"TimeTakenInMs" = $timer.ElapsedMilliseconds; "SuccessCount" = 1} return; } # Sanitize Storage container SanitizeStorage #scan and save results to storage RunAzSDKScan if ($isAzSDKAvailable) { #helper schedule not needed anymore Remove-AzureRmAutomationSchedule -Name $CAHelperScheduleName -ResourceGroupName $AutomationAccountRG -AutomationAccountName $AutomationAccountName -Force } PublishEvent -EventName "CA Scan Completed" -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds} } catch { PublishEvent -EventName "CA Scan Error" -Properties @{ "ErrorRecord" = ($_ | Out-String) } -Metrics @{"TimeTakenInMs" = $scanAgentTimer.ElapsedMilliseconds; "SuccessCount" = 0} } |