Modules/Core/ARIResourceDataPull.psm1
|
<#
.Synopsis Main module for Resource Extraction .DESCRIPTION This module is the main module for the Azure Resource Graphs that will be run against the environment. .Link https://github.com/microsoft/ARI/Core/Start-AzureResourceExtraction.psm1 .COMPONENT This powershell Module is part of Azure Resource Inventory (ARI) .NOTES Version: 4.0.1 First Release Date: 15th Oct, 2024 Authors: Claudio Merola #> Function Start-AzureResourceDataPull { Param($ManagementGroup,$Subscriptions,$SubscriptionID,$ResourceGroup,$SecurityCenter,$SkipAdvisory,$SkipPolicy,$IncludeTags,$QuotaUsage,$TagKey,$TagValue) Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Starting Extractor function') Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Powershell Edition: ' + ([string]$psversiontable.psEdition)) Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Powershell Version: ' + ([string]$psversiontable.psVersion)) #Field for tags if ($IncludeTags.IsPresent) { Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+"Tags will be included") $GraphQueryTags = ",tags " } else { Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+"Tags will be ignored") $GraphQueryTags = "" } <###################################################### Subscriptions ######################################################################> Write-Progress -activity 'Azure Inventory' -Status "1% Complete." -PercentComplete 2 -CurrentOperation 'Discovering Subscriptions..' if (![string]::IsNullOrEmpty($ManagementGroup)) { Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Management group name supplied: ' + $ManagmentGroupName) $group = az account management-group entities list --query "[?name =='$ManagementGroup']" | ConvertFrom-Json if ($group.Count -lt 1) { Write-Host "ERROR:" -NoNewline -ForegroundColor Red Write-Host "Management Group $ManagementGroup not found!" Write-Host "" Write-Host "Please check the Management Group name and try again." Write-Host "" Exit } else { Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Management groups found: ' + $group.count) foreach ($item in $group) { $Subscriptions = @() $GraphQuery = "resourcecontainers | where type == 'microsoft.resources/subscriptions' | mv-expand managementGroupParent = properties.managementGroupAncestorsChain | where managementGroupParent.name =~ '$($item.name)' | summarize count()" $EnvSize = az graph query -q $GraphQuery --output json --only-show-errors | ConvertFrom-Json $EnvSizeNum = $EnvSize.data.'count_' if ($EnvSizeNum -ge 1) { $Loop = $EnvSizeNum / 1000 $Loop = [math]::ceiling($Loop) $Looper = 0 $Limit = 0 while ($Looper -lt $Loop) { $GraphQuery = "resourcecontainers | where type == 'microsoft.resources/subscriptions' | mv-expand managementGroupParent = properties.managementGroupAncestorsChain | where managementGroupParent.name =~ '$($item.name)' | project id = subscriptionId" $Resource = (az graph query -q $GraphQuery --skip $Limit --first 1000 --output json --only-show-errors).tolower() | ConvertFrom-Json foreach ($Sub in $Resource.data) { $Subscriptions += az account show --subscription $Sub.id --output json --only-show-errors | ConvertFrom-Json } Start-Sleep 2 $Looper ++ Write-Progress -Id 1 -activity "Running Subscription Inventory Job" -Status "$Looper / $Loop of Subscription Jobs" -PercentComplete (($Looper / $Loop) * 100) $Limit = $Limit + 1000 } } Write-Progress -Id 1 -activity "Running Subscription Inventory Job" -Status "$Looper / $Loop of Subscription Jobs" -Completed } } } $SubCount = [string]$Subscriptions.id.count Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Number of Subscriptions Found: ' + $SubCount) Write-Progress -activity 'Azure Inventory' -Status "3% Complete." -PercentComplete 3 -CurrentOperation "$SubCount Subscriptions found.." <######################################################## INVENTORY LOOPs #######################################################################> $ExtractionRuntime = Measure-Command -Expression { Write-Progress -Id 1 -activity "Running Inventory Jobs" -Status "1% Complete." -Completed function Invoke-APIRequest { param($url,$header) $APIResult = Invoke-RestMethod -Uri $url -Headers $header -Method GET return $APIResult.value } Write-Progress -activity 'Azure Inventory' -Status "4% Complete." -PercentComplete 4 -CurrentOperation "Starting Resources extraction jobs.." if(![string]::IsNullOrEmpty($ResourceGroup) -and [string]::IsNullOrEmpty($SubscriptionID)) { Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Resource Group Name present, but missing Subscription ID.') Write-Output '' Write-Output 'If Using the -ResourceGroup Parameter, the Subscription ID must be informed' Write-Output '' Exit } else { $Subscri = $Subscriptions.id $RGQueryExtension = '' $TagQueryExtension = '' $MGQueryExtension = '' if(![string]::IsNullOrEmpty($ResourceGroup) -and ![string]::IsNullOrEmpty($SubscriptionID)) { $RGQueryExtension = "| where resourceGroup in~ ('$([String]::Join("','",$ResourceGroup))')" } elseif(![string]::IsNullOrEmpty($TagKey) -and ![string]::IsNullOrEmpty($TagValue)) { $TagQueryExtension = "| where isnotempty(tags) | mvexpand tags | extend tagKey = tostring(bag_keys(tags)[0]) | extend tagValue = tostring(tags[tagKey]) | where tagKey =~ '$TagKey' and tagValue =~ '$TagValue'" } elseif (![string]::IsNullOrEmpty($ManagementGroup)) { $MGQueryExtension = "| join kind=inner (resourcecontainers | where type == 'microsoft.resources/subscriptions' | mv-expand managementGroupParent = properties.managementGroupAncestorsChain | where managementGroupParent.name =~ '$ManagementGroup' | project subscriptionId, managanagementGroup = managementGroupParent.name) on subscriptionId" $MGContainerExtension = "| mv-expand managementGroupParent = properties.managementGroupAncestorsChain | where managementGroupParent.name =~ '$ManagementGroup'" } } $GraphQuery = "resources $RGQueryExtension $TagQueryExtension $MGQueryExtension | project id,name,type,tenantId,kind,location,resourceGroup,subscriptionId,managedBy,sku,plan,properties,identity,zones,extendedLocation$($GraphQueryTags) | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Resources') $Resources += Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Resources' $GraphQuery = "networkresources $RGQueryExtension $TagQueryExtension $MGQueryExtension | project id,name,type,tenantId,kind,location,resourceGroup,subscriptionId,managedBy,sku,plan,properties,identity,zones,extendedLocation$($GraphQueryTags) | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Network Resources') $Resources += Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Network Resources' $GraphQuery = "recoveryservicesresources $RGQueryExtension $TagQueryExtension | where type =~ 'microsoft.recoveryservices/vaults/backupfabrics/protectioncontainers/protecteditems' or type =~ 'microsoft.recoveryservices/vaults/backuppolicies' $MGQueryExtension | project id,name,type,tenantId,kind,location,resourceGroup,subscriptionId,managedBy,sku,plan,properties,identity,zones,extendedLocation$($GraphQueryTags) | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Backup Resources') $Resources += Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Backup Items' $GraphQuery = "desktopvirtualizationresources $RGQueryExtension $MGQueryExtension| project id,name,type,tenantId,kind,location,resourceGroup,subscriptionId,managedBy,sku,plan,properties,identity,zones,extendedLocation$($GraphQueryTags) | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for AVD Resources') $Resources += Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Virtual Desktop' $GraphQuery = "resourcecontainers $RGQueryExtension $TagQueryExtension $MGContainerExtension | project id,name,type,tenantId,kind,location,resourceGroup,subscriptionId,managedBy,sku,plan,properties,identity,zones,extendedLocation$($GraphQueryTags) | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Resource Containers') $ResourceContainers = Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Subscriptions and Resource Groups' if (!($SkipPolicy.IsPresent)) { $GraphQuery = "policyresources | where type == 'microsoft.authorization/policyassignments' | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Policies Resources') $Policies = Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Policies' } if (!($SkipAdvisory.IsPresent)) { $GraphQuery = "advisorresources $RGQueryExtension $MGQueryExtension | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Advisories') $Advisories = Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Advisories' } if ($SecurityCenter.IsPresent) { $GraphQuery = "securityresources $RGQueryExtension | where type =~ 'microsoft.security/assessments' and properties['status']['code'] == 'Unhealthy' $MGQueryExtension | order by id asc" Write-Debug ((get-date -Format 'yyyy-MM-dd_HH_mm_ss')+' - '+'Invoking Inventory Loop for Security Resources') $Security = Invoke-ResourceInventoryLoop -GraphQuery $GraphQuery -FSubscri $Subscri -LoopName 'Security Center' } Write-Progress -activity 'Azure Inventory' -Status "4% Complete." -PercentComplete 4 -CurrentOperation "Starting API Extraction.." <# $Token = az account get-access-token --only-show-errors --output json | ConvertFrom-Json $header = @{ 'Authorization' = 'Bearer ' + $Token.accessToken } $AzURL = 'management.azure.com' $ResourceHealthHistoryDate = (Get-Date).AddMonths(-12) $APIResults = @() $VMLocations = $Resources | Where-Object {$_.Type -in 'microsoft.compute/virtualmachines','microsoft.compute/virtualmachinescalesets'} foreach ($Sub in $Subscri) { #ResourceHealth Events $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.ResourceHealth/events?api-version=2022-10-01&queryStartTime=' + $ResourceHealthHistoryDate) $ResourceHealth = Invoke-APIRequest -url $url -header $header #Support Tickets $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.Support/supportTickets?api-version=2020-04-01') $SupTickets = Invoke-APIRequest -url $url -header $header #Quotas $Locations = ($VMLocations | Where-Object {$_.subscriptionId -eq $Sub} | Group-Object -Property Location).name $Quotas = @() foreach ($loc in $Locations) { $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.Compute/locations/'+ $loc +'/providers/Microsoft.Quota/usages?api-version=2023-02-01') $Quotas += Invoke-APIRequest -url $url -header $header } #Maintenances $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.Maintenance/publicMaintenanceConfigurations?api-version=2023-09-01-preview') $Maintenances = Invoke-APIRequest -url $url -header $header #Managed Identities $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.ManagedIdentity/userAssignedIdentities?api-version=2023-01-31') $Identities = Invoke-APIRequest -url $url -header $header #Advisor Score $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.Advisor/advisorScore?api-version=2023-01-01') $ADVScore = Invoke-APIRequest -url $url -header $header #Availability Status $url = ('https://' + $AzURL + '/subscriptions/' + $Sub + '/providers/Microsoft.ResourceHealth/availabilityStatuses?api-version=2024-02-01') $AvailStatus = Invoke-APIRequest -url $url -header $header $tmp = @{ 'Subscription' = $Sub; 'ResourceHealth' = $ResourceHealth; 'SupportTickets' = $SupTickets; 'Quotas' = $Quotas; 'Maintenance' = $Maintenances; 'ManagedIdentities' = $Identities; 'AdvisorScore' = $ADVScore; 'AvailabilityStatus' = $AvailStatus } $APIResults += $tmp } Remove-Variable -Name VMLocations #VM Reservation $url = 'https://management.azure.com/providers/Microsoft.Capacity/reservations?api-version=2022-11-01' $Reservation = Invoke-APIRequest -url $url -header $header $Body = @{ reportType = "OverallSummaryReport" subscriptionList = @($Subscri) carbonScopeList = @("Scope1") dateRange = @{ start = "2024-05-01" end = "2024-05-30" } } $url = 'https://management.azure.com/providers/Microsoft.Carbon/carbonEmissionReports?api-version=2023-04-01-preview' #$url = 'https://management.azure.com/providers/Microsoft.Carbon/queryCarbonEmissionDataAvailableDateRange?api-version=2023-04-01-preview' $APIResult = Invoke-RestMethod -Uri $url -Headers $header -Body ($Body | ConvertTo-Json) -Method POST -ContentType 'application/json' $APIResult.value #> <######################################################### QUOTA JOB ######################################################################> if($QuotaUsage.isPresent) { Start-Job -Name 'Quota Usage' -ScriptBlock { $Location = @() Foreach($Sub in $($args[1])) { $Locs = ($($args[0]) | Where-Object {$_.subscriptionId -eq $Sub.id -and $_.Type -in 'microsoft.compute/virtualmachines','microsoft.compute/virtualmachinescalesets'} | Group-Object -Property Location).name $Val = @{ 'Loc' = $Locs; 'Sub' = $Sub.name } $Location += $Val } $Quotas = @() Foreach($Loc in $Location) { if($Loc.Loc.count -eq 1) { $Quota = az vm list-usage --location $Loc.Loc --subscription $Loc.Sub -o json | ConvertFrom-Json $Quota = $Quota | Where-Object {$_.CurrentValue -ge 1} $Q = @{ 'Location' = $Loc.Loc; 'Subscription' = $Loc.Sub; 'Data' = $Quota } $Quotas += $Q } else { foreach($Loc1 in $Loc.loc) { $Quota = az vm list-usage --location $Loc1 --subscription $Loc.Sub -o json | ConvertFrom-Json $Quota = $Quota | Where-Object {$_.CurrentValue -ge 1} $Q = @{ 'Location' = $Loc1; 'Subscription' = $Loc.Sub; 'Data' = $Quota } $Quotas += $Q } } } $Quotas } -ArgumentList $Resources, $Subscriptions } Write-Progress -activity 'Azure Inventory' -PercentComplete 20 Write-Progress -Id 1 -activity "Running Inventory Jobs" -Status "100% Complete." -Completed } $tmp = [pscustomobject]@{ ExtractionRunTime = $ExtractionRuntime Resources = $Resources ResourceContainers = $ResourceContainers Policies = $Policies Advisories = $Advisories Security = $Security } return $tmp } |