JobLogsForICM

1.1

JobLogsForICM.ps1


<#PSScriptInfo
 
.VERSION 1.1
 
.GUID f7037068-1135-4a5d-bd75-82acdbc1f0d5
 
.AUTHOR nali2@microsoft.com
 
.COMPANYNAME
 
.COPYRIGHT
 
.TAGS
 
.LICENSEURI
 
.PROJECTURI
 
.ICONURI
 
.EXTERNALMODULEDEPENDENCIES
 
.REQUIREDSCRIPTS
 
.EXTERNALSCRIPTDEPENDENCIES
 
.RELEASENOTES
 
 
.PRIVATEDATA
 
#>

<#
 
.DESCRIPTION
 Job logs collector tool
 
#> 

param (
    [Parameter(Mandatory = $true)]
    [string]$subscriptionId,

    [Parameter(Mandatory = $true)]
    [string]$resourceGroupName,

    [Parameter(Mandatory = $true)]
    [string]$automationAccountName,

    [Parameter(Mandatory = $true)]
    [string[]]$jobIds
)


<#
.SYNOPSIS
    This PowerShell script collects required information of Azure automation and jobs to Microsoft CSS team for further investigation.
 
.DESCRIPTION
 
    This PowerShell script is designed to collect all logs in an Azure Automation account.
 
.PARAMETER subscriptionId
    Required. Subscription of the Azure Automation account.
  
.PARAMETER resourceGroupName
    Required. The name of the resource group of the Azure Automation account.
     
.PARAMETER automationAccountName
    Required. The name of the Azure Automation account.
 
.PARAMETER jobIds
    Required. The list of job ids which has issues.
 
.NOTES
    AUTHOR: Nina Li
    LASTEDIT: Mar 20, 2024
 
    example: .\JobLogsForICM.ps1 -subscriptionId <subscription id> -resourceGroupName <resource group name> -automationAccountName <automation account> -jobIds <job id 1>,<job id 2>
#>


$ErrorActionPreference = "SilentlyContinue"
$global:useLogFile = $true
$global:logFile = ""

[string]$global:defaultOutputDir = ""

Set-Variable REGEXE -Value ([string] "$($env:systemroot)\system32\reg.exe")
Set-Variable REGEDIT -Value ([string] "$($env:systemroot)\regedit.exe")

function enableScriptExecution
{
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Setting script execution policy to unrestricted."

    try
    {
        $execp = Get-ExecutionPolicy

        if(-not($execp -eq "Unrestricted") -and -not($execp -contains 'Bypass'))
        {
            if(-not($force) -and -not($PSCmdlet.ShouldContinue(("Your current policy " + $execp +" is not executable policy, so the script cannot be loaded, see details https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.security/set-executionpolicy?view=powershell-7.2."), "Would you still like to set execution policy with unrestricted?")))
            {
                Write-Host "User has chosen to reject this request, skipping log collection, please set execution policy to bypass or unrestricted before continuing"
                exit
            }
        }

        Set-ExecutionPolicy unrestricted

    }
    catch 
    {
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Failed to set script's execution policy."
    }
}

function Get-Timestamp
{
    $timestamp = (Get-Date).ToString("hh_mm_MM_dd_yyyy")
    return $timestamp
}

function Get-TemporaryDirectoryLocation
{
    [OutputType([String])]
    Param ($tempPath)

    $dirName = $env:computername
    $dirName += "_"
    $dirName += Get-Timestamp
    $path = ""

    try 
    {
        $path = [System.IO.Path]::GetTempPath() + $dirName    
    }
    catch 
    {
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Exception: $_.Exception.Message"
    }

    return $path    
}

Function Write-Log
{
    [cmdletbinding()]
    Param
    (
        [Parameter(Mandatory=$False)]
        [ValidateSet("INFO","WARN","ERROR","FATAL","DEBUG")]
        [String]
        $Level = "INFO",

        [Parameter(Mandatory=$True)]
        [String]
        $FunctionName,

        [Parameter(Mandatory=$True)]
        [AllowEmptyString()]
        [String]
        $Message
    )

    $timestamp = Get-Timestamp
    $log = "$Level || $FunctionName || $Message"
    if($global:uselogfile)
    {
        Add-Content $global:logfile -Value $log
        Write-Output $log
    }
    else 
    {
        Write-Output $log
    }
}

function init
{
    
    enableScriptExecution

    $global:defaultOutputDir = (Get-TemporaryDirectoryLocation).ToString()
    if($true -eq [string]::IsNullOrEmpty($global:defaultOutputDir))
    {
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Default output dir empty: $global:defaultOutputDir"
        throw 'init failed'
    }
    New-Item -Path $global:defaultOutputDir -ItemType "directory" -Force | Out-Null
    $global:logFile = $global:defaultOutputDir + "\..\Jobs.log"

    if (Test-Path $global:logFile)
    {
        Write-Output "Removing tool's old log file."
        Remove-Item $global:logFile -Force
    }
    
    if($True -eq $global:useLogFile)
    {
        Write-Output "Tool's log data is being redirected to $global:logFile"
    }    

    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Starting data collection."
}

# Function to log in to Azure
function Login-AzAccount {
    try
    {
        # This script login to azure by user credential from local powershell window.
        "Logging in to Azure..."
        Connect-AzAccount -SubscriptionId $subscriptionId
        #Connect-AzAccount -SubscriptionId "aa1ec426-0f22-4a21-9ab1-291094563bc4"
    }
    catch {
        Write-Error -Message $_.Exception
        throw $_.Exception
    }
}

function Show-Menu
{
    Clear-Host
    Write-Host "================ Please select the platform of job execution, select option #2 if you have both cloud jobs and HRW jobs ================"
    Write-Host 
    Write-Host "1: Run on Azure"
    Write-Host "2: Run on HRW (Hybrid runbook worker)"
    Write-Host "Q: Press 'Q' to quit."
    Write-Host "============================================================================================"
}

function enableScriptExecution
{
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Setting script execution policy to unrestricted."

    try
    {
        $execp = Get-ExecutionPolicy

        if(-not($execp -eq "Unrestricted") -and -not($execp -contains 'Bypass'))
        {
            if(-not($force) -and -not($PSCmdlet.ShouldContinue(("Your current policy " + $execp +" is not executable policy, so the script cannot be loaded, see details https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.security/set-executionpolicy?view=powershell-7.2."), "Would you still like to set execution policy with unrestricted?")))
            {
                Write-Host "User has chosen to reject this request, skipping log collection, please set execution policy to bypass or unrestricted before continuing"
                exit
            }
        }

        Set-ExecutionPolicy unrestricted

    }
    catch 
    {
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Failed to set script's execution policy."
    }
}

function getAccountInfo
{
    $AADirectory = "$global:defaultOutputDir\AccountInfo"
    if ($False -eq (Test-Path $AADirectory) )
    {
        New-Item -Path $jobsDirectory -ItemType "directory" -Force | Out-Null
    }

    $AAinfo = Get-AzAutomationAccount -ResourceGroupName $resourceGroupName -Name $automationAccountName
    $region = $AAinfo.Location
    $network = $AAinfo.PublicNetworkAccess

    Write-Log -FunctionName $MyInvocation.MyCommand -Message "=== AA info ==="
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Region: $region"
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Is public network enabled: $network"

    $AA = Invoke-AzRestMethod -Method GET -Uri "https://management.azure.com/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.Automation/automationAccounts/$automationAccountName?api-version=2023-11-01"
    $content = $AA.Content
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Save detailed automation account info ..."
   
    $content | Out-File $AADirectory\$automationAccountName.log
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "..............................."
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "..............................."
}

function getCloudJobInfo
{
    init

    getAccountInfo

    $jobsDirectory = "$global:defaultOutputDir\CloudJobLogs"
    if ($False -eq (Test-Path $jobsDirectory) )
    {
        New-Item -Path $jobsDirectory -ItemType "directory" -Force | Out-Null
    }

    foreach ($jobId in $jobIds){
        $job = Get-AzAutomationJob -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -Id $jobId
        $id = $job.JobId
        $state = $job.Status
        $param = $job.JobParameters | ConvertTo-Json
        $startTime = $job.StartTime
        $endTime = $job.EndTime
        $runbookName = $job.RunbookName
        $except = $job.Exception
        $logs = Get-AzAutomationJobOutput -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -Id $jobId -Stream "Any" | ft Time, Type, Summary | Out-String -Width 4096
        $runbook = Get-AzAutomationRunbook -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -Name $runbookName
        $runbookModified = $runbook.LastModifiedTime

        $script = Invoke-AzRestMethod -Method GET -Uri "https://management.azure.com/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.Automation/automationAccounts/$automationAccountName/runbooks/$runbookName/content?api-version=2023-11-01"
        $content = $script.Content

        Write-Log -FunctionName $MyInvocation.MyCommand -Message "=== job info ==="
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Job id: $id"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Runbook name: $runbookName"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Last modified time of this runbook: $runbookModified"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Job status: $state"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Start Time: $startTime"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "End Time: $endTime"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Input parameters: $param"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "=== job logs ==="
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Saving runbook content ..."
        $content | Out-File $jobsDirectory\$runbookName_$id.ps1.txt
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Runbook content saved!"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Save job exception if any"
        $except | Out-File $jobsDirectory\exception_$id.log
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Saving job logs ..."
        $logs | Out-File $jobsDirectory\$id.log
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Job logs saved!"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "..............................."
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "..............................."
    }
}

function getHRWJobInfo
{
    init

    getAccountInfo

    $HRWDirectory = "$global:defaultOutputDir\HRWJobLogs"
    if ($False -eq (Test-Path $HRWDirectory) )
    {
        New-Item -Path $HRWDirectory -ItemType "directory" -Force | Out-Null
    }

    foreach ($jobId in $jobIds){
        $job = Get-AzAutomationJob -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -Id $jobId

        $id = $job.JobId
        $state = $job.Status
        $param = $job.JobParameters | ConvertTo-Json
        $startTime = $job.StartTime
        $endTime = $job.EndTime
        $runbookName = $job.RunbookName
        $except = $job.Exception
        $logs = Get-AzAutomationJobOutput -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -Id $jobId -Stream "Any" | ft Time, Type, Summary | Out-String -Width 4096
        $runbook = Get-AzAutomationRunbook -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -Name $runbookName
        $runbookModified = $runbook.LastModifiedTime

        $script = Invoke-AzRestMethod -Method GET -Uri "https://management.azure.com/subscriptions/$subscriptionId/resourceGroups/$resourceGroupName/providers/Microsoft.Automation/automationAccounts/$automationAccountName/runbooks/$runbookName/content?api-version=2023-11-01"
        $content = $script.Content

        Write-Log -FunctionName $MyInvocation.MyCommand -Message "=== job info ==="
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Job id: $id"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Runbook name: $runbookName"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Last modified time of this runbook: $runbookModified"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Job status: $state"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Start Time: $startTime"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "End Time: $endTime"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Input parameters: $param"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "=== job logs ==="
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Saving runbook content ..."
        $content | Out-File $HRWDirectory\$runbookName_$id.ps1.txt
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Runbook content saved!"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Save job exception if any"
        $except | Out-File $HRWDirectory\exception_$id.log
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Saving job logs ..."
        $logs | Out-File $HRWDirectory\$id.log
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Job logs saved!"


        $HRWG = $job.HybridWorker

        if (!$HRWG)
        {
            Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Failed to find any HRW for this job, please double check if this job is running on HRW"
            Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Skipping HRW info collection..."
        }
        else{
            $HRW = Get-AzAutomationHybridRunbookWorker -AutomationAccountName $automationAccountName -ResourceGroupName $resourceGroupName -HybridRunbookWorkerGroupName $HRWG
            $currentTime = [DateTime]::UtcNow
            
            foreach ($hw in $HRW){
                $worker = $hw.WorkerName
                $id = $hw.Name
                $regtime = $hw.RegisteredDateTime
                $lasttime = $hw.LastSeenDateTime
                Write-Log -FunctionName $MyInvocation.MyCommand -Message "=== HRW info ==="
                Write-Log -FunctionName $MyInvocation.MyCommand -Message "HRW Name:$worker"
                Write-Log -FunctionName $MyInvocation.MyCommand -Message "HRW id: $id"
                Write-Log -FunctionName $MyInvocation.MyCommand -Message "Current datetime in UTC: $currentTime"
                Write-Log -FunctionName $MyInvocation.MyCommand -Message "HRW registration time: $regtime"
                Write-Log -FunctionName $MyInvocation.MyCommand -Message "Last seen time: $lasttime"
            }
        }
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "..............................."
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "..............................."
    }
}

function Create-Zip
{
    [cmdletbinding()]
    Param
    (
        [Parameter(Mandatory=$True)]
        [String]
        $Source,

        [Parameter(Mandatory=$True)]
        [String]
        $Destination
    )

    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Compressing logs collected in folder: $Source"
    
    try 
    {
        Add-Type -Path "C:\Windows\Microsoft.Net\assembly\GAC_MSIL\System.IO.Compression.FileSystem\v4.0_4.0.0.0__b77a5c561934e089\System.IO.Compression.FileSystem.dll"
        [System.IO.Compression.ZipFile]::CreateFromDirectory($Source, $Destination, [System.IO.Compression.CompressionLevel]::Optimal, $false)
    }
    catch 
    {
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Exception: $_.Exception.Message"
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Failed to create zip with path: $Destination"
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Collected logs are stored at: $Source. Please manually zip the folder."
        exit
    }    
}

function CheckLogsForErrors
{
    [CmdletBinding()]
    Param (
        [Parameter(Position = 0, Mandatory = $True)]
        [String]
        $OutputFile
    )

    try
    {
        Get-ChildItem -Path $global:defaultOutputDir\* -recurse -exclude *.dmp,*.exe,*.etl, *.dll | Select-String -List error | Format-List * | Out-File -FilePath $OutputFile
    }
    catch
    {
        Write-Log -Level "Error" -FunctionName $MyInvocation.MyCommand -Message "Failed to CheckLogsForErrors."
        Write-Log -FunctionName $MyInvocation.MyCommand -Message "Exception: $_.Exception.Message"
    }
}

function archiveLogs
{
    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Data collection completed."

    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Analyzing collected logs for errors."
    $analysisResultsFile = "$global:defaultOutputDir\errors.txt"
    CheckLogsForErrors -OutputFile $analysisResultsFile

    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Analysis completed and written to file: $analysisResultsFile"

    Copy-Item "$global:defaultOutputDir\..\Jobs.log" -Destination $global:defaultOutputDir

    Create-Zip -Source "$global:defaultOutputDir" -Destination "$global:defaultOutputDir.zip"

    Remove-Item -Recurse -Force -Path $global:defaultOutputDir

    Write-Log -FunctionName $MyInvocation.MyCommand -Message "Collected logs available at: $global:defaultOutputDir.zip"

    Invoke-Expression "explorer '/select,$global:defaultOutputDir.zip'"
}

function main
{
    Login-AzAccount

    $AAinfo = Get-AzAutomationAccount -ResourceGroupName $resourceGroupName -Name $automationAccountName
    if (!$AAinfo){
        Write-Host "In your subscription $subscriptionId, Automation account $automationAccountName under resource group $resourceGroupName does not exist, please double check name and restart the script!"
        return
    }
    
    Show-Menu    

    $selection = Read-Host "Please select an option"
    switch ($selection)
    {
        '1'
        {
            'You selected option #1'
            getCloudJobInfo
        }
        '2'
        {
            'You selected option #2'
            getHRWJobInfo
        }
        'q' 
        {
            return
        }
        'Q'
        {
            return
        }
    }
    archiveLogs
}

main