ReplicationReport.ps1
|
<#==============================================================================
File Name : ReplicationReport.ps1 Original Author : Kenneth C. Mazie (kcmjr @ kcmjr.com) : Description : This script will query multiple Tegile Zebi Controllers, : parse the replication logs, and populate an Excel spreadsheet. : The results are pulled from back as far as a preset number of : days. It then emails the resulting spreadsheet. X number of : prior runs are retained to avoid filling the disk. : Arguments : Named commandline parameters: (all are optional) : "-console" - Displays console output during run. : "-debug" - Switches email recipient and some criteria. : Notes : Settings are loaded from an XML file located in the script folder. : See the end of the script for config file example. The config file should : use the same name as the script but with a .XML extension. : Requirements : Requires PS v5. Requires Posh-SSH module. : https://www.powershelladmin.com/wiki/SSH_from_PowerShell_using_the_SSH.NET_library#The_PowerShell_gallery_version : Warnings : Make absolutely sure the proper user is in the config file AND on each Zebi : as well as is listed in the sshd_config file on the zebi. : ----------------------------------------------------------------------- : IMPORTANT!! Firmware updates to the array will DELETE any custom SSH : users you create. Every time you update the firmware you MUST go into : the array and recreate your customs SSH user and password. Usually the : entry in the sshd_config file allowing the custom user access will remain : but best to verify that as well. : ----------------------------------------------------------------------- : Legal : Public Domain. Modify and redistribute freely. No rights reserved. : SCRIPT PROVIDED "AS IS" WITHOUT WARRANTIES OR GUARANTEES OF : ANY KIND. USE AT YOUR OWN RISK. NO TECHNICAL SUPPORT PROVIDED. : Credits : Code snippets and/or ideas came from many sources around the web. : Last Update by : Kenneth C. Mazie (email kcmjr AT kcmjr.com for comments or to report bugs) Version History : v1.0 - 02-03-16 - Original Change History : v1.1 - 03-09-16 - Added XML config file : v2.0 - 09-02-16 - Added detection of failed jobs. Retooled parser. : v3.0 - 09-12-16 - Retooled for Pos-SSH v1.7.6. Eliminated PSEXEC : requirement. Eliminated work file requirement. Added save only : the last X copies. Added debug name change. Added note on email : that refers to failures found. : v4.0 - 11-04-16 - Complete redesign of time calculation. Now : uses epoch starting at 01-01-1970 as reference. This avoids issues : with using the week of the year. Also corrected for up to 4 different : variations is log formats seen in v3.5 Zebi firmware. : v4.00 - 02-13-17 - Minor bug fixes. : v4.01 - 06-19-17 - fixed issue with Posh-SSH install : v5.00 - 05-03-18 - Added notations about firmware upgrades causing issues #===============================================================================#> #requires -version 5.0 <#PSScriptInfo .VERSION 5.00 .GUID e68e2b85-b354-4f66-a2f0-2df5128d0557 .AUTHOR Kenneth C. Mazie (kcmjr AT kcmjr.com) .DESCRIPTION Opens an SSH connection to a Tegile Intelliflash (Zebi) SAN, then parses the logs looking for replication results. Emails an Excel spreadsheet. Can be adopted for any Linux or Unix based system. #> Param ( [switch]$Debug = $false, [switch]$Console = $false ) clear-host #--[ Store all the start up variables so you can clean up when the script finishes. ]-- if ($startupvariables) { try {Remove-Variable -Name startupvariables -Scope Global -ErrorAction SilentlyContinue } catch { } } New-Variable -force -name startupVariables -value ( Get-Variable | ForEach-Object { $_.Name } ) #If (!(Get-Module Posh-SSH)){Find-Module | Install-Module -Confirm $true } #If (!(Get-Module Posh-SSH)){ Import-Module "Posh-SSH" #-ErrorAction SilentlyContinue #} If ($Debug){$Script:Debug = $true} If ($Console){$Script:Console = $true} $ErrorActionPreference = "silentlycontinue" $Script:Attach = $true $Script:Datetime = Get-Date -Format "MM-dd-yyyy_HH:mm" #--[ Current date $ time ]-- $Script:Today = Get-Date -Format "MM-dd-yyyy" #--[ Current date ]-- $Script:ThisYear = Get-Date -Format "yyyy" #--[ Current year ]-- $EpochDiff = New-TimeSpan '01 January 1970 00:00:00' $(Get-Date) #--[ Seconds since 01-01-1970 ]-- $EpochSecs = [INT] $EpochDiff.TotalSeconds #--[ Rounded ]-- $EpochDays = [INT] (($EpochDiff.TotalSeconds)/86400) #--[ Converted to days ]-- $Script:StatusFail = $false $Computer = $Env:ComputerName $Script:ScriptName = ($MyInvocation.MyCommand.Name).split(".")[0] $Script:ConfigFile = "$PSScriptRoot\$ScriptName.xml" #--[ Excel Non-Interactive Fix ]------------------------------------------------ Try{ If (!(Test-path -Path "C:\Windows\System32\config\systemprofile\Desktop")){New-Item -Type Directory -Name "C:\Windows\System32\config\systemprofile\Desktop"} If (!(Test-path -Path "C:\Windows\SysWOW64\config\systemprofile\Desktop")){New-Item -Type Directory -Name "C:\Windows\SysWOW64\config\systemprofile\Desktop"} }Catch{ Write-host "Cannot create system profile desktop for Excel. Please create manually. Script aborted." -forgroundcolor red break } #--[ Excel will crash when run non-interactively via a scheduled task if these folders don't exist ]-- #--[ Functions ]---------------------------------------------------------------- Function ResetVariables { Get-Variable | Where-Object { $startupVariables -notcontains $_.Name } | ForEach-Object { try { Remove-Variable -Name "$($_.Name)" -Force -Scope "global" -ErrorAction SilentlyContinue -WarningAction SilentlyContinue }catch{ } } } Function SendEmail { If ($Script:Debug){$ErrorActionPreference = "stop"} $email = New-Object System.Net.Mail.MailMessage $email.From = $Script:EmailFrom $email.IsBodyHtml = $Script:EmailHTML If ($Script:Debug){ $email.To.Add($Script:DebugEmail) }Else{ $email.To.Add($Script:EmailTo) } $email.Subject = $Script:EmailSubject $email.Body = $Script:ReportBody If ($Script:Attach -eq $true){ $email.Attachments.Add("$PSScriptRoot\$Script:SaveFile.xlsx") } $smtp = new-object Net.Mail.SmtpClient($Script:SmtpServer) $smtp.Send($email) If ($Script:Log){Add-content -Path "$PSScriptRoot\debug.txt" -Value "-------------[ Sending Email ]--------------"} If ($Script:Console){Write-Host "`nEmail sent...`n"} } #--[ Read and load configuration file ]----------------------------------------- If (!(Test-Path "$PSScriptRoot\$ScriptName.xml")){ #--[ Error out if configuration file doesn't exist ]-- Write-host "MISSING CONFIG FILE. Script aborted." -forgroundcolor red break }Else{ [xml]$Script:Configuration = Get-Content "$PSScriptRoot\$ScriptName.xml" #--[ Load configuration ]-- $Script:DebugEmail = $Script:Configuration.Settings.Email.Debug $Script:EmailTo = $Script:Configuration.Settings.Email.To $Script:EmailHTML = $Script:Configuration.Settings.Email.HTML $Script:EmailSubject = $Script:Configuration.Settings.Email.Subject $Script:EmailFrom = $Script:Configuration.Settings.Email.From $Script:SmtpServer = $Script:Configuration.Settings.Email.SmtpServer $Script:UserName = $Script:Configuration.Settings.Credentials.Username $Script:Password = $Script:Configuration.Settings.Credentials.Password [array]$Script:Targets = $Script:Configuration.Settings.General.Targets $Script:DaysBack = $Script:Configuration.Settings.General.Days $Script:ReportsToKeep = $Script:Configuration.Settings.General.Reports #--[ How many reports to retain ]-- $Script:FileName = $Script:Configuration.Settings.General.FileName $Script:SPwd = ConvertTo-SecureString $Script:Password -AsPlainText -Force $Script:Cred = New-Object System.Management.Automation.PSCredential ($Script:UserName, $Script:SPwd) } #------------------------------------------------------------------------------- Get-ChildItem -Path "$PSScriptRoot\" | Where-Object {-not $_.PsIsContainer -and $_.Name -like "*.xlsx"} | Sort-Object -Descending -Property LastTimeWrite | Select-Object -Skip $Script:ReportsToKeep | Remove-Item If ($Script:Debug){ [string]$Script:TodayDate = Get-Date -Format MM-dd-yyyy_hhmm #--[ Adds hour and min to filename for debugging ]-- }Else{ [string]$Script:TodayDate = Get-Date -Format MM-dd-yyyy } [string]$Script:SaveFile = $Script:FileName+"-"+$Script:TodayDate If (Test-Path "$PSScriptRoot\$Script:SaveFile.xlsx"){ #--[ If not in debug mode will rename exisiting XLS to allow it to overwritten ]-- Rename-Item -Path "$PSScriptRoot\$Script:SaveFile.xlsx" -NewName "$PSScriptRoot\$Script:SaveFile-old.xlsx" } $Script:ReportBody = "Attached are the replication results from the prior "+$Script:DaysBack+" days." $Script:ReportBody += "<br>- All Zebi controllers are included regardless of whether they actively participate in replication or not." $Script:ReportBody += "<br>- Each controller has it's own tab on the spreadsheet." $Script:ReportBody += "<br>- Only outgoing replication jobs are tracked." $Color = @{ "1" = "green" "2" = "red" "3" = "yellow" "4" = "cyan" "5" = "magenta" } $MissingType = [System.Type]::Missing $Excel = "" $Excel = New-Object -ComObject Excel.Application If ($Script:Console -or $Script:Debug){ $Excel.visible = $True $Excel.DisplayAlerts = $true $Excel.ScreenUpdating = $true $Excel.UserControl = $true $Excel.Interactive = $true }Else{ $Excel.visible = $False $Excel.DisplayAlerts = $false $Excel.ScreenUpdating = $false $Excel.UserControl = $false $Excel.Interactive = $false } $TargetCount = $Script:Targets.Target.count $Workbook = $Excel.Workbooks.Add() $Workbook.Worksheets.Add($MissingType, $Workbook.Worksheets.Item($Workbook.Worksheets.Count), ($TargetCount) - $Workbook.Worksheets.Count, $Workbook.Worksheets.Item(1).Type) | Out-Null #--[ Note that new sheets are always adding in reverse ]-- $SheetCount = 1 $Target = @() Foreach ($Target in $Script:Targets.Target){ if ($Console){Write-Host "`n--[ Processing Target: $Target ]-----------------------------------" -ForegroundColor Yellow } $Count = 1 $Worksheet = $workbook.Sheets.Item($SheetCount) $Worksheet.Activate() $Worksheet.name = $Target $Row = 1 $Col = 1 $WorkSheet.Cells.Item($Row,$Col) = "Date:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Source IP:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Source Pool:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Target IP:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Target Pool:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Status:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Parsed Data:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.Cells.Item($Row,$Col) = "Raw Data:" $WorkSheet.Cells.Item($Row,$Col).font.bold = $true $WorkSheet.Cells.Item($Row,$Col).HorizontalAlignment = 1 $Col++ $WorkSheet.application.activewindow.splitcolumn = 0 $WorkSheet.application.activewindow.splitrow = 1 $WorkSheet.application.activewindow.freezepanes = $true $Resize = $WorkSheet.UsedRange [void]$Resize.EntireColumn.AutoFit() $Cmd = 'cat /var/log/tegile/zebi/zebirep.log | egrep "has completed|has failed"' #--[ The command sent to the Target controller ]-- #--[ Some optional other commands that could be used ]-- # 'uname -a ' # '/usr/sbin/zfs list -o name' # 'pwd' # 'df -h' # '/usr/sbin/dladm show-link –S' # '/usr/sbin/fmadm faulty' # '/usr/sbin/zpool list' # '/usr/sbin/zfs list -o name' # 'zfs list -t snapshot' # 'zfs list -o space -r hostname$/Local' # 'zpool iostat 2' #--[ Display ZFS I/O statistics every 2 seconds ]-- # 'zpool iostat -v 2' #--[ Display detailed ZFS I/O statistics every 2 seconds ]-- Get-SSHSession | Remove-SSHSession | Out-Null #--[ Clear out previous session if it still exists ]-- $SSH = New-SshSession -ComputerName $Target -Credential $Cred -AcceptKey #--[ Open new SSH session ]-- Sleep -Milliseconds 500 If ($SshSessions.connected){ #--[ verify SSH connection was successful ]----------- if ($Console){Write-Host "-- SSH Connected..." -ForegroundColor green} $Script:Return = $(Invoke-SSHCommand -SSHSession $SSH -Command $Cmd).Output #--[ Invoke SSH command and capture the output as a string ]-- $Row = 2 $Col = 1 $SourcePool = "" $DestPool = "" $TargetPool = "" $TargetIP = "" $SourceIP = "" $Line = "" $RepJob = "" $Status = "" #--[ Process each line of the input file ]-- ForEach ($Line in $Script:Return){ $JobDiffDays = "99" $JobDate = "" $JobDay = "" $JobMonth = "" $JobYear = "" $RepJob = $Line.Split("]")[2] If ($RepJob.Contains('null')){ $JobDate = (($RepJob.Split('@')[2]).split('-')[1]).split(' ')[0] }Else{ $JobDate = ($RepJob.Split('@')[1]).split('-')[1] } If ($JobDate -ne ""){ $JobYear = $JobDate.substring(0,4) $JobMonth = $JobDate.substring(4,2) $JobDay = $JobDate.substring(6,2) $JobDate = $JobMonth+'-'+$JobDay+'-'+$JobYear If (!([string]$JobDate -as [DateTime])){ $JobDate = (($RepJob.Split('@')[2]).split('-')[1]).split('=')[0] $JobYear = $JobDate.substring(0,4) $JobMonth = $JobDate.substring(4,2) $JobDay = $JobDate.substring(6,2) $JobDate = $JobMonth+'-'+$JobDay+'-'+$JobYear } $JobDiff = New-TimeSpan $JobDate $(Get-Date) #--[ Days since date of job ]-- $JobDiffDays = [int] (($JobDiff.TotalSeconds)/86400) #--[ Rounded & Converted to days ]-- If ($Line -like "*Abort*"){ $RepStatus = "ABORTED" $SourcePool = ($RepJob.Split('@')[0]).Split(' ')[2] $TargetPool = ($RepJob.Split(':')[1]).Split(' ')[0] $TargetIP = ($RepJob.Split(':')[0]).Split('>')[1] }Elseif($Line -like "*Abandon*"){ $RepStatus = "ABANDONED" }Else{ $RepStatus = $Line.Split("]")[3] If (($RepJob.Split('@')[0]).Split(' ')[2] -eq "from"){ $SourcePool = ($RepJob.Split('@')[0]).Split(' ')[3] $TargetPool = ($RepJob.Split(':')[1])#.Split('/')[0] $TargetIP = ($RepJob.Split(':')[0]).Split('>')[1] }Else{ $SourcePool = ($RepJob.Split('@')[0]).Split(' ')[2] $TargetPool = ($RepJob.Split(':')[1])#.Split('/')[0] $TargetIP = ($RepJob.Split(':')[0]).Split('>')[1] } } If($RepJob -like "*@Auto*"){ #$RepStatus = $Line.Split("]")[3] $SourcePool = ($RepJob.Split('@')[0]).Split(' ')[2] $TargetPool = (($RepJob.Split('>')[1]).Split(':')[1]).split(" ")[0] $TargetIP = ($RepJob.Split('>')[1]).Split(':')[0] } } #--[ Dump collected data into spreadsheet ]-- if ([int]$JobDiffDays -le [int]$Script:DaysBack){ if ($Console){ Write-Host "`n`n-----------------------------------------------" Write-Host "Today ="$Script:Today -ForegroundColor Yellow Write-Host "Date of Job ="$JobDate -ForegroundColor Yellow Write-Host "Days since job ran ="$JobDiffDays -ForegroundColor Yellow #Write-Host "Current Week ="$Script:CurrentWeek -ForegroundColor Yellow #Write-Host "Result ="$DateMathResult -ForegroundColor Yellow Write-Host "Match Criteria ="$Script:DaysBack -ForegroundColor Yellow Write-Host "Parsed Data ="$RepJob -ForegroundColor Yellow } if ($Console){Write-Host "-- Matched criteria --" -ForegroundColor magenta } if ($Console){Write-Host "Writing to tab ="$Target -ForegroundColor magenta } if ($Console){Write-Host "Writing to row ="$row -ForegroundColor magenta } #--[ Job run date ]----------------------------------------------------- $WorkSheet.Cells.Item($Row,$Col) = "$JobDate" Clear-Variable $JobDate $Col++ #--[ Source IP ]-------------------------------------------------------- $WorkSheet.Cells.Item($Row,$Col) = $Target $Col++ #--[ Source Pool ]------------------------------------------------------ if ($Console){Write-Host "Source Pool ="$SourcePool -ForegroundColor Yellow} $WorkSheet.Cells.Item($Row,$Col) = $SourcePool $Col++ #--[ Target IP ]-------------------------------------------------------- #$dest = ($a.Split("]")[0]).split(" ")[3] $destpool = (($a.Split("]")[0]).split(" ")[3]).Split(":")[1] if ($Console){Write-Host "Target IP ="$TargetIP} $WorkSheet.Cells.Item($Row,$Col) = $TargetIP $Col++ #--[ Destination Pool ]------------------------------------------------- if ($Console){Write-Host "Target Pool ="$TargetPool -ForegroundColor Yellow} $WorkSheet.Cells.Item($Row,$Col) = $TargetPool $Col++ #--[ Job Status ]------------------------------------------------------- If ($RepStatus.Contains("has failed")){ $Status = "FAILED" $Script:StatusFail = $true if ($Console){Write-Host "Job Status ="$Status -ForegroundColor Red} $WorkSheet.Cells.Item($Row,$Col).Interior.ColorIndex = 3 } If ($RepStatus -eq "aborted"){ $Status = "ABORTED" #$Script:StatusFail = $true if ($Console){Write-Host "Job Status ="$Status -ForegroundColor Red} $WorkSheet.Cells.Item($Row,$Col).Interior.ColorIndex = 6 } If ($RepStatus -eq "abandoned"){ $Status = "ABANDONED" #$Script:StatusFail = $true if ($Console){Write-Host "Job Status ="$Status -ForegroundColor Red} $WorkSheet.Cells.Item($Row,$Col).Interior.ColorIndex = 6 } If ($RepStatus.Contains("has completed")){ $Status = "GOOD" if ($Console){Write-Host "Job Status ="$Status -ForegroundColor Green} } $WorkSheet.Cells.Item($Row,$Col) = $Status $Col++ #--[ Parsed Data ]------------------------------------------------------ $Col++ $WorkSheet.Cells.Item($Row,$Col) = $RepJob $Col++ #--[ Raw Data ]--------------------------------------------------------- if ($Console){Write-Host "Raw data = "$Line -ForegroundColor Cyan } $WorkSheet.Cells.Item($Row,$Col) = $Line $Line = "" $RepJob = "" $Status = "" $Col = 1 $Row++ $Resize = $WorkSheet.UsedRange [void]$Resize.EntireColumn.AutoFit() } } }Else{ $Col = 6 $WorkSheet.Cells.Item(3,2) = "There is a connection issue with this target!!" $WorkSheet.Cells.Item(4,2) = "Recent firmware updates may have deleted the SSH user!!" $WorkSheet.Cells.Item(5,2) = "Please verify the SSH user on the array!!" if ($Console){Write-Host "`n=================================================" -ForegroundColor Red} if ($Console){Write-Host " There is a connection issue with this target. " -ForegroundColor Yellow} if ($Console){Write-Host " Verify that the SSH user exists on the array. " -ForegroundColor Yellow} if ($Console){Write-Host " If a user OTHER than ZEBIADMIN was being used " -ForegroundColor Cyan} if ($Console){Write-Host " and a firmware upgrade has been performed the " -ForegroundColor Cyan} if ($Console){Write-Host " the SSH user on the array has been deleted. " -ForegroundColor Cyan} if ($Console){Write-Host "=================================================" -ForegroundColor Red} } $SheetCount++ } If (Test-Path "$PSScriptRoot\$Script:SaveFile-old.xlsx"){Remove-Item -Path "$PSScriptRoot\$Script:SaveFile-old.xlsx" -Confirm:$false -Force:$true} $Workbook.SaveAs("$PSScriptRoot\$Script:SaveFile.xlsx") $Excel.Quit() $Excel = $Null If($Script:StatusFail){ $Script:ReportBody += "<br><br><font color=red>A replication failure was detected and is noted on the report</font><br>" }Else{ $Script:ReportBody += "<br><br><font color=green>No replication failures were detected.</font><br>" } $Script:ReportBody += "<br>Script executed on "+$Datetime+"<br><br>" Sleep -Seconds 2 SendEmail [gc]::Collect() [gc]::WaitForPendingFinalizers() ResetVariables if ($Console){Write-Host "--- Completed ---" -ForegroundColor Red } <#-----------------------------[ Config File ]--------------------------------- The configuration file must be named using the same name as the script and ".xml" and must reside in the same folder as the script. Below is the format and element list: <!-- Settings & Configuration File --> <Settings> <General> <ScriptName>ReplicationReport</ScriptName> <FileName>ZebiReplLogs</FileName> <DebugTarget>test-server</DebugTarget> <Targets> <Target>10.100.1.1</Target> <Target>10.100.1.2</Target> <Target>10.100.1.3</Target> <Target>10.100.1.4</Target> <Target>10.100.1.5</Target> <Target>10.100.1.6</Target> <Target>10.100.1.7</Target> <Target>10.100.1.8</Target> </Targets> <Days>2</Days> <Reports>20</Reports> </General> <Email> <From>WeeklyReports@mydomain.com</From> <To>me@mydomain.com</To> <Subject>Weekly Zebi Replication Status Report</Subject> <HTML>$true</HTML> <SmtpServer>10.100.1.10</SmtpServer> </Email> <Credentials> <UserName>zebiadminuser</UserName> <Password>zebiadminpwd</Password> </Credentials> </Settings> #> #> |