Private/DownloadFilesInParallel.ps1
|
function DownloadFilesInParallel { [CmdletBinding()] param ( [Parameter(Mandatory)] [System.Collections.ArrayList] $FileInfoList, [Parameter(Mandatory)] [string] $OutputPath, [Parameter()] [int] $MaxSimultaneousTransfers = 4 ) $ENDED_BITSJOB_STATES = @("Suspended", "Error", "TransientError", "Transferred", "Canceled") $ENDED_JOB_STATES = @("Blocked", "Completed", "Disconnected", "Failed", "Stopped", "Suspended") try { $startTime = Get-Date $minutesTaken = 0 $currentDownloadNumber = 0 $downloadsFinished = 0 $totalUpdateFiles = $FileInfoList.Count $activeTransfers = New-Object -TypeName "System.Collections.ArrayList" $filesLeft = New-Object -TypeName "System.Collections.Queue" $FileInfoList | ForEach-Object { $filesLeft.Enqueue($_) } while ($filesLeft.Count -gt 0 -or $activeTransfers.Count -gt 0) { $elapsedTime = (Get-Date) - $startTime $minutesTakenNew = [int]([Math]::Floor($elapsedTime.TotalMinutes)) if ($minutesTakenNew -gt $minutesTaken) { Write-Verbose "Downloading $totalUpdateFiles update file$( if ($totalUpdateFiles -ne 1) { "s" } ), $downloadsFinished update file$( if ($downloadsFinished -ne 1) { "s" } ) downloaded." Write-Verbose "Elapsed time: $minutesTakenNew minute$( if ($minutesTakenNew -ne 1) { "s" } )." $minutesTaken = $minutesTakenNew } # Start another transfer if we're below the max simultaneous transfers threshold if ($filesLeft.Count -gt 0 -and $activeTransfers.Count -lt $MaxSimultaneousTransfers) { $currentDownloadNumber += 1 $downloadProgress = "$currentDownloadNumber/$totalUpdateFiles" $parsedFileInfo = $filesLeft.Dequeue() $fileDownloadUri = $parsedFileInfo["Uri"] $fileName = $parsedFileInfo["FileName"] $fileDownloadPath = Join-Path $OutputPath $fileName -ErrorAction Stop $fileDownloadPathTemp = "$fileDownloadPath.tmp" if (Test-Path $fileDownloadPath) { Write-Verbose "Skipping already downloaded update file $fileName ($downloadProgress)." continue } if (Test-Path $fileDownloadPathTemp) { Write-Verbose "Restarting download for incomplete update file $fileName ($downloadProgress)..." try { Remove-Item -Path $fileDownloadPathTemp -ErrorAction Stop } catch { Write-Error -Message "Could not remove incomplete update file ($fileDownloadPathTemp). Please remove this file manually and try again." return } } else { Write-Output "Downloading update file $fileName ($downloadProgress)..." } try { $transferParams = GetBitsTransferSplatBase -Source $fileDownloadUri $newBitsJob = Start-BitsTransfer @transferParams ` -DisplayName "Import-WsusUpdate parallel download for $fileName ($downloadProgress)" ` -Destination $fileDownloadPathTemp ` -Asynchronous ` -ErrorAction Stop [void]($activeTransfers.Add(@{ "FileName" = $fileName "TempPath" = $fileDownloadPathTemp "Path" = $fileDownloadPath "BitsJob" = $newBitsJob })) # Immediately restart the loop to queue up another job asap. continue } catch { Write-Warning "BITS transfer for update file $fileName failed with the following error: $_" Write-Verbose "Full error info:" Write-Verbose ($_ | Format-List -Force | Out-String) Write-Warning "Retrying download once more with Invoke-WebRequest as a fallback..." try { $webRequestParams = GetWebRequestSplatBase -Uri $fileDownloadUri $webRequestParams["OutFile"] = $fileDownloadPathTemp # TODO: Deserialization in job prevents WebSession from working, although this likely won't matter right now. # "Cannot convert the "Microsoft.PowerShell.Commands.WebRequestSession" value of type "Deserialized.Microsoft.PowerShell.Commands.WebRequestSession" to type "Microsoft.PowerShell.Commands.WebRequestSession". $webRequestParams["SessionVariable"] = $null $webRequestParams["WebSession"] = $null $job = Start-Job -Name "Import-WsusUpdate parallel download fallback for $fileName ($downloadProgress)" -ErrorAction Stop -ScriptBlock { [CmdletBinding()] param () # Hide progress since this is a parallel background download and it will speed up # Invoke-WebRequest. $ProgressPreference = "SilentlyContinue" $webRequestParams = $using:webRequestParams [void](Invoke-WebRequest @webRequestParams -ErrorAction Stop) } [void]($activeTransfers.Add(@{ "FileName" = $fileName "TempPath" = $fileDownloadPathTemp "Path" = $fileDownloadPath "WebRequestJob" = $job })) } catch { Write-Warning "Failed to start fallback download for update file $fileName (URI: $fileDownloadUri) with the following error: $_" throw } } } # We don't need to start another job - track progress of our current transfers if ($activeTransfers.Count -gt 0) { $transferJustFinished = $false for ($index = 0; $index -lt $activeTransfers.Count; $index += 1) { $activeTransfer = $activeTransfers[$index] if ($null -ne $activeTransfer["BitsJob"]) { $updateBitsJob = Get-BitsTransfer -JobId $activeTransfer["BitsJob"].JobId -ErrorAction Stop if ($updateBitsJob.JobState -notin $ENDED_BITSJOB_STATES) { continue } if ($updateBitsJob.JobState -ne "Transferred") { Write-Error -Message "Download for update file $( $activeTransfer["FileName"] ) ended unfinished with state $( $updateBitsJob.JobState )." ` -Category OperationStopped ` -ErrorId "ParallelBITSTransferFailed" return } Complete-BitsTransfer -BitsJob $activeTransfer["BitsJob"] -ErrorAction Stop } elseif ($null -ne $activeTransfer["WebRequestJob"]) { $jobState = $activeTransfer["WebRequestJob"] if ($jobState.State -notin $ENDED_JOB_STATES) { continue } if ($jobState.State -ne "Completed") { Write-Error -Message "Fallback download for update file $( $activeTransfer["FileName"] ) ended unfinished with state $( $jobState.State )." ` -Category OperationStopped ` -ErrorId "ParallelInvokeWebRequestTransferFailed" return } Remove-Job -Job $activeTransfer["WebRequestJob"] -ErrorAction SilentlyContinue -Confirm:$false } else { throw "Unexpected error: BitsJob and WebRequestJob were both null." } Write-Verbose "Finished downloading update file $( $activeTransfer["FileName"] )..." Write-Verbose "Moving finished download to its proper path $( $activeTransfer["Path"] )." Move-Item -Path $activeTransfer["TempPath"] -Destination $activeTransfer["Path"] -ErrorAction Stop $downloadsFinished += 1 $activeTransfers.RemoveAt($index) $index -= 1 $transferJustFinished = $true } if ($transferJustFinished) { # Skip sleep and start new job immediately continue } } # Check for progress updates every so often Start-Sleep -Milliseconds 500 } } finally { if ($activeTransfers.Count -gt 0) { Write-Verbose "Cleaning up $( $activeTransfers.Count ) active transfer$( if ( $activeTransfers.Count -gt 1 ) { "s" } )." while ($activeTransfers.Count -gt 0) { if ($null -ne $activeTransfers[0]["BitsJob"]) { Remove-BitsTransfer -BitsJob $activeTransfers[0]["BitsJob"] -ErrorAction SilentlyContinue } elseif ($null -ne $activeTransfers[0]["WebRequestJob"]) { [void](Stop-Job -Job $activeTransfers[0]["WebRequestJob"] -ErrorAction SilentlyContinue -Confirm:$false) Remove-Job -Job $activeTransfers[0]["WebRequestJob"] -ErrorAction SilentlyContinue -Confirm:$false } else { Write-Verbose "Skipping removal of lingering transfer as both BitsJob and WebRequestJob were null." } $activeTransfers.RemoveAt(0) } } } } # Copyright (c) 2023 AJ Tek Corporation. All Rights Reserved. |