private/Microsoft-PrePostDeploymentScript.ps1
|
function Microsoft-PrePostDeploymentScript { [CmdletBinding()] param ( [parameter(Mandatory = $false)] [String] $armTemplate, [parameter(Mandatory = $false)] [String] $ResourceGroupName, [parameter(Mandatory = $false)] [String] $DataFactoryName, [parameter(Mandatory = $false)] [Bool] $predeployment=$true, [parameter(Mandatory = $false)] [Bool] $deleteDeployment=$false, [parameter(Mandatory = $false)] [String] $selectiveArmTemplate ,[parameter(Mandatory = $false)] [Bool] $DeleteNotInSource = $false ,[parameter(Mandatory = $false)] [Bool] $StartActiveTriggers = $true ) function getPipelineDependencies { param([System.Object] $activity) if ($activity.Pipeline) { return @($activity.Pipeline.ReferenceName) } elseif ($activity.Activities) { $result = @() $activity.Activities | ForEach-Object{ $result += getPipelineDependencies -activity $_ } return $result } elseif ($activity.ifFalseActivities -or $activity.ifTrueActivities) { $result = @() $activity.ifFalseActivities | Where-Object {$_ -ne $null} | ForEach-Object{ $result += getPipelineDependencies -activity $_ } $activity.ifTrueActivities | Where-Object {$_ -ne $null} | ForEach-Object{ $result += getPipelineDependencies -activity $_ } return $result } elseif ($activity.defaultActivities) { $result = @() $activity.defaultActivities | ForEach-Object{ $result += getPipelineDependencies -activity $_ } if ($activity.cases) { $activity.cases | ForEach-Object{ $_.activities } | ForEach-Object{$result += getPipelineDependencies -activity $_ } } return $result } else { return @() } } function pipelineSortUtil { param([Microsoft.Azure.Commands.DataFactoryV2.Models.PSPipeline]$pipeline, [Hashtable] $pipelineNameResourceDict, [Hashtable] $visited, [System.Collections.Stack] $sortedList) if ($visited[$pipeline.Name] -eq $true) { return; } $visited[$pipeline.Name] = $true; $pipeline.Activities | ForEach-Object{ getPipelineDependencies -activity $_ -pipelineNameResourceDict $pipelineNameResourceDict} | ForEach-Object{ pipelineSortUtil -pipeline $pipelineNameResourceDict[$_] -pipelineNameResourceDict $pipelineNameResourceDict -visited $visited -sortedList $sortedList } $sortedList.Push($pipeline) } function Get-SortedPipelines { param( [string] $DataFactoryName, [string] $ResourceGroupName ) $pipelines = Get-AzDataFactoryV2Pipeline -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $ppDict = @{} $visited = @{} $stack = new-object System.Collections.Stack $pipelines | ForEach-Object{ $ppDict[$_.Name] = $_ } $pipelines | ForEach-Object{ pipelineSortUtil -pipeline $_ -pipelineNameResourceDict $ppDict -visited $visited -sortedList $stack } $sortedList = new-object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSPipeline] while ($stack.Count -gt 0) { $sortedList.Add($stack.Pop()) } $sortedList } function triggerSortUtil { param([Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger]$trigger, [Hashtable] $triggerNameResourceDict, [Hashtable] $visited, [System.Collections.Stack] $sortedList) if ($visited[$trigger.Name] -eq $true) { return; } $visited[$trigger.Name] = $true; if ($trigger.Properties.DependsOn) { $trigger.Properties.DependsOn | Where-Object {$_ -and $_.ReferenceTrigger} | ForEach-Object{ triggerSortUtil -trigger $triggerNameResourceDict[$_.ReferenceTrigger.ReferenceName] -triggerNameResourceDict $triggerNameResourceDict -visited $visited -sortedList $sortedList } } $sortedList.Push($trigger) } function Get-SortedTriggers { param( [string] $DataFactoryName, [string] $ResourceGroupName ) $triggers = Get-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName $triggerDict = @{} $visited = @{} $stack = new-object System.Collections.Stack $triggers | ForEach-Object{ $triggerDict[$_.Name] = $_ } $triggers | ForEach-Object{ triggerSortUtil -trigger $_ -triggerNameResourceDict $triggerDict -visited $visited -sortedList $stack } $sortedList = new-object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSTrigger] while ($stack.Count -gt 0) { $sortedList.Add($stack.Pop()) } $sortedList } function Get-SortedLinkedServices { param( [string] $DataFactoryName, [string] $ResourceGroupName ) $linkedServices = Get-AzDataFactoryV2LinkedService -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName $LinkedServiceHasDependencies = @('HDInsightLinkedService', 'HDInsightOnDemandLinkedService', 'AzureBatchLinkedService') $Akv = 'AzureKeyVaultLinkedService' $HighOrderList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService] $RegularList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService] $AkvList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService] $linkedServices | ForEach-Object { if ($_.Properties.GetType().Name -in $LinkedServiceHasDependencies) { $HighOrderList.Add($_) } elseif ($_.Properties.GetType().Name -eq $Akv) { $AkvList.Add($_) } else { $RegularList.Add($_) } } $SortedList = New-Object Collections.Generic.List[Microsoft.Azure.Commands.DataFactoryV2.Models.PSLinkedService]($HighOrderList.Count + $RegularList.Count + $AkvList.Count) $SortedList.AddRange($HighOrderList) $SortedList.AddRange($RegularList) $SortedList.AddRange($AkvList) $SortedList } $templateJson = Get-Content $armTemplate | ConvertFrom-Json $resources = $templateJson.resources #Triggers Write-Host "Getting triggers" $triggersInTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/triggers" } $triggerNamesInTemplate = $triggersInTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40)} #Do not stop/start triggers not included in selective ARM template (if provided) $triggersUpdatedInTemplate = $triggersInTemplate $triggerNamesUpdatedInTemplate = $triggerNamesInTemplate if ($selectiveArmTemplate) { Write-Host "Using selective template for stopping/starting triggers" $selectiveTemplateJson = Get-Content $selectiveArmTemplate | ConvertFrom-Json $selectiveResources = $selectiveTemplateJson.resources $triggersUpdatedInTemplate = $selectiveResources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/triggers" } $triggerNamesUpdatedInTemplate = $triggersUpdatedInTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40)} } $triggersDeployed = Get-SortedTriggers -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $triggersToStop = $triggersDeployed | Where-Object { $triggerNamesUpdatedInTemplate -contains $_.Name } | ForEach-Object { New-Object PSObject -Property @{ Name = $_.Name TriggerType = $_.Properties.GetType().Name } } $triggersToDelete = $triggersDeployed | Where-Object { $triggerNamesInTemplate -notcontains $_.Name } | ForEach-Object { New-Object PSObject -Property @{ Name = $_.Name TriggerType = $_.Properties.GetType().Name } } $triggersToStart = $triggersUpdatedInTemplate | Where-Object { $_.properties.runtimeState -eq "Started" -and ($_.properties.pipelines.Count -gt 0 -or $_.properties.pipeline.pipelineReference -ne $null)} | ForEach-Object { New-Object PSObject -Property @{ Name = $_.name.Substring(37, $_.name.Length-40) TriggerType = $_.Properties.type } } if ($predeployment -eq $true) { #Stop all triggers Write-Host "Stopping deployed triggers`n" $triggersToStop | ForEach-Object { if ($_.TriggerType -eq "BlobEventsTrigger" -or $_.TriggerType -eq "CustomEventsTrigger") { Write-Host "Unsubscribing" $_.Name "from events" $status = Remove-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name while ($status.Status -ne "Disabled"){ Start-Sleep -s 15 $status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name } } Write-Host "Stopping trigger" $_.Name Stop-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force } } else { if ($DeleteNotInSource -eq $true) { #Deleted resources #pipelines Write-Host "Getting pipelines" $pipelinesADF = Get-SortedPipelines -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $pipelinesTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/pipelines" } $pipelinesNames = $pipelinesTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40)} $deletedpipelines = $pipelinesADF | Where-Object { $pipelinesNames -notcontains $_.Name } #dataflows $dataflowsADF = Get-AzDataFactoryV2DataFlow -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $dataflowsTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/dataflows" } $dataflowsNames = $dataflowsTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40) } $deleteddataflow = $dataflowsADF | Where-Object { $dataflowsNames -notcontains $_.Name } #datasets Write-Host "Getting datasets" $datasetsADF = Get-AzDataFactoryV2Dataset -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $datasetsTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/datasets" } $datasetsNames = $datasetsTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40) } $deleteddataset = $datasetsADF | Where-Object { $datasetsNames -notcontains $_.Name } #linkedservices Write-Host "Getting linked services" $linkedservicesADF = Get-SortedLinkedServices -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $linkedservicesTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/linkedservices" } $linkedservicesNames = $linkedservicesTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40)} $deletedlinkedservices = $linkedservicesADF | Where-Object { $linkedservicesNames -notcontains $_.Name } #Integrationruntimes Write-Host "Getting integration runtimes" $integrationruntimesADF = Get-AzDataFactoryV2IntegrationRuntime -DataFactoryName $DataFactoryName -ResourceGroupName $ResourceGroupName $integrationruntimesTemplate = $resources | Where-Object { $_.type -eq "Microsoft.DataFactory/factories/integrationruntimes" } $integrationruntimesNames = $integrationruntimesTemplate | ForEach-Object {$_.name.Substring(37, $_.name.Length-40)} $deletedintegrationruntimes = $integrationruntimesADF | Where-Object { $integrationruntimesNames -notcontains $_.Name } #Delete resources Write-Host "Deleting triggers" $triggersToDelete | ForEach-Object { Write-Host "Deleting trigger " $_.Name $trig = Get-AzDataFactoryV2Trigger -name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName if ($trig.RuntimeState -eq "Started") { if ($_.TriggerType -eq "BlobEventsTrigger" -or $_.TriggerType -eq "CustomEventsTrigger") { Write-Host "Unsubscribing trigger" $_.Name "from events" $status = Remove-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name while ($status.Status -ne "Disabled"){ Start-Sleep -s 15 $status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name } } Stop-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force } Remove-AzDataFactoryV2Trigger -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force } Write-Host "Deleting pipelines" $deletedpipelines | ForEach-Object { Write-Host "Deleting pipeline " $_.Name Remove-AzDataFactoryV2Pipeline -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force } Write-Host "Deleting dataflows" $deleteddataflow | ForEach-Object { Write-Host "Deleting dataflow " $_.Name Remove-AzDataFactoryV2DataFlow -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force } Write-Host "Deleting datasets" $deleteddataset | ForEach-Object { Write-Host "Deleting dataset " $_.Name Remove-AzDataFactoryV2Dataset -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force } Write-Host "Deleting linked services" $deletedlinkedservices | ForEach-Object { Write-Host "Deleting Linked Service " $_.Name Remove-AzDataFactoryV2LinkedService -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force } Write-Host "Deleting integration runtimes" $deletedintegrationruntimes | ForEach-Object { Write-Host "Deleting integration runtime " $_.Name Remove-AzDataFactoryV2IntegrationRuntime -Name $_.Name -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Force } } if ($deleteDeployment -eq $true) { Write-Host "Deleting ARM deployment ... under resource group: " $ResourceGroupName $deployments = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName $deploymentsToConsider = $deployments | Where { $_.DeploymentName -like "ArmTemplate_master*" -or $_.DeploymentName -like "ArmTemplateForFactory*" } | Sort-Object -Property Timestamp -Descending $deploymentName = $deploymentsToConsider[0].DeploymentName Write-Host "Deployment to be deleted: " $deploymentName $deploymentOperations = Get-AzResourceGroupDeploymentOperation -DeploymentName $deploymentName -ResourceGroupName $ResourceGroupName $deploymentsToDelete = $deploymentOperations | Where { $_.properties.targetResource.id -like "*Microsoft.Resources/deployments*" } $deploymentsToDelete | ForEach-Object { Write-host "Deleting inner deployment: " $_.properties.targetResource.id Remove-AzResourceGroupDeployment -Id $_.properties.targetResource.id } Write-Host "Deleting deployment: " $deploymentName Remove-AzResourceGroupDeployment -ResourceGroupName $ResourceGroupName -Name $deploymentName } #Start active triggers - after cleanup efforts if ($StartActiveTriggers) { Write-Host "Starting active triggers" $triggersToStart | ForEach-Object { if ($_.TriggerType -eq "BlobEventsTrigger" -or $_.TriggerType -eq "CustomEventsTrigger") { Write-Host "Subscribing" $_.Name "to events" $status = Add-AzDataFactoryV2TriggerSubscription -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name while ($status.Status -ne "Enabled"){ Start-Sleep -s 15 $status = Get-AzDataFactoryV2TriggerSubscriptionStatus -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name } } Write-Host "Starting trigger" $_.Name Start-AzDataFactoryV2Trigger -ResourceGroupName $ResourceGroupName -DataFactoryName $DataFactoryName -Name $_.Name -Force } } } } |