Functions/Export-DatabricksJobConfigAsJson.ps1
|
<#
.SYNOPSIS Exports DataBricks jobs and Saves as json. .DESCRIPTION Exports Databricks jobs and saves as json. .PARAMETER BearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER Region Azure Region - must match the URL of your Databricks workspace, example northeurope .PARAMETER LocalOutputPath Local directroy to save json files. .PARAMETER JobConfig The job config to export as a json file. .EXAMPLE Export-DatabricksJobConfigAsJson -BearerToken $bearerToken -Region $config.Region -LocalOutputPath $localOutputPath -JobConfig $job -Verbose .NOTES Author: Sabin IO #> Function Export-DatabricksJobConfigAsJson { [cmdletbinding()] Param( [parameter(Mandatory = $false)][string]$BearerToken, [parameter(Mandatory = $false)][string]$Region, [parameter(Mandatory = $true)][string]$LocalOutputPath, [parameter(Mandatory = $true)][psobject]$JobConfig ) $invalidChars = [System.IO.Path]::GetInvalidFileNameChars() -join '' $re = "[{0}]" -f [regex]::Escape($invalidChars) $jobName = ($JobConfig.name -replace $re) $jobFileName = $jobName + '.job.config.json' <# We need to replace the existing cluster id with a named cluster for cross workspace #> if (($JobConfig.PSObject.Properties.Item('existing_cluster_id'))) { Write-Output "Job Name $($JobConfig.name) has existing_cluster_id, replacing with existing_cluster_name" $cluster = Get-DatabricksClusters -BearerToken $BearerToken -Region $Region -ClusterId $JobConfig.existing_cluster_id $JobConfig.PSObject.Properties.Remove('existing_cluster_id') $JobConfig | Add-Member -NotePropertyName 'existing_cluster_name' -NotePropertyValue $cluster.cluster_name } <# Shallow sort job config into ordered hashtable for source control purposes #> if (($JobConfig.PSObject.Properties.Item('new_cluster'))) { $new_cluster = $JobConfig.new_cluster | Remove-JobNewClusterMetaDataAsPSObject $JobConfig.PSObject.Properties.Remove('new_cluster') $new_cluster_sorted = $new_cluster | ConvertTo-OrderedHashtableFromPSCustomObject $new_cluster_psobject = [PSCustomObject]$new_cluster_sorted $JobConfig | Add-Member -NotePropertyName 'new_cluster' -NotePropertyValue $new_cluster_psobject } $tmp = $JobConfig | Remove-JobMetaDataAsPSObject $JobConfigAsJson = $tmp | ConvertTo-OrderedHashtableFromPSCustomObject | ConvertTo-Json -Depth 100 $LocalExportPath = Join-Path $LocalOutputPath $jobFileName Write-Output "Exporting job `"$($jobName)`" to `"$($LocalExportPath)`"" New-Item -Force -Path $LocalExportPath -Value $JobConfigAsJson -Type file | Out-Null } |