Functions/Publish-DatabricksJobToWorkspaceByName.ps1
|
<#
.SYNOPSIS Deploys DataBricks job from configuration json file to a workspace .DESCRIPTION Deploys DataBricks job from configuration json file to a workspace .PARAMETER config Configuration json file from the environment used to workout whether to deploy a job(s) from a folder or file(s) .PARAMETER bearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER clusterConfig The name path of the job configuration files. .EXAMPLE Publish-DatabricksJobToWorkspaceByName -config $config -bearerToken 'dapi1234567890' -jobConfig '<path-to-file>' .NOTES Author: Sabin IO #> Function Publish-DatabricksJobToWorkspaceByName { [cmdletbinding()] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', 'global:DatabricksURI')] Param( [parameter(Mandatory = $true)]$config, [parameter(Mandatory = $true)][string]$bearerToken, [parameter(Mandatory = $true)][string]$jobConfig, [parameter(Mandatory = $true)][bool]$removeSchedule ) try { $job = Get-Content -Raw -Path $jobConfig | ConvertFrom-Json Connect-Databricks -BearerToken $bearerToken -Region $config.region <# If key existing_cluster_name is present we need to replace it with existing_cluster_id for the workspace being deployed to #> if ($job.PSObject.Properties.Item('existing_cluster_name')) { $ClusterId = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $job.existing_cluster_name }).cluster_id if ($ClusterId) { $job.PSObject.Properties.Remove('existing_cluster_name') $job | Add-Member -NotePropertyName 'existing_cluster_id' -NotePropertyValue $ClusterId } else { throw "No cluster name of `"$($job.existing_cluster_name)`" found. Please check the cluster exists on the target workspace before trying again" } } # "existing_cluster_name": "KYTE_AUTO (do not use)", # Connect-Databricks -BearerToken $bearerToken -Region $config.region $ExistingJobConfig = Get-DatabricksJobs -BearerToken $bearerToken -Region $config.Region | Where-Object { $_.settings.name -eq $job.name } # $ClusterId = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).cluster_id # $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state if ($ExistingJobConfig) { Write-Output "Job found with this name `"$($job.name)`" - resetting job..." $reset = [PSCustomObject]@{ job_id = $ExistingJobConfig.job_id new_settings = @{} } [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $Headers = Get-Headers $PSBoundParameters # TODO: Replace API call with call to azure.databricks.cicd.tools $uri = "$global:DatabricksURI/api/2.0/jobs/reset" <# If we need to remove the schedules for dev/test we do that here before we edit the job. #> if ($removeSchedule -eq $true) { if ($job.PSObject.Properties.Item('schedule')) { $job.PSObject.Properties.Remove('schedule') } } $reset.new_settings = $job $BodyText = $reset | ConvertTo-Json -Depth 100 Write-Output "Request Body: $($BodyText)" Invoke-RestMethod -Uri $uri -Body $BodyText -Method 'POST' -Headers $Headers return $ExistingJobConfig.job_id } else { Write-Output "No job found with this name `"$($job.name)`" - creating new job..." [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $Headers = Get-Headers $PSBoundParameters # TODO: Replace API call with call to azure.databricks.cicd.tools $uri = "$global:DatabricksURI/api/2.0/jobs/create" <# If we need to remove the schedules for dev/test we do that here before we create the job. #> if ($removeSchedule -eq $true) { if ($job.PSObject.Properties.Item('schedule')) { $job.PSObject.Properties.Remove('schedule') } } $BodyText = $job | ConvertTo-Json -Depth 100 Write-Output "Request Body: $($BodyText)" $response = Invoke-RestMethod -Uri $uri -Body $BodyText -Method 'POST' -Headers $Headers return $response.job_id } } catch { #uh oh throw $_.Exception } } |