Functions/Publish-DatabricksClusterConfigToWorkspace.ps1
|
<#
.SYNOPSIS Deploys DataBricks Cluster from configuration json file to a workspace .DESCRIPTION Deploys DataBricks Cluster from configuration json file to a workspace .PARAMETER config Configuration json file from the environment used to workout whether to deploy a clusters from a folder or file(s) .PARAMETER bearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER clusterConfig The name path of the clusters configuration files. .EXAMPLE Publish-DatabricksClusterConfigToWorkspace -config $config -bearerToken 'dapi1234567890' -clusterConfig '<path-to-file>' .NOTES Author: Sabin IO #> Function Publish-DatabricksClusterConfigToWorkspace { [cmdletbinding()] Param( [parameter(Mandatory = $true)]$config, [parameter(Mandatory = $true)][string]$bearerToken, [parameter(Mandatory = $true)][string]$clusterConfig, $databricksURI ) try { $cluster = Get-Content -Raw -Path $clusterConfig | ConvertFrom-Json Connect-Databricks -BearerToken $bearerToken -Region $config.region $ExistingClusterConfig = Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $cluster.cluster_name } $ClusterId = $ExistingClusterConfig.cluster_id $ExistingClusterConfig | Remove-ClusterMetaDataAsPSObject # $clusterAsJson = $tmp | ConvertTo-OrderedHashtableFromPSCustomObject | ConvertTo-Json -Depth 100 if ($ExistingClusterConfig) { # This a complete hack but seems to be the only way we've got it to work! # 1. Convert the objects to an ordered hash table # 2. Convert them back to PSObjects # 3. Compare the PSObject Properties for differences! $sourceConfigOrderedHashtable = $cluster | ConvertTo-OrderedHashtableFromPSCustomObject $targetConfigOrderedHashtable = $ExistingClusterConfig | ConvertTo-OrderedHashtableFromPSCustomObject $sourceConfigOrderedObject = New-Object PSObject -Property $sourceConfigOrderedHashtable $targetConfigOrderedObject = New-Object PSObject -Property $targetConfigOrderedHashtable Compare-Object -ReferenceObject $sourceConfigOrderedObject.PSObject.Properties ` -DifferenceObject $targetConfigOrderedObject.PSObject.Properties ` -OutVariable Diffs if ($Diffs.Count -gt 0) { Write-Output "Cluster `"$($cluster.cluster_name)`" exists - updating cluster" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $Headers = GetHeaders $PSBoundParameters $uri = "$databricksURI/api/2.0/clusters/edit" # We need to add the cluster_id back into the config as this is what tell the API what cluster to edit $cluster | Add-Member -NotePropertyName 'cluster_id' -NotePropertyValue $ClusterId $BodyText = $cluster | ConvertTo-Json -Depth 100 Write-Output "Request Body: $($BodyText)" $response = Invoke-RestMethod -Uri $uri -Body $BodyText -Method 'POST' -Headers $Headers return $response.cluster_id } else { Write-Warning "Cluster `"$($cluster.cluster_name)`" unchanged - not deploying to prevent unnecessary restart of cluster" } return $ClusterId } else { Write-Output "No cluster found with this name `"$($cluster.cluster_name)`" - creating new cluster" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $Headers = GetHeaders $PSBoundParameters $uri = "$databricksURI/api/2.0/clusters/create" $BodyText = $cluster | ConvertTo-Json -Depth 100 Write-Output "Request Body: $($BodyText)" $response = Invoke-RestMethod -Uri $uri -Body $BodyText -Method 'POST' -Headers $Headers return $response.cluster_id } } catch { #uh oh throw $_.Exception } } |