Functions/Publish-DatabricksClusterLibrariesToWorkspaceByName.ps1
|
<#
.SYNOPSIS Deploys DataBricks Cluster from configuration json file to a workspace .DESCRIPTION Deploys DataBricks Cluster from configuration json file to a workspace .PARAMETER config Configuration json file from the environment used to workout whether to deploy a clusters from a folder or file(s) .PARAMETER bearerToken Your Databricks Bearer token to authenticate to your workspace (see User Settings in Datatbricks WebUI) .PARAMETER clusterConfig The name path of the clusters configuration files. .EXAMPLE Publish-DatabricksClusterLibrariesToWorkspaceByName -config $config -bearerToken 'dapi1234567890' -clusterConfig '<path-to-file>' .NOTES Author: Sabin IO #> Function Publish-DatabricksClusterLibrariesToWorkspaceByName { [cmdletbinding()] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', 'global:DatabricksURI')] Param( [parameter(Mandatory = $true)]$config, [parameter(Mandatory = $true)][string]$bearerToken, [parameter(Mandatory = $true)][string]$librariesConfig ) try { $libraries = Get-Content -Raw -Path $librariesConfig | ConvertFrom-Json Connect-Databricks -BearerToken $bearerToken -Region $config.region $ClusterId = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).cluster_id $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state $libs = Get-DatabricksLibraries -BearerToken $bearerToken -Region $config.region -ClusterId $ClusterId $diffs = [PSCustomObject]@{ pypi_in_source_not_in_target = @{} pypi_in_target_not_in_source = @{} cran_in_source_not_in_target = @{} cran_in_target_not_in_source = @{} egg_in_source_not_in_target = @{} egg_in_target_not_in_source = @{} whl_in_source_not_in_target = @{} whl_in_target_not_in_source = @{} maven_in_source_not_in_target = @{} maven_in_target_not_in_source = @{} } <# Build Source Lib Objects to Compare #> $sourceLibsPypiObjects = $libraries.libraries | ForEach-Object { if ($_.pypi) { [PSCustomObject]@{ package = $_.pypi.package repo = $_.pypi.repo } } } $sourceLibsCranObjects = $libraries.libraries | ForEach-Object { if ($_.cran) { [PSCustomObject]@{ package = $_.cran.package repo = $_.cran.repo } } } $sourceLibsEggObjects = $libraries.libraries | ForEach-Object { if ($_.egg) { [PSCustomObject]@{ egg = $_.egg } } } $sourceLibsWhlObjects = $libraries.libraries | ForEach-Object { if ($_.whl) { [PSCustomObject]@{ whl = $_.whl } } } $sourceLibsMavenObjects = $libraries.libraries | ForEach-Object { if ($_.maven) { [PSCustomObject]@{ coordinates = $_.maven.coordinates repo = $_.maven.repo exclusions = $_.maven.exclusions } } } <# Build Target Lib Objects to Compare #> $targetLibsPypiObjects = $libs | ForEach-Object { if ($_.library.pypi) { [PSCustomObject]@{ package = $_.library.pypi.package repo = $_.library.pypi.repo } } } $targetLibsCranObjects = $libs | ForEach-Object { if ($_.library.cran) { [PSCustomObject]@{ package = $_.library.pypi.package repo = $_.library.pypi.repo } } } $targetLibsEggObjects = $libs | ForEach-Object { if ($_.library.egg) { [PSCustomObject]@{ egg = $_.library.egg } } } $targetLibsWhlObjects = $libs | ForEach-Object { if ($_.library.whl) { [PSCustomObject]@{ whl = $_.library.whl } } } $targetLibsMavenObjects = $libs | ForEach-Object { if ($_.library.maven) { [PSCustomObject]@{ coordinates = $_.library.maven.coordinates repo = $_.library.maven.repo exclusions = $_.library.maven.exclusions } } } <# Get Libs that do not exist on either side #> $pypiLibsInSourceNotInTarget = $sourceLibsPypiObjects | Where-Object { $_.package -notin $targetLibsPypiObjects.package } $pypiLibsInTargetNotInSource = $targetLibsPypiObjects | Where-Object { $_.package -notin $sourceLibsPypiObjects.package } $cranLibsInSourceNotInTarget = $sourceLibsCranObjects | Where-Object { $_.package -notin $targetLibsCranObjects.package } $cranLibsInTargetNotInSource = $targetLibsCranObjects | Where-Object { $_.package -notin $sourceLibsCranObjects.package } $eggLibsInSourceNotInTarget = $sourceLibsEggObjects | Where-Object { $_.egg -notin $targetLibsEggObjects.egg } $eggLibsInTargetNotInSource = $targetLibsEggObjects | Where-Object { $_.egg -notin $sourceLibsEggObjects.egg } $whlLibsInSourceNotInTarget = $sourceLibsWhlObjects | Where-Object { $_.whl -notin $targetLibsWhlObjects.whl } $whlLibsInTargetNotInSource = $targetLibsWhlObjects | Where-Object { $_.whl -notin $sourceLibsWhlObjects.whl } $mavenLibsInSourceNotInTarget = $sourceLibsMavenObjects | Where-Object { $_.coordinates -notin $targetLibsMavenObjects.coordinates } $mavenLibsInTargetNotInSource = $targetLibsMavenObjects | Where-Object { $_.coordinates -notin $sourceLibsMavenObjects.coordinates } [bool]$isDifferences = $false if ($pypiLibsInSourceNotInTarget) { $diffs.pypi_in_source_not_in_target = $pypiLibsInSourceNotInTarget # $diffCounts.pypi_in_source_not_in_target = 1 Write-Output "PyPi Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Output $pypiLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($pypiLibsInTargetNotInSource) { $diffs.pypi_in_target_not_in_source = $pypiLibsInTargetNotInSource # $diffCounts.pypi_in_target_not_in_source = 1 Write-Output "PyPi Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Output $pypiLibsInTargetNotInSource [bool]$isDifferences = $true } if ($cranLibsInSourceNotInTarget) { $diffs.cran_in_source_not_in_target = $cranLibsInSourceNotInTarget Write-Output "Cran Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Output $cranLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($cranLibsInTargetNotInSource) { $diffs.cran_in_target_not_in_source = $cranLibsInTargetNotInSource Write-Output "Cran Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Output $cranLibsInTargetNotInSource [bool]$isDifferences = $true } if ($eggLibsInSourceNotInTarget) { $diffs.egg_in_source_not_in_target = $eggLibsInSourceNotInTarget Write-Output "Egg Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Output $eggLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($eggLibsInTargetNotInSource) { $diffs.egg_in_target_not_in_source = $eggLibsInTargetNotInSource Write-Output "Egg Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Output $eggLibsInTargetNotInSource Write-Output "`$isDifferences is not being set to true as Prod still has egg files set to install across all clusters (Not Recommended)" # [bool]$isDifferences = $true } if ($whlLibsInSourceNotInTarget) { $diffs.whl_in_source_not_in_target = $whlLibsInSourceNotInTarget Write-Output "Whl Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Output $whlLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($whlLibsInTargetNotInSource) { $diffs.whl_in_target_not_in_source = $whlLibsInTargetNotInSource Write-Output "Whl Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Output $whlLibsInTargetNotInSource [bool]$isDifferences = $true } if ($mavenLibsInSourceNotInTarget) { $diffs.maven_in_source_not_in_target = $mavenLibsInSourceNotInTarget Write-Output "Maven Library/Libraries in Source not in Source `"$($libraries.cluster_name)`"" Write-Output $mavenLibsInSourceNotInTarget [bool]$isDifferences = $true } if ($mavenLibsInTargetNotInSource) { $diffs.maven_in_target_not_in_source = $mavenLibsInTargetNotInSource Write-Output "Maven Library/Libraries in Target not in Source `"$($libraries.cluster_name)`"" Write-Output $mavenLibsInTargetNotInSource [bool]$isDifferences = $true } Write-Output $diffs | Format-List Write-Output "Value of `$isDifferences is $($isDifferences) for `"$($libraries.cluster_name)`"" if ($isDifferences -eq $true) { if ($ClusterState -eq 'TERMINATED') { Write-Output "Cluster State for `"$($libraries.cluster_name)`" is TERMINATED, starting cluster and waiting 30 seconds..." Start-DatabricksCluster -ClusterId $ClusterId -BearerToken $bearerToken Start-Sleep -Seconds 30 } while ( $ClusterState -ne 'RUNNING') { Write-Output "Cluster State for `"$($libraries.cluster_name)`" is not RUNNING, waiting 30 seconds..." Start-Sleep -Seconds 30 $ClusterState = (Get-DatabricksClusters | Where-Object { $_.cluster_name -eq $libraries.cluster_name }).state } $libraries.PSObject.Properties.Remove('cluster_name') $libraries | Add-Member -NotePropertyName 'cluster_id' -NotePropertyValue $ClusterId [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 $Headers = Get-Headers $PSBoundParameters # TODO: Replace API call with call to azure.databricks.cicd.tools $uri = "$global:DatabricksURI/api/2.0/libraries/install" $BodyText = $libraries | ConvertTo-Json -Depth 100 Write-Output "Request Body: $BodyText" Write-Output "Installing libraries to REST API: $uri" $Response = Invoke-RestMethod -Uri $uri -Body $BodyText -Method 'POST' -Headers $Headers return $Response } } catch { #uh oh throw $_.Exception } } |