utils/AmbariUtils.ps1
|
<#
.Synopsis Utility function to fetch cluster's hadoop configs based on the cluster type .DESCRIPTION This function takes the cluster DNS name and type and fetches the relevant Hadoop configs, For now the configs to fetch are the ones that can be modified directly in the ARM template, as below clusterIdentity.xml, core-site.xml, gateway.xml, hbase-env.xml, hbase-site.xml, hdfs-site.xml hive-env.xml, hive-site.xml, mapred-site, oozie-site.xml, oozie-env.xml, storm-site.xml, tez-site.xml webhcat-site.xml, yarn-site.xml, spark2-defaults, etc... .EXAMPLE ## To be added. .LINK https://github.com/mabushaireh/HDICloner #> #TODO: Get the last version by the user and the last version after Cluster creation, Compare two versions and show the changes only. function Get-AmbariUtilsConfig { # Read cluster DNS name and required HDP configs from Main # Fetch the relevant configs [CmdletBinding()] param ( [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Cluster Dns Name")] [string] $ClustDnsName, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Ambari Username")] [string] $Username, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Ambari Password")] [string] $Password ) Show-Debug "Value set for ClustDnsName is $ClustDnsName" Show-Debug "Value set for Username is $Username" Show-Debug "Value set for Password is $Password" Show-Info "Attempting to fetch relevant configs for cluster name $ClustDnsName" #Use the below to avoid manual prompt for password when calling Ambari API. $SecPasswd = ConvertTo-SecureString $Password -AsPlainText -Force $Credentials = New-Object System.Management.Automation.PSCredential($Username, $SecPasswd) ## Subject to change, first fetch the common configs, then get the ones relevant to cluster type #TODO: Try to get the differences in the files. # Find a Way to use $credential param containing user/password to avoid extra prompt for password. $HTTPSUri = "https://$ClustDnsName.azurehdinsight.net/api/v1/clusters/$ClustDnsName/configurations" $request = Invoke-WebRequest -Uri $HTTPSUri -Credential $Credentials -UseBasicParsing if ($request.StatusCode -ne 200) { Show-Error ("Unable to retrive Ambari Configurations: " + $request.StatusDescription) return } $HDPConfigVersions = $request.Content Show-Debug $HDPConfigVersions $versions = ConvertFrom-Json $HDPConfigVersions -AsHashtable $lastVersion = ($versions["items"][$versions["items"].Count-1]).tag $configs = @{} foreach ($version in $versions["items"]){ if ($version.tag -eq $lastVersion){ Show-Debug ("Get Configuration type [" + $version.type + "] for latest version [" + $version.tag + "]") $HTTPSUri = "https://$ClustDnsName.azurehdinsight.net/api/v1/clusters/$ClustDnsName/configurations?tag=$lastVersion&type=" + $version.type $request = Invoke-WebRequest -Uri $HTTPSUri -Credential $Credentials -UseBasicParsing if ($request.StatusCode -ne 200) { Show-Error ("Unable to retrive Ambari Configurations: " + $request.StatusDescription) return } $result = $request.Content $configs.Add($version.type, $result) } } return $configs } #TODO: OPTION 1: Function to return the Service Requested for restart. # OPTION 2: param RestartIfRequired: boolean, If true function should restart the required service automatically and return scuess if completed. function Set-AmbariUtilsConfig { [CmdletBinding()] param ( [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Cluster Dns Name")] [string] $ClustDnsName, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Ambari Username")] [string] $Username, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Ambari Password")] [string] $Password, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Config Type")] [string] $Type, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Config Tag")] [string] $Tag, [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Config Tag")] [string] $Properties ) Show-Debug "Value set for ClustDnsName is $ClustDnsName" Show-Debug "Value set for Username is $Username" Show-Debug "Value set for Password is $Password" Show-Debug "Value set for Type is $Type" Show-Debug "Value set for Tag is $Tag" Show-Debug "Value set for Properties is $Properties" Show-Info "Attempting to set relevant configs for cluster name $ClustDnsName" $HTTPSUri = "https://$ClustDnsName.azurehdinsight.net/api/v1/clusters/$ClustDnsName" $headers = @{ "X-Requested-By" = "ambari" } $body = '[{"Clusters":{ "desired_config":[{ "type":"' + $Type +'", "tag":"' + $Tag +'", "properties":' + $Properties +', "service_config_version_note":"' + $Tag +'"}]}}]' Show-Debug "Body after modifications is : $body" $request = Invoke-WebRequest -Uri $HTTPSUri -Credential $Credentials -UseBasicParsing -Method Put -Body $body -Headers $headers if ($request.StatusCode -ne 200) { Show-Error ("Unable to retrive Ambari Configurations: " + $request.StatusDescription) return } } function Parse-AmbariUtilsConfig { [CmdletBinding()] param ( [Parameter(Mandatory = $true, ParameterSetName = "Main", HelpMessage = "Config file content")] [string] $ConfigContent ) $parsed = ConvertFrom-Json $ConfigContent -AsHashtable $properties = ConvertTo-Json $parsed["items"][0]["properties"] -Depth 100 return @{ Type = $parsed["items"][0]["type"] Properties = $properties } } # Restart All Required: #body: #{"RequestInfo":{"command":"RESTART","context":"Restart all required services","operation_level":"host_component"},"Requests/resource_filters":[{"hosts_predicate":"HostRoles/stale_configs: true"}]} #headers: ##TODO: Compile all the config variables into proper JSON suitable to be added to ARM template as below # Might want to do this in Main. #"configurations": { #"hive-site": $HiveSiteConfig #"spark2-defaults": $Spark2Defaults #}/ |