Scripts/Sync-NFSv3ExtendedGroupsFromLDAP.ps1
|
<#
.SYNOPSIS Synchronizes NFSv3 extended groups from LDAP server. .DESCRIPTION This command syncs NFSv3 extended groups with the user and group data fetched from an LDAP server. .PARAMETER ResourceGroupName The name of the Resource Group in Azure where the storage account resides. .PARAMETER StorageAccountName The name of the Azure storage account. .PARAMETER LDAPServerIP The IP address of the LDAP server. .PARAMETER LDAPSearchBase The search base for querying LDAPServer. .PARAMETER UserFilter Optional filter for selecting users in LDAP server. .PARAMETER GroupFilter Optional filter for selecting groups in LDAP server. .PARAMETER UidFilePath Optional file path for specifying UIDs of users which needs to be synced. The file should contain one Uid per line. .PARAMETER LDAPUserName Optional username for authenticating with LDAP server. Example: "cn=admin,dc=mydomain,dc=com" .PARAMETER LDAPCredential Optional PSCredential object for authenticating with LDAP server. .EXAMPLE Sync-NFSv3ExtendedGroupsFromLDAP -ResourceGroupName "MyRG" -StorageAccountName "MyStorage" -LDAPServerIP "192.168.1.1" -LDAPSearchBase "dc=mydomain,dc=com" .NOTES Written by: [Azure Blob NFS] Date: [October 10, 2024] #> function Sync-NFSv3ExtendedGroupsFromLDAP { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string]$ResourceGroupName, [Parameter(Mandatory=$true)] [string]$StorageAccountName, [Parameter(Mandatory=$true)] [string]$LDAPServerIP, [Parameter(Mandatory=$true)] [string]$LDAPSearchBase, [Parameter(Mandatory=$false)] [string]$UserFilter = "(&(objectClass=posixAccount)(uid=*))", [Parameter(Mandatory=$false)] [string]$GroupFilter = "(&(objectClass=posixGroup)(gidNumber=*))", [Parameter(Mandatory=$false)] [string]$UidFilePath, [Parameter(Mandatory=$false, ParameterSetName = 'UserNameSet')] [string]$LDAPUserName, [Parameter(Mandatory=$false, ParameterSetName = 'PsCredSet')] [PSCredential]$LDAPCredential ) # logMessage is written in this way to keep the indentation correct in log file. $logMessage = @" Sync-NFSv3ExtendedGroupsFromLDAP started, ResourceGroupName : $ResourceGroupName, StorageAccountName : $StorageAccountName, LDAPServerIP : $LDAPServerIP, LDAPSearchBase : $LDAPSearchBase, UserFilter : $UserFilter, GroupFilter : $GroupFilter "@ if ($UidFilePath) { $logMessage += ",`n UidFilePath : $UidFilePath" } if ($LDAPUserName) { $logMessage += ",`n LDAPUserName : $LDAPUserName" } if ($LDAPCredential) { $logMessage += ",`n LDAPUserName from PSCredential: $($LDAPCredential.UserName)" } Write-NFSv3ExtendedGroupsLog $logMessage if ($null -eq $LDAPCredential) { # Get admin username and password of LDAP server. if ($null -eq $LDAPUserName) { $LDAPUserName = Read-Host "Enter User Name for LDAP Server Administrator" } $ldapPassword = Read-Host "Password" -AsSecureString # Create a PSCredential object. $LDAPCredential = New-Object System.Management.Automation.PSCredential($LDAPUserName, $ldapPassword) } # Connect to LDAP server. Connect-LdapServer -Server $LDAPServerIP -Credential $LDAPCredential #-Verbose # Create and send LDAP search request for fetching users. $userSearchRequest = New-Object DirectoryServices.Protocols.SearchRequest( $LDAPSearchBase, $UserFilter, [System.DirectoryServices.Protocols.SearchScope]::Subtree) $userSearchResult = $LdapConnection.SendRequest($userSearchRequest) # Create and send LDAP search request for fetching groups. $groupSearchRequest = New-Object DirectoryServices.Protocols.SearchRequest( $LDAPSearchBase, $groupSearchFilter, [System.DirectoryServices.Protocols.SearchScope]::Subtree) $groupSearchResult = $LdapConnection.SendRequest($groupSearchRequest) # # Create uidToUidNumberMap and uidToGidNumberMap which maps uid to corresponding uidNumber and gidNumber. # # For ldapUser1 with uidNumber 1000 and gidNumber 1001, # uidToUidNumberMap[ldapUser1] = 1000 # uidToGidNumberMap[ldapUser1] = 1001 # $uidToUidNumberMap = @{} $uidToGidNumberMap = @{} $ldapUsers = @() foreach ($entry in $userSearchResult.Entries) { if (($null -eq $entry.Attributes["uidNumber"]) -or ($null -eq $entry.Attributes["uid"])) { continue } $uidNumber = $entry.Attributes["uidNumber"].GetValues([string])[0] $uid = $entry.Attributes["uid"].GetValues([string])[0] $uidToUidNumberMap[$uid] = $uidNumber $ldapUsers += $uidNumber if (-not ($null -eq $entry.Attributes["gidNumber"])) { $gidNumber = $entry.Attributes["gidNumber"].GetValues([string])[0] $uidToGidNumberMap[$uid] = $gidNumber } } # Create uidNumberToExtendedGroupsMap which maps uidNumber to array of extended group ids. $uidNumberToExtendedGroupsMap = @{} foreach ($entry in $groupSearchResult.Entries) { if (($null -eq $entry.Attributes["memberuid"]) -or ($null -eq $entry.Attributes["gidNumber"])) { continue } # Get gidNumber and all the memberUids which are part of this group. $gidNumber = $entry.Attributes["gidnumber"].GetValues([string])[0] $memberUidArray = $entry.Attributes["memberuid"].GetValues([string]) foreach ($memberUid in $memberUidArray) { if ($uidToUidNumberMap.ContainsKey($memberUid)) { $uidNumber = $uidToUidNumberMap[$memberUid] if (-not $uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)) { $uidNumberToExtendedGroupsMap[$uidNumber] = @() } $uidNumberToExtendedGroupsMap[$uidNumber] += $gidNumber } } } # Get all the existing LocalUsers for this account. $localUsers = Get-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -IncludeNFSv3 # Initialize hashtable to store LocalUser object with name as key. $localUserMap = @{} # Populate localUserMap hashtable. foreach ( $user in $localUsers ) { $localUserMap[$user.Name] = $user } # Get the uids of localusers which needs to be updated from provided file. [int[]]$uidsFromFile = @() if (-not [string]::IsNullOrEmpty($UidFilePath)) { $uidsFromFile = Get-Content -Path $UidFilePath | ForEach-Object { try { [int]$_ } catch { Write-NFSv3ExtendedGroupsLog -Message "Invalid Uid entry [$_] in $UidFilePath" -LogLevel "ERROR" } } | Where-Object { $_ -is [int] } # Ensure only integers are kept. } $logScript = "$PSScriptRoot\Write-NFSv3ExtendedGroupsLog.ps1" $jobList = [System.Collections.Generic.List[System.Management.Automation.Job]]::new() # # There is a limit of 10 Write Ops per second by using ARM with Azure Storage. To avoid requests getting # throttled because of this limit, keep maxParallelJobs less than or equal to 10. # # https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#storage-throttling # $maxParallelJobs = 10 $ldapUsersSize = $ldapUsers.Length $chunkUsersSize = [Math]::Ceiling($ldapUsersSize / $maxParallelJobs) $numOfIterations = [Math]::Min($maxParallelJobs, $ldapUsersSize) for ( $i = 0; $i -lt $numOfIterations; $i++ ) { # Slice the ldapUsers array into equal size of chunks each of which will be handled by separate processes. $startIndex = $i * $chunkUsersSize $endIndex = [Math]::Min($startIndex + $chunkUsersSize - 1, $ldapUsersSize - 1) $chunkUsers = $ldapUsers[$startIndex..$endIndex] # Start a new background job for each chunk of users. $job = Start-Job -ScriptBlock { param ( $chunkUsers, $uidNumberToExtendedGroupsMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $uidsFromFile ) # Dot-Source internal logging function cause this background job will create different runspace. $Global:NFSv3ExtendedGroupsLogFile = $using:NFSv3ExtendedGroupsLogFile . $logScript # # Convert uidsFromFile to HashMap for O(1) search. Can not pass complex HashMap objects to background # jobs, therefore it was initially stored in an array. # $uidsFromFileSet = [System.Collections.Generic.HashSet[int]]::new() foreach ( $uid in $uidsFromFile ) { $uidsFromFileSet.Add($uid) | Out-Null } foreach ($uidNumber in $chunkUsers) { # # If user has provided file to tell which Uids to upload, # Verify this Uid is present in that list and skip if not. # if ( $uidsFromFileSet.Count -gt 0 ) { if ( -not $uidsFromFileSet.Contains($uidNumber) ) { # If this uid is not part of list provided by user, skip. Write-NFSv3ExtendedGroupsLog "$uidNumber is not present in the file, skipping" | Out-Null continue } } # Get extended group gids for this uid. $gidArray = @() if ($uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)) { $gidArray = $uidNumberToExtendedGroupsMap[$uidNumber] } # Convert ArrayList to Int32 array. [int32[]]$extendedGroupsArray = [int32[]]$gidArray | Sort-Object # NFSv3 localuser have name in "nfsv3_<uid>" format. $localUserName = "nfsv3_$($uidNumber)" # Check if local user with same uidNumber exist in localuser database. if ( $localUserMap.ContainsKey($localUserName) ) { # Sort extended group array before comparing. $existingLocalUser = $localUserMap[$localUserName] $existingExtendedGroups = $existingLocalUser.ExtendedGroups | Sort-Object $difference = Compare-Object -ReferenceObject $existingExtendedGroups -DifferenceObject $extendedGroupsArray # Don't PUT local user if database is already updated. if ($difference.Count -eq 0) { Write-NFSv3ExtendedGroupsLog "ExtendedGroups already up to date for Uid $uidNumber" | Out-Null continue } } # Create localuser. Set-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -UserName $localUserName -IsNfSv3Enabled $true -ExtendedGroups $extendedGroupsArray } } -ArgumentList $chunkUsers, $uidNumberToExtendedGroupsMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $uidsFromFile $jobList.Add($job) } # Wait for all jobs to complete. $jobList | ForEach-Object { $_ | Wait-Job } | Out-Null # Clean up completed jobs $jobList | ForEach-Object { $_ | Remove-Job } Write-NFSv3ExtendedGroupsLog "Sync-NFSv3ExtendedGroupsFromLDAP completed" } |