Scripts/SyncNFSv3ExtendedGroupsFromAD.ps1
|
<#
.SYNOPSIS Synchronizes NFSv3 extended groups from Active Directory (AD). .DESCRIPTION This command syncs NFSv3 extended groups with the user and group data fetched from an AD server. .PARAMETER ResourceGroupName The name of the Resource Group in Azure where the storage account resides. .PARAMETER StorageAccountName The name of the Azure storage account. .PARAMETER ADServerIP The IP address of the AD server. .PARAMETER ADSearchBase The search base for querying Active Directory. .PARAMETER UserFilter Optional filter for selecting users in Active Directory. .PARAMETER GroupFilter Optional filter for selecting groups in Active Directory. .PARAMETER UidFilePath Optional file path for specifying UIDs of users which needs to be synced. The file should contain one UID per line. .PARAMETER ADUserName Optional username for authenticating with Active Directory. .PARAMETER ADCredential Optional PSCredential object for authenticating with Active Directory. .EXAMPLE Sync-NFSv3ExtendedGroupsFromAD -ResourceGroupName "MyRG" -StorageAccountName "MyStorage" -ADServerIP "192.168.1.1" -ADSearchBase "dc=mydomain,dc=com" .NOTES Written by: [Azure Blob NFS] Date: [October 10, 2024] #> function Sync-NFSv3ExtendedGroupsFromAD { [CmdletBinding()] param ( [Parameter(Mandatory=$true)] [string]$ResourceGroupName, [Parameter(Mandatory=$true)] [string]$StorageAccountName, [Parameter(Mandatory=$true)] [string]$ADServerIP, [Parameter(Mandatory=$true)] [string]$ADSearchBase, [Parameter(Mandatory=$false)] [string]$UserFilter = "uidNumber -ge 0", [Parameter(Mandatory=$false)] [string]$GroupFilter = "gidNumber -ge 0", [Parameter(Mandatory=$false)] [string]$UidFilePath, [Parameter(Mandatory=$false, ParameterSetName = 'UserNameSet')] [string]$ADUserName, [Parameter(Mandatory=$false, ParameterSetName = 'PsCredSet')] [PSCredential]$ADCredential ) # logMessage is written in this way to keep the indentation correct in log file. $logMessage = @" Sync-NFSv3ExtendedGroupsFromAD started, ResourceGroupName : $ResourceGroupName, StorageAccountName : $StorageAccountName, ADServerIP : $ADServerIP, ADSearchBase : $ADSearchBase, UserFilter : $UserFilter, GroupFilter : $GroupFilter "@ if ($UidFilePath) { $logMessage += ",`n UidFilePath : $UidFilePath" } if ($ADUserName) { $logMessage += ",`n ADUserName : $ADUserName" } if ($ADCredential) { $logMessage += ",`n ADUserName from PSCredential: $($ADCredential.UserName)" } Write-NFSv3ExtendedGroupsLog $logMessage -VerbosePreference $VerbosePreference if ($null -eq $ADCredential) { # Get admin username and password of AD server. if ($null -eq $ADUserName) { $ADUserName = Read-Host "Enter User Name for Active Directory Administrator [$ADServerIP]:" } $adPassword = Read-Host "Password:" -AsSecureString # Create a PSCredential object $ADCredential = New-Object System.Management.Automation.PSCredential($ADUserName, $adPassword) } # Get all the users from Active Directory. try { $adUsers = Get-ADUser -Credential $ADCredential -Filter $UserFilter -SearchBase $ADSearchBase -Properties uidNumber,gidNumber,memberOf -Server $ADServerIP -ErrorAction Stop Write-NFSv3ExtendedGroupsLog "Get-ADUser successfully fetched $($adUsers.Count) entries" -VerbosePreference $VerbosePreference } catch { # Kill the script if Get-ADUser fails. Write-NFSv3ExtendedGroupsLog "Get-ADUser failed with error: $_" -LogLevel "ERROR" return } # Get all the groups from Active Directory. try { $adGroups = Get-ADGroup -Credential $ADCredential -Filter $GroupFilter -SearchBase $ADSearchBase -Properties gidNumber -Server $ADServerIP -ErrorAction Stop Write-NFSv3ExtendedGroupsLog "Get-ADGroup successfully fetched $($adGroups.Count) entries" -VerbosePreference $VerbosePreference } catch { # Kill the script if Get-ADGroup fails. Write-NFSv3ExtendedGroupsLog "Get-ADGroup failed with error: $_" -LogLevel "ERROR" return } # Initialize hashtable to store gidNumber of AD groups with DistinguishedName as key. $groupDNToGidMap = @{} # # Populate groupDNToGidMap hashtable. # # TODO: Currently $group.gidNumber is making subsequent calls to AD server to fetch gidNumber. # Fix this to improve performance. # foreach ( $group in $adGroups ) { if ( $null -ne $group.gidNumber ) { $groupDNToGidMap[$group.DistinguishedName] = $group.gidNumber[0] } } # Get all the existing LocalUsers for this account. try { $localUsers = Get-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -IncludeNFSv3 -ErrorAction Stop Write-NFSv3ExtendedGroupsLog "Fetched $($localUsers.Count) LocalUsers from Azure Storage server" -VerbosePreference $VerbosePreference } catch { Write-NFSv3ExtendedGroupsLog "Get-AzStorageLocalUser failed with error: $_" -LogLevel "ERROR" } # Initialize hashtable to store LocalUser object with name as key. $localUserMap = @{} # Populate localUserMap hashtable. foreach ( $user in $localUsers ) { $localUserMap[$user.Name] = $user } # Get the uids of localusers which needs to be updated from provided file. [int[]]$uidsFromFile = @() if (-not [string]::IsNullOrEmpty($UidFilePath)) { $uidsFromFile = Get-Content -Path $UidFilePath | ForEach-Object { try { [int]$_ } catch { Write-NFSv3ExtendedGroupsLog -Message "Invalid Uid entry [$_] in $UidFilePath" -LogLevel "ERROR" } } | Where-Object { $_ -is [int] } # Ensure only integers are kept. } $logScript = "$PSScriptRoot\WriteNFSv3ExtendedGroupsLog.ps1" $jobList = [System.Collections.Generic.List[System.Management.Automation.Job]]::new() # # There is a limit of 10 Write Ops per second by using ARM with Azure Storage. To avoid requests getting # throttled because of this limit, keep maxParallelJobs less than or equal to 10. # # https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#storage-throttling # $maxParallelJobs = 10 $usersSize = $adUsers.Length $chunkUsersSize = [Math]::Ceiling($usersSize / $maxParallelJobs) $numOfIterations = [Math]::Ceiling($usersSize / $chunkUsersSize) for ( $i = 0; $i -lt $numOfIterations; $i++ ) { # Slice the adUsers array into equal size of chunks each of which will be handled by separate processes. $startIndex = $i * $chunkUsersSize $endIndex = [Math]::Min($startIndex + $chunkUsersSize - 1, $usersSize - 1) $chunkUsers = $adUsers[$startIndex..$endIndex] # Start a new background job for each chunk of adUsers. $job = Start-Job -ScriptBlock { param ( $chunkUsers, $groupDNToGidMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $uidsFromFile, $VerbosePreference ) # Dot-Source internal logging function and import required modules cause this background job will create different runspace. Import-Module Az.Storage 4>&1 | Out-Null $Global:NFSv3ExtendedGroupsLogFile = $using:NFSv3ExtendedGroupsLogFile . $logScript # # Convert uidsFromFile to HashMap for O(1) search. Can not pass complex HashMap objects to background # jobs, therefore it was initially stored in an array. # $uidsFromFileSet = [System.Collections.Generic.HashSet[int]]::new() foreach ( $uid in $uidsFromFile ) { $uidsFromFileSet.Add($uid) } foreach ($user in $chunkUsers) { # Don't process if uidNumber of user is null. if ( $null -eq $user.uidNumber ) { Write-NFSv3ExtendedGroupsLog "UidNumber is null for $($user.DistinguishedName)" -LogLevel "WARNING" continue } # Get uidNumber, gidNumber and group membership info from $user object. $uidNumber = $user.uidNumber $primaryGidNumber = $user.gidNumber $userGroups = $user.memberOf # Create list of gids this user is part of and add primaryGidNumber to it. $gidArray = New-Object System.Collections.ArrayList if (-not ($null -eq $primaryGidNumber)) { $gidArray.Add($primaryGidNumber) | Out-Null } else { Write-NFSv3ExtendedGroupsLog "Primary gidNumber is null for Uid: $uid" -LogLevel "WARNING" } # # If user has provided file to tell which Uids to upload, # Verify this Uid is present in that list and skip if not. # if ( $uidsFromFileSet.Count -gt 0 ) { if ( -not $uidsFromFileSet.Contains($uidNumber) ) { # If this uid is not part of list provided by user, skip. Write-NFSv3ExtendedGroupsLog "$uidNumber is not present in the file, skipping" -VerbosePreference $VerbosePreference continue } } # Get the gidNumber of these groups and add to gidArray. foreach ($group in $userGroups) { $groupGidNumber = $groupDNToGidMap[$group] $gidArray.Add($groupGidNumber) | Out-Null } # Convert ArrayList to Int32 array. [int32[]]$extendedGroupsArray = $gidArray.ToArray([int32]) | Sort-Object # NFSv3 localuser have name in "nfsv3_<uid>" format. $localUserName = "nfsv3_$($uidNumber)" # Check if local user with same uidNumber exist in localuser database. if ( $localUserMap.ContainsKey($localUserName) ) { # Sort extended group array before comparing. $existingLocalUser = $localUserMap[$localUserName] $existingExtendedGroups = $existingLocalUser.ExtendedGroups | Sort-Object $difference = Compare-Object -ReferenceObject $existingExtendedGroups -DifferenceObject $extendedGroupsArray # Don't PUT local user if database is already updated. if ($difference.Count -eq 0) { $extendedGroupsArrayCommaSeparated = $extendedGroupsArray -join ',' Write-NFSv3ExtendedGroupsLog "ExtendedGroups already up to date for Uid $uidNumber, ExtendedGroups: [$extendedGroupsArrayCommaSeparated]" -VerbosePreference $VerbosePreference continue } } # Create/update localuser. try { Set-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -UserName $localUserName -IsNfSv3Enabled $true -ExtendedGroups $extendedGroupsArray -ErrorAction Stop | Out-Null # On success. $extendedGroupsArrayCommaSeparated = $extendedGroupsArray -join ',' Write-NFSv3ExtendedGroupsLog "Set-AzStorageLocalUser completed successfully for /$ResourceGroupName/$StorageAccountName/$localUserName, ExtendedGroups: [$extendedGroupsArrayCommaSeparated]" -VerbosePreference $VerbosePreference } catch { # On failure. Write-NFSv3ExtendedGroupsLog "Set-AzStorageLocalUser failed for /$ResourceGroupName/$StorageAccountName/$localUserName with error: $_" -LogLevel "ERROR" } } } -ArgumentList $chunkUsers, $groupDNToGidMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $uidsFromFile, $VerbosePreference $jobList.Add($job) } # Wait for all jobs to complete. Write-NFSv3ExtendedGroupsLog "Waiting for all jobs to complete..." -VerbosePreference $VerbosePreference $jobList | ForEach-Object { $_ | Wait-Job } | Out-Null Write-NFSv3ExtendedGroupsLog "All jobs completed" -VerbosePreference $VerbosePreference # Receive the job output and write it to the console. $jobList | ForEach-Object { Receive-Job -Job $_ -Keep } # Clean up completed jobs. $jobList | ForEach-Object { $_ | Remove-Job } Write-NFSv3ExtendedGroupsLog "Sync-NFSv3ExtendedGroupsFromAD completed" -VerbosePreference $VerbosePreference } |