Scripts/Sync-NFSv3ExtendedGroupsFromFlatFile.ps1
|
<#
.SYNOPSIS Synchronizes NFSv3 extended groups from Linux User (/etc/passwd) and Groups (/etc/group) file. .DESCRIPTION This command syncs NFSv3 extended groups with the user and group data fetched from User and Groups file. .PARAMETER ResourceGroupName The name of the Resource Group in Azure where the storage account resides. .PARAMETER StorageAccountName The name of the Azure storage account. .PARAMETER UsersPath File path for user data in format (userName:PassWord:uidNumber:gidNumber:blah:blah:blah) .PARAMETER GroupsPath File path for group data in format (groupName:password:gidNumber:user1,user2,user3) .EXAMPLE Sync-NFSv3ExtendedGroupsFromFlatFile -ResourceGroupName "MyRG" -StorageAccountName "MyStorage" -UsersPath "/Path/To/UserFile" -GroupsPath "/Path/To/GroupsFile" .NOTES Written by: [Azure Blob NFS] Date: [October 10, 2024] #> function Sync-NFSv3ExtendedGroupsFromFlatFile { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string]$ResourceGroupName, [Parameter(Mandatory=$true)] [string]$StorageAccountName, [Parameter(Mandatory=$true)] [string]$UsersPath, [Parameter(Mandatory=$true)] [string]$GroupsPath ) # logMessage is written in this way to keep the indentation correct in log file. $logMessage = @" Sync-NFSv3ExtendedGroupsFromFlatFile started, ResourceGroupName : $ResourceGroupName, StorageAccountName : $StorageAccountName, UsersPath : $UsersPath, GroupsPath : $GroupsPath "@ Write-NFSv3ExtendedGroupsLog $logMessage # Get content of users data. $fileContent = Get-Content -Path $UsersPath # Create dictionaries indexd with userName for storing uid, gid and extendedGroups. $fileUsers = @() $userNameToUidNumberMap = [System.Collections.Generic.Dictionary[string, string]]::new() $userNameToGidNumberMap = [System.Collections.Generic.Dictionary[string, string]]::new() $extendedGroupsType = [System.Collections.Generic.Dictionary[string, System.Collections.ArrayList]] $uidNumberToExtendedGroupsMap = New-Object $extendedGroupsType # # Iterate through users data and store uid, gid and userName. # Also create new object for this particular user to store extendedGroups later. # # User file format: # userName:PassWord:uidNumber:gidNumber:blah:blah:blah # foreach ($line in $fileContent) { $elements = $line -split ':' $fileUsers += $elements[2] $userNameToUidNumberMap.Add($elements[0], $elements[2]) $userNameToGidNumberMap.Add($elements[0], $elements[3]) } # Get content of groups data. $fileContent = Get-Content -Path $GroupsPath # # Iterate through groups data and add this group's gid as extendedGroup of all the users who are part of this group. # # Group file format: # groupName:password:gidNumber:user1,user2,user3 # foreach ($line in $fileContent) { $elements = $line -split ':' $memberUsers = $elements[3] -split ',' # Iterate through all the users and update their extendedGroups list if they are part of this group. foreach ($memberUser in $memberUsers) { if ([string]::IsNullOrEmpty($memberUser)) { continue } $uidNumber = $userNameToUidNumberMap[$memberUser] if (-not $uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)) { $uidNumberToExtendedGroupsMap.Add($uidNumber, (New-Object System.Collections.ArrayList)) } $uidNumberToExtendedGroupsMap[$uidNumber].Add([int32]$elements[2]) | Out-Null } } # Get all the existing LocalUsers for this account. $localUsers = Get-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -IncludeNFSv3 # Initialize hashtable to store LocalUser object with name as key. $localUserMap = @{} # Populate localUserMap hashtable. foreach ( $user in $localUsers ) { $localUserMap[$user.Name] = $user } $logScript = "$PSScriptRoot\Write-NFSv3ExtendedGroupsLog.ps1" $jobList = [System.Collections.Generic.List[System.Management.Automation.Job]]::new() # # There is a limit of 10 Write Ops per second by using ARM with Azure Storage. To avoid requests getting # throttled because of this limit, keep maxParallelJobs less than or equal to 10. # # https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#storage-throttling # $maxParallelJobs = 10 $fileUsersSize = $fileUsers.Length $chunkUsersSize = [Math]::Ceiling($fileUsersSize / $maxParallelJobs) $numOfIterations = [Math]::Min($maxParallelJobs, $fileUsersSize) for ( $i = 0; $i -lt $numOfIterations; $i++ ) { # Slice the fileUsers array into equal size of chunks each of which will be handled by separate processes. $startIndex = $i * $chunkUsersSize $endIndex = [math]::Min($startIndex + $chunkUsersSize - 1, $fileUsersSize -1) $chunkUsers = $fileUsers[$startIndex..$endIndex] # Start a new background job for each chunk of users. $job = Start-Job -ScriptBlock { param ( $chunkUsers, $uidNumberToExtendedGroupsMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript ) # Dot-Source internal logging function cause this background job will create different runspace. $Global:NFSv3ExtendedGroupsLogFile = $using:NFSv3ExtendedGroupsLogFile . $logScript foreach ($uidNumber in $chunkUsers) { # Create ArrayList to store gidNumber of extendedGroups. $gidArray = New-Object System.Collections.ArrayList if ($uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)){ $gidArray = $uidNumberToExtendedGroupsMap[$uidNumber] } # Convert ArrayList to Int32 array. [int32[]]$extendedGroupsArray = $gidArray.ToArray([int32]) | Sort-Object # NFSv3 localuser have name in "nfsv3_<uid>" format. $localUserName = "nfsv3_$($uidNumber)" # Check if local user with same uidNumber exist in localuser database. if ( $localUserMap.ContainsKey($localUserName) ) { # Sort extended group array before comparing. $existingLocalUser = $localUserMap[$localUserName] $existingExtendedGroups = $existingLocalUser.ExtendedGroups | Sort-Object $difference = Compare-Object -ReferenceObject $existingExtendedGroups -DifferenceObject $extendedGroupsArray # Don't PUT local user if database is already updated. if ($difference.Count -eq 0) { Write-NFSv3ExtendedGroupsLog "ExtendedGroups already up to date for Uid $uidNumber" | Out-Null continue } } # Create localuser. Set-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -UserName $localUserName -IsNfSv3Enabled $true -ExtendedGroups $extendedGroupsArray } } -ArgumentList $chunkUsers, $uidNumberToExtendedGroupsMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $uidArray $jobList.Add($job) } # Wait for all jobs to complete. $jobList | ForEach-Object { $_ | Wait-Job } | Out-Null # Clean up completed jobs $jobList | ForEach-Object { $_ | Remove-Job } Write-NFSv3ExtendedGroupsLog "Sync-NFSv3ExtendedGroupsFromFlatFile completed" } |