Scripts/SyncNFSv3ExtendedGroupsFromFlatFile.ps1
|
<#
.SYNOPSIS Synchronizes NFSv3 extended groups from Linux User (/etc/passwd) and Groups (/etc/group) file. .DESCRIPTION This command syncs NFSv3 extended groups with the user and group data fetched from User and Groups file. .PARAMETER ResourceGroupName The name of the Resource Group in Azure where the storage account resides. .PARAMETER StorageAccountName The name of the Azure storage account. .PARAMETER UsersPath File path for user data in format (userName:PassWord:uidNumber:gidNumber:blah:blah:blah) .PARAMETER GroupsPath File path for group data in format (groupName:password:gidNumber:user1,user2,user3) .EXAMPLE Sync-NFSv3ExtendedGroupsFromFlatFile -ResourceGroupName "MyRG" -StorageAccountName "MyStorage" -UsersPath "/Path/To/UserFile" -GroupsPath "/Path/To/GroupsFile" .NOTES Written by: [Azure Blob NFS] Date: [October 10, 2024] #> function Sync-NFSv3ExtendedGroupsFromFlatFile { [CmdletBinding()] param( [Parameter(Mandatory=$true)] [string]$ResourceGroupName, [Parameter(Mandatory=$true)] [string]$StorageAccountName, [Parameter(Mandatory=$true)] [string]$UsersPath, [Parameter(Mandatory=$true)] [string]$GroupsPath ) # logMessage is written in this way to keep the indentation correct in log file. $logMessage = @" Sync-NFSv3ExtendedGroupsFromFlatFile started, ResourceGroupName : $ResourceGroupName, StorageAccountName : $StorageAccountName, UsersPath : $UsersPath, GroupsPath : $GroupsPath "@ Write-NFSv3ExtendedGroupsLog $logMessage -VerbosePreference $VerbosePreference # Get content of users data. $fileContent = Get-Content -Path $UsersPath # Create dictionaries indexed with userName for storing uid, gid and extendedGroups. $fileUids = @() $userNameToUidNumberMap = [System.Collections.Generic.Dictionary[string, string]]::new() $uidNumberToExtendedGroupsMap = [System.Collections.Generic.Dictionary[string, System.Collections.ArrayList]]::new() # # Iterate through users data and store uid, gid and userName. # Also create new object for this particular user to store extendedGroups later. # # User file format: # userName:PassWord:uidNumber:gidNumber:blah:blah:blah # foreach ($line in $fileContent) { $elements = $line -split ':' # Verify if each line has all the required fileds. if ($elements.Length -lt 4) { Write-NFSv3ExtendedGroupsLog "'$line' in '$UsersPath' does not have correct format. Each line should be in 'userName:PassWord:uidNumber:gidNumber:blah:blah:blah' format" -LogLevel "WARNING" continue } # Get user, uidNumber and primaryGidNumber from parsed elements. $user = $elements[0] $uidNumber = $elements[2] $primaryGidNumber = $elements[3] # Verify if required fileds have expected data type. if ([string]::IsNullOrEmpty($user)) { Write-NFSv3ExtendedGroupsLog "UserName filed is null or empty in '$line' in '$UsersPath'" -LogLevel "WARNING" continue } if (-not ($uidNumber -match '\d+$' -and $primaryGidNumber -match '\d+$')) { Write-NFSv3ExtendedGroupsLog "Invalid Uid or Gid in '$line' in '$UsersPath', Uid and Gid should have integer value" -LogLevel "WARNING" continue } # Add these values to corresponding array and maps. $fileUids += $uidNumber $userNameToUidNumberMap.Add($user, $uidNumber) # Add primaryGidNumber of this user to uidNumberToExtendedGroupsMap. if (-not $uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)) { $uidNumberToExtendedGroupsMap.Add($uidNumber, (New-Object System.Collections.ArrayList)) } $uidNumberToExtendedGroupsMap[$uidNumber].Add([int32]$primaryGidNumber) | Out-Null } # Get content of groups data. $fileContent = Get-Content -Path $GroupsPath # # Iterate through groups data and add this group's gid as extendedGroup of all the users who are part of this group. # # Group file format: # groupName:password:gidNumber:user1,user2,user3 # foreach ($line in $fileContent) { $elements = $line -split ':' # Verify if each line has all the required fileds. if ($elements.Length -lt 4) { Write-NFSv3ExtendedGroupsLog "'$line' in '$GroupsPath' does not have correct format. Each line should be in 'groupName:password:gidNumber:user1,user2,user3' format" -LogLevel "WARNING" continue } $group = $elements[0] $gidNumber = $elements[2] $memberUsers = $elements[3] -split ',' # Verify if required fileds have expected data type. if ([string]::IsNullOrEmpty($group)) { Write-NFSv3ExtendedGroupsLog "GroupName filed is null or empty in '$line' in '$GroupsPath'" -LogLevel "WARNING" continue } if ($gidNumber -notmatch '\d+$') { Write-NFSv3ExtendedGroupsLog "Invalid Gid in '$line' in '$GroupsPath', Gid should have integer value" -LogLevel "WARNING" continue } # Skip processing if there are no memberUser for this group. if ([string]::IsNullOrEmpty($elements[3])) { Write-NFSv3ExtendedGroupsLog "No member user for Gid: $gidNumber" -LogLevel "WARNING" continue } # Iterate through all the users and update their extendedGroups list if they are part of this group. foreach ($memberUser in $memberUsers) { if ([string]::IsNullOrEmpty($memberUser)) { Write-NFSv3ExtendedGroupsLog "One of the memberUser filed is null or empty in '$line' in '$GroupsPath'" -LogLevel "WARNING" continue } # Skip if this user does not exist in '/etc/passwd' file. if (-not $userNameToUidNumberMap.ContainsKey($memberUser)) { Write-NFSv3ExtendedGroupsLog "User: $memberUser does not exist in '$UsersPath' but member of group id: $gidNumber" -LogLevel "WARNING" continue } $uidNumber = $userNameToUidNumberMap[$memberUser] if (-not $uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)) { $uidNumberToExtendedGroupsMap.Add($uidNumber, (New-Object System.Collections.ArrayList)) } $uidNumberToExtendedGroupsMap[$uidNumber].Add([int32]$gidNumber) | Out-Null } } # Get all the existing LocalUsers for this account. try { $localUsers = Get-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -IncludeNFSv3 -ErrorAction Stop Write-NFSv3ExtendedGroupsLog "Fetched $($localUsers.Count) LocalUsers from Azure Storage server" -VerbosePreference $VerbosePreference } catch { Write-NFSv3ExtendedGroupsLog "Get-AzStorageLocalUser failed with error: $_" -LogLevel "ERROR" } # Initialize hashtable to store LocalUser object with name as key. $localUserMap = @{} # Populate localUserMap hashtable. foreach ( $user in $localUsers ) { $localUserMap[$user.Name] = $user } $logScript = "$PSScriptRoot\WriteNFSv3ExtendedGroupsLog.ps1" $jobList = [System.Collections.Generic.List[System.Management.Automation.Job]]::new() # # There is a limit of 10 Write Ops per second by using ARM with Azure Storage. To avoid requests getting # throttled because of this limit, keep maxParallelJobs less than or equal to 10. # # https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#storage-throttling # $maxParallelJobs = 10 $fileUidsSize = $fileUids.Length $chunkUidsSize = [Math]::Ceiling($fileUidsSize / $maxParallelJobs) $numOfIterations = [Math]::Ceiling($fileUidsSize / $chunkUidsSize) for ( $i = 0; $i -lt $numOfIterations; $i++ ) { # Slice the fileUids array into equal size of chunks each of which will be handled by separate processes. $startIndex = $i * $chunkUidsSize $endIndex = [math]::Min($startIndex + $chunkUidsSize - 1, $fileUidsSize -1) $chunkUids = $fileUids[$startIndex..$endIndex] # Start a new background job for each chunk of users. $job = Start-Job -ScriptBlock { param ( $chunkUids, $uidNumberToExtendedGroupsMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $VerbosePreference ) # Dot-Source internal logging function and import required modules cause this background job will create different runspace. Import-Module Az.Storage 4>&1 | Out-Null $Global:NFSv3ExtendedGroupsLogFile = $using:NFSv3ExtendedGroupsLogFile . $logScript foreach ($uidNumber in $chunkUids) { # Create ArrayList to store gidNumber of extendedGroups. $gidArray = New-Object System.Collections.ArrayList if ($uidNumberToExtendedGroupsMap.ContainsKey($uidNumber)){ $gidArray = $uidNumberToExtendedGroupsMap[$uidNumber] } # Don't process if extended groups array is empty. if (-not $gidArray -or $gidArray.Count -eq 0) { Write-NFSv3ExtendedGroupsLog "ExtendedGroups array is null or empty for Uid: $uidNumber" -VerbosePreference $VerbosePreference continue } # Convert ArrayList to Int32 array. [int32[]]$extendedGroupsArray = $gidArray.ToArray([int32]) | Sort-Object # NFSv3 localuser have name in "nfsv3_<uid>" format. $localUserName = "nfsv3_$($uidNumber)" # Check if local user with same uidNumber exist in localuser database. if ( $localUserMap.ContainsKey($localUserName) ) { # Sort extended group array before comparing. $existingLocalUser = $localUserMap[$localUserName] $existingExtendedGroups = $existingLocalUser.ExtendedGroups | Sort-Object $difference = Compare-Object -ReferenceObject $existingExtendedGroups -DifferenceObject $extendedGroupsArray # Don't PUT local user if database is already updated. if ($difference.Count -eq 0) { $extendedGroupsArrayCommaSeparated = $extendedGroupsArray -join ',' Write-NFSv3ExtendedGroupsLog "ExtendedGroups already up to date for Uid: $uidNumber, ExtendedGroups: [$extendedGroupsArrayCommaSeparated]" -VerbosePreference $VerbosePreference continue } } # Create/update localuser. try { Set-AzStorageLocalUser -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -UserName $localUserName -IsNfSv3Enabled $true -ExtendedGroups $extendedGroupsArray -ErrorAction Stop | Out-Null $extendedGroupsArrayCommaSeparated = $extendedGroupsArray -join ',' Write-NFSv3ExtendedGroupsLog "Set-AzStorageLocalUser completed successfully for /$ResourceGroupName/$StorageAccountName/$localUserName, ExtendedGroups: [$extendedGroupsArrayCommaSeparated]" -VerbosePreference $VerbosePreference } catch { # On failure. Write-NFSv3ExtendedGroupsLog "Set-AzStorageLocalUser failed for /$ResourceGroupName/$StorageAccountName/$localUserName with error: $_" -LogLevel "ERROR" } } } -ArgumentList $chunkUids, $uidNumberToExtendedGroupsMap, $localUserMap, $ResourceGroupName, $StorageAccountName, $logScript, $VerbosePreference $jobList.Add($job) } # Wait for all jobs to complete. Write-NFSv3ExtendedGroupsLog "Waiting for all jobs to complete..." -VerbosePreference $VerbosePreference $jobList | ForEach-Object { $_ | Wait-Job } | Out-Null Write-NFSv3ExtendedGroupsLog "All jobs completed" -VerbosePreference $VerbosePreference # Receive the job output and write it to the console. $jobList | ForEach-Object { Receive-Job -Job $_ } # Clean up completed jobs. $jobList | ForEach-Object { $_ | Remove-Job } Write-NFSv3ExtendedGroupsLog "Sync-NFSv3ExtendedGroupsFromFlatFile completed" -VerbosePreference $VerbosePreference } |