tasks/python.tasks.ps1
|
$SkipInstallPythonPoetry = $false $SkipInitialisePythonPoetry = $false $SkipRunFlake8 = $false $SkipRunPyTest = $false $SkipRunBehave = $false $SkipBuildPythonPackages = $false $SkipPublishPythonPackages = $false $PythonPoetryProject = "" $PythonPublishUser = "user" $PythonRepositoryName = "ci-python-feed" $PythonPackageRepoUrl = "" # e.g. https://pkgs.dev.azure.com/myOrg/Project/_packaging/myfeed/pypi/upload $PythonPackagePreReleaseTag = $null $PythonPackagesFilenameFilter = "*.whl" $PythonSourceDirectory = "src" $PoetryPath = Get-Command poetry -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Path $PyTestResultsPath = Join-Path $here "pytest-test-results.xml" $BehaveResultsPath = Join-Path $here "behave-test-results.xml" $PythonCoverageReportPath = Join-Path $CoverageDir "coverage.xml" task EnsurePython -If { $PythonPoetryProject -ne "" } { if (!(Get-Command python -ErrorAction SilentlyContinue)) { throw "A Python installation could not be found. Please install Python and ensure it is available on the PATH environment variable." } } task InstallPythonPoetry -If { !$SkipInstallPythonPoetry } EnsurePython,{ $existingPoetry = Get-Command poetry -ErrorAction SilentlyContinue if (!$existingPoetry -and !$PoetryPath) { # The install script will honour this environment variable. If not explicitly set, we set it to: # - On build servers we install within the working directory to ensure it's part of the build agent caching # - Otherwise, we install to the user profile directory in a cross-platform way $env:POETRY_HOME ??= $IsRunningOnBuildServer ? (Join-Path $here ".poetry") : (Join-Path ($IsWindows ? $env:USERPROFILE : $env:HOME) ".poetry") $poetryBinPath = Join-Path $env:POETRY_HOME "bin" # If the poetry binary is not found, install it if (!(Test-Path (Join-Path $poetryBinPath "poetry"))) { Write-Build White "Installing Poetry $env:POETRY_VERSION: $env:POETRY_HOME" Invoke-WebRequest -Uri https://install.python-poetry.org/ -OutFile get-poetry.py exec { & python get-poetry.py --yes } Remove-Item get-poetry.py -Force } # Ensure the poetry tool is avaliable to the rest of the build process $script:PoetryPath = Join-Path $poetryBinPath "poetry" Write-Build Green "Poetry now available: $PoetryPath" if ($poetryBinPath -notin ($env:PATH -split [System.IO.Path]::PathSeparator)) { Write-Build White "Adding Poetry to PATH: $poetryBinPath" $env:PATH = "$poetryBinPath{0}$env:PATH" -f [System.IO.Path]::PathSeparator } } else { if (!$PoetryPath) { # Ensure $PoetryPath is set if poetry was already available in the PATH $script:PoetryPath = $existingPoetry.Path } Write-Build Green "Poetry already installed: $PoetryPath" } } task UpdatePoetryLockfile -If { !$IsRunningOnBuildServer } InstallPythonPoetry,{ Write-Build White "Ensuring poetry.lock is up-to-date - no packages will be updated" Push-Location $PythonPoetryProject exec { & $script:PoetryPath lock --no-update } } task InitialisePythonPoetry -If { $PythonPoetryProject -ne "" -and !$SkipInitialisePythonPoetry } InstallPythonPoetry,UpdatePoetryLockfile,{ if (!(Test-Path (Join-Path $PythonPoetryProject "pyproject.toml"))) { throw "pyproject.toml not found in $PythonPoetryProject" } # Default to using virtual environments in the project directory, unless already set $env:POETRY_VIRTUALENVS_IN_PROJECT ??= "true" # Define the global poetry arguments we will use for all poetry commands $script:poetryGlobalArgs = @( "--no-interaction" "--directory=$PythonPoetryProject" "-v" ) Write-Build White "poetryGlobalArgs: $poetryGlobalArgs" exec { & $script:PoetryPath install @poetryGlobalArgs --with dev,test } } task RunFlake8 -If { $PythonPoetryProject -ne "" -and !$SkipRunFlake8 } InitialisePythonPoetry,{ Write-Build White "Running flake8" # Explicitly change directory as Flake8 does not run when Poetry has the '--directory' argument Set-Location $PythonPoetryProject exec { & $script:PoetryPath run --no-interaction -v flake8 src -vv } } task RunPythonTests -If { $PythonPoetryProject -ne "" || ($SkipRunPyTest && $SkipRunBehave) } InitialisePythonPoetry,{ Write-Build White "Removing previous Python coverage results" # Explicity change directory as 'coverage erase' does not run when Poetry has the '--directory' argument Set-Location $PythonPoetryProject exec { & $script:PoetryPath run coverage erase } $pythonTestErrors = @() try { try { if (!$SkipRunPyTest) { _runPyTests } } catch { $pythonTestErrors += "PyTest Errors, check previous output for details" } try { if (!$SkipRunBehave) { _runBehave } } catch { $pythonTestErrors += "Behave Errors, check previous output for details" } } finally { _generatePythonCoverageXml if ($pythonTestErrors) { $pythonTestsErrorMsg = "{0}{1}" -f [Environment]::NewLine, ($pythonTestErrors -join [Environment]::NewLine) throw $pythonTestsErrorMsg } } } function _runPyTests { # Explicity change directory as PyTest does not run when Poetry has the '--directory' argument Set-Location $PythonPoetryProject exec { & $script:PoetryPath run --no-interaction -v ` pytest ` --cov=$PythonSourceDirectory ` --cov-report= ` --cov-append ` --junitxml=$PyTestResultsPath } } function _runBehave { # Explicity change directory as PyTest does not run when Poetry has the '--directory' argument Set-Location $PythonPoetryProject $testReportsPath = (Join-Path $here "behave-test-reports-temp") if (Test-Path $testReportsPath) { Remove-Item -Path $testReportsPath -Recurse -Force } New-Item -Path $testReportsPath -ItemType Directory | Out-Null try { exec { & $script:PoetryPath run coverage run --source=$PythonSourceDirectory -m behave --junit --junit-directory $testReportsPath } } finally { exec { & $script:PoetryPath run junitparser merge --glob $testReportsPath/*.xml $BehaveResultsPath } } } function _generatePythonCoverageXml { # Explicity change directory as PyTest does not run when Poetry has the '--directory' argument Set-Location $PythonPoetryProject Write-Build White "Generating Python coverage report" exec { & $script:PoetryPath run coverage xml -o $PythonCoverageReportPath } } task BuildPythonPackages -If { $PythonPoetryProject -ne "" -and !$SkipBuildPythonPackages } Version,EnsurePackagesDir,InitialisePythonPoetry,{ if (Test-Path (Join-Path $PythonPoetryProject "dist")) { Remove-Item (Join-Path $PythonPoetryProject "dist") -Recurse -Force } # Apply python pre-release versioning conventions # Ideally the repo will have a GitVersion configuration such that: # - The master/main branch has a pre-release tag of 'rc' (release candidate) # - All other branches have a pre-release tag of 'b' (beta) # - Tagged commits have no pre-release tag # However, as a fallback we must ensure that we always have a PEP440 compliant pre-release tag # even if GitVersion does not $safePreReleaseLabel = Get-PythonPackagePreReleaseLabelFromSemVer -PreReleaseLabel $GitVersion.PreReleaseLabel $PythonPackagePreReleaseTag ??= "{0}{1}" -f $safePreReleaseLabel, $GitVersion.PreReleaseNumber $pythonPackageVersion = "$($GitVersion.MajorMinorPatch)$PythonPackagePreReleaseTag" Write-Build White "Building Python packages with version: $pythonPackageVersion" exec { & $script:PoetryPath version @poetryGlobalArgs $pythonPackageVersion } # Make the Python package version available to the rest of the build, since it could be different to the GitVersion Set-BuildServerVariable -Name "PythonPackageVersion" -Value $pythonPackageVersion # Build the package(s) # For the moment we live with the fact that Poetry's output path is not configurable exec { & $script:PoetryPath build @poetryGlobalArgs } } task PublishPythonPackages -If { $PythonPoetryProject -ne "" -and !$SkipPublishPythonPackages } InitialisePythonPoetry,{ # Copy the Python packages from the standard packaging output folder to where Poetry expects to find them $distPath = Join-Path $PythonPoetryProject "dist" $pythonPackages = gci $distPath/$PythonPackagesFilenameFilter if (!$pythonPackages ) { Write-Warning "No Python packages found, skipping publish" } elseif (!$PythonPackageRepoUrl) { Write-Warning "PythonPackageRepoUrl build variable not set, skipping publish" } elseif (!$env:PYTHON_PACKAGE_REPOSITORY_KEY) { Write-Warning "PYTHON_PACKAGE_REPOSITORY_KEY environment variable not set, skipping publish" } else { exec { Write-Build White "Registering Python repository $PythonRepositoryName -> $PythonPackageRepoUrl" & $script:PoetryPath config @poetryGlobalArgs repositories.$PythonRepositoryName $PythonPackageRepoUrl } exec { Write-Build White "Publishing Python packages to $PythonRepositoryName" & $script:PoetryPath publish @poetryGlobalArgs -u $PythonPublishUser -p $env:PYTHON_PACKAGE_REPOSITORY_KEY -r $PythonRepositoryName } } } task BuildPython -If { $PythonPoetryProject -ne "" } InitialisePythonPoetry,RunFlake8 |