diff --git a/CHANGELOG.md b/CHANGELOG.md index 775791a..cd8089f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +- [**#96**](https://github.com/psake/PowerShellBuild/issues/96) + `Test-PSBuildScriptAnalysis` now fails the build when PSScriptAnalyzer + reports findings at or above the configured severity threshold. The + severity counts were computed from `$_Severity` — an undefined variable — + instead of `$_.Severity`, so all three counts were always zero and the + `Error`, `Warning`, and `Information` thresholds could never fail a build. + Script analysis reported its findings and the build passed regardless. + Because the default `FailBuildOnSeverityLevel` is `Error`, any consumer + running the `Analyze` task had a gate that only ever looked like it was + working. See the + [v0.8 → v1.0 migration guide](docs/migration-v0.8-to-v1.0.md) — a build + that passed before may now correctly fail. +- [**#96**](https://github.com/psake/PowerShellBuild/issues/96) + `Test-PSBuildScriptAnalysis` no longer fails with a path-resolution error + when `SettingsPath` is not supplied. An unsupplied path was forwarded to + PSScriptAnalyzer as `-Settings ''`, which resolved against the current + directory and threw before any analysis ran, so the function's own + documented example could not run as written. - [**#102**](https://github.com/psake/PowerShellBuild/issues/102) `Test-PSBuildPester` no longer raises a parameter-binding error from its cleanup logic when the optional `ModuleName` parameter is not supplied. diff --git a/PowerShellBuild/Public/Test-PSBuildScriptAnalysis.ps1 b/PowerShellBuild/Public/Test-PSBuildScriptAnalysis.ps1 index 9d47628..deff786 100644 --- a/PowerShellBuild/Public/Test-PSBuildScriptAnalysis.ps1 +++ b/PowerShellBuild/Public/Test-PSBuildScriptAnalysis.ps1 @@ -28,12 +28,27 @@ function Test-PSBuildScriptAnalysis { Write-Verbose ($LocalizedData.SeverityThresholdSetTo -f $SeverityThreshold) - $analysisResult = Invoke-ScriptAnalyzer -Path $Path -Settings $SettingsPath -Recurse -Verbose:$VerbosePreference - $errors = ($analysisResult.where({ $_Severity -eq 'Error' })).Count - $warnings = ($analysisResult.where({ $_Severity -eq 'Warning' })).Count - $infos = ($analysisResult.where({ $_Severity -eq 'Information' })).Count + $invokeScriptAnalyzerParameters = @{ + Path = $Path + Recurse = $true + } + # An unsupplied SettingsPath must not be forwarded. PSScriptAnalyzer resolves an empty + # -Settings value against the current directory and fails before any analysis runs. + if (-not [string]::IsNullOrWhiteSpace($SettingsPath)) { + $invokeScriptAnalyzerParameters.Settings = $SettingsPath + } + + $analysisResult = Invoke-ScriptAnalyzer @invokeScriptAnalyzerParameters -Verbose:$VerbosePreference + + # A single diagnostic record comes back as a scalar rather than a collection, and Windows + # PowerShell 5.1 does not expose .Where() or .Count on every scalar type. Wrapping in @() + # guarantees collection semantics on both engines. + $analysisRecords = @($analysisResult) + $errorCount = ($analysisRecords.Where({ $_.Severity -eq 'Error' })).Count + $warningCount = ($analysisRecords.Where({ $_.Severity -eq 'Warning' })).Count + $informationCount = ($analysisRecords.Where({ $_.Severity -eq 'Information' })).Count - if ($analysisResult) { + if ($analysisRecords.Count -gt 0) { Write-Host $LocalizedData.PSScriptAnalyzerResults -ForegroundColor Yellow $analysisResult | Format-Table -AutoSize } @@ -43,22 +58,22 @@ function Test-PSBuildScriptAnalysis { return } 'Error' { - if ($errors -gt 0) { + if ($errorCount -gt 0) { throw $LocalizedData.ScriptAnalyzerErrors } } 'Warning' { - if ($errors -gt 0 -or $warnings -gt 0) { + if ($errorCount -gt 0 -or $warningCount -gt 0) { throw $LocalizedData.ScriptAnalyzerWarnings } } 'Information' { - if ($errors -gt 0 -or $warnings -gt 0 -or $infos -gt 0) { + if ($errorCount -gt 0 -or $warningCount -gt 0 -or $informationCount -gt 0) { throw $LocalizedData.ScriptAnalyzerWarnings } } default { - if ($analysisResult.Count -ne 0) { + if ($analysisRecords.Count -ne 0) { throw $LocalizedData.ScriptAnalyzerIssues } } diff --git a/docs/migration-v0.8-to-v1.0.md b/docs/migration-v0.8-to-v1.0.md index 0ecfa48..04b0b6e 100644 --- a/docs/migration-v0.8-to-v1.0.md +++ b/docs/migration-v0.8-to-v1.0.md @@ -10,8 +10,10 @@ This guide helps you upgrade a consumer `build.ps1` (or equivalent) from PowerShellBuild **0.8.x** to **1.0.0**. -It only covers **breaking changes**. For new features and bug fixes that -do not require user action, see [`CHANGELOG.md`](../CHANGELOG.md). +It covers **breaking changes**, plus any **behavioral change that can +require action when you upgrade** — including bug fixes that make a +previously passing build start failing. For new features and fixes that +need nothing from you, see [`CHANGELOG.md`](../CHANGELOG.md). ## Quick Start @@ -20,6 +22,9 @@ One line per break; follow the link for details and migration steps. - [Minimum supported PowerShell version is now 5.1](#minimum-supported-powershell-version-is-now-51) — the manifest requires PowerShell 5.1+; the support floor is Windows PowerShell 5.1 or PowerShell 7.4+. +- [Script analysis now actually fails the build](#script-analysis-now-actually-fails-the-build) + — the `Analyze` task's severity threshold never fired in 0.8.x; a build + that passed before may now correctly fail. > More entries will follow as the Phase 2 migrations to > Microsoft.PowerShell.PlatyPS 1.x and psake 5.x land. @@ -104,6 +109,53 @@ Tracked in PR record and platform validation details in [#120 (comment)](https://github.com/psake/PowerShellBuild/issues/120#issuecomment-5028978464). +### Script analysis now actually fails the build + +This is a bug fix, but it is listed here because it can turn a build that +passed on 0.8.x red on 1.0.0 without anything else changing. + +In 0.8.x, `Test-PSBuildScriptAnalysis` counted findings by severity using +`$_Severity` — an undefined variable — rather than `$_.Severity`. Every +count was therefore zero, and the `Error`, `Warning`, and `Information` +thresholds could never fail a build. The `Analyze` task printed its +findings and then passed. Since `FailBuildOnSeverityLevel` defaults to +`Error`, this affected every consumer who ran the task: the gate reported +problems but never enforced them. + +The counts are now correct, so the threshold you have configured is +enforced for the first time. + +**No configuration change is required.** If your build starts failing at +the `Analyze` task after upgrading, the analyzer findings it reports were +present before too — they were simply never enforced. You have three +options: + +1. Fix the reported findings (recommended — they are real). +2. Exclude specific rules through your PSScriptAnalyzer settings file, + pointed to by `$PSBPreference.Test.ScriptAnalysis.SettingsPath`. +3. Raise the bar or turn enforcement off: + +**Report findings without failing the build:** + + $PSBPreference.Test.ScriptAnalysis.FailBuildOnSeverityLevel = 'None' + +**Fail only on errors, ignoring warnings:** + + $PSBPreference.Test.ScriptAnalysis.FailBuildOnSeverityLevel = 'Error' + +To see what will be enforced before you upgrade, run PSScriptAnalyzer +against your built module directly: + + Invoke-ScriptAnalyzer -Path ./Output/MyModule/1.0.0 -Recurse | + Group-Object -Property Severity + +A related fix ships alongside it: omitting `-SettingsPath` no longer +fails with a path-resolution error, so +`Test-PSBuildScriptAnalysis -Path ./Output/MyModule/0.1.0 -SeverityThreshold Error` +now runs as documented instead of throwing. + +Tracked in issue #96. + ## Adding an entry (for PR contributors) Every breaking-change PR that lands in v1.0.0 must add an entry here for diff --git a/tests/Test-PSBuildScriptAnalysis.tests.ps1 b/tests/Test-PSBuildScriptAnalysis.tests.ps1 new file mode 100644 index 0000000..600a114 --- /dev/null +++ b/tests/Test-PSBuildScriptAnalysis.tests.ps1 @@ -0,0 +1,332 @@ +# Unit tests for Test-PSBuildScriptAnalysis (psake/PowerShellBuild#96). +# +# The tests are layered: +# +# 1. Severity gating runs against a mocked Invoke-ScriptAnalyzer so every finding/threshold +# combination is exercised deterministically. Real analysis cannot cover the whole matrix: +# PSScriptAnalyzer rule severities can change between releases, and no rule in the default +# rule set reliably emits an Information record. +# 2. A smaller end-to-end layer runs the real analyzer over scripts generated into $TestDrive, +# proving the function is actually wired to PSScriptAnalyzer and honors a settings file. +# +# The generated scripts contain deliberate analyzer violations. They are written at runtime and +# never checked in, so the repository's own script analysis can never discover them (see #97). + +Describe 'Test-PSBuildScriptAnalysis' { + + BeforeAll { + $script:moduleRoot = Split-Path -Path $PSScriptRoot -Parent + Import-Module -Name ([IO.Path]::Combine($script:moduleRoot, 'Output', 'PowerShellBuild')) -Force + + $script:cleanPath = Join-Path -Path $TestDrive -ChildPath 'clean' + $script:errorFindingPath = Join-Path -Path $TestDrive -ChildPath 'errorfinding' + $script:warningFindingPath = Join-Path -Path $TestDrive -ChildPath 'warningfinding' + foreach ($directory in @($script:cleanPath, $script:errorFindingPath, $script:warningFindingPath)) { + New-Item -Path $directory -ItemType Directory -Force > $null + } + + # No findings under the default rule set. + Set-Content -Path (Join-Path -Path $script:cleanPath -ChildPath 'Clean.ps1') -Value @( + 'function Get-CleanResult {' + ' [CmdletBinding()]' + ' param()' + '' + " Write-Output -InputObject 'clean'" + '}' + ) + + # Triggers PSAvoidUsingConvertToSecureStringWithPlainText, an Error-severity rule. + Set-Content -Path (Join-Path -Path $script:errorFindingPath -ChildPath 'ErrorFinding.ps1') -Value @( + 'function Get-ErrorFinding {' + ' [CmdletBinding()]' + ' param()' + '' + " ConvertTo-SecureString -String 'plaintext' -AsPlainText -Force" + '}' + ) + + # Triggers PSAvoidUsingWriteHost, a Warning-severity rule. + Set-Content -Path (Join-Path -Path $script:warningFindingPath -ChildPath 'WarningFinding.ps1') -Value @( + 'function Get-WarningFinding {' + ' [CmdletBinding()]' + ' param()' + '' + " Write-Host 'warning finding'" + '}' + ) + + # An empty settings file leaves the default rule set in place. + $script:defaultSettingsPath = Join-Path -Path $TestDrive -ChildPath 'DefaultSettings.psd1' + Set-Content -Path $script:defaultSettingsPath -Value '@{}' + + # Excludes the only rule the warning fixture triggers, so a settings file that is + # actually passed through to the analyzer produces no findings at all. + $script:excludeWriteHostSettingsPath = Join-Path -Path $TestDrive -ChildPath 'ExcludeWriteHost.psd1' + Set-Content -Path $script:excludeWriteHostSettingsPath -Value @( + '@{' + " ExcludeRules = @('PSAvoidUsingWriteHost')" + '}' + ) + } + + Context 'Command surface' { + + It 'Is exported by the module' { + Get-Command -Name 'Test-PSBuildScriptAnalysis' -Module 'PowerShellBuild' -ErrorAction SilentlyContinue | + Should -Not -BeNullOrEmpty + } + + It 'Has a synopsis in the help' { + (Get-Help -Name 'Test-PSBuildScriptAnalysis').Synopsis | Should -Not -BeNullOrEmpty + } + + It 'Has at least one example in the help' { + (Get-Help -Name 'Test-PSBuildScriptAnalysis').Examples.Example | Should -Not -BeNullOrEmpty + } + + It 'Requires the Path parameter' { + $command = Get-Command -Name 'Test-PSBuildScriptAnalysis' + $command.Parameters['Path'].Attributes.Where({ $_.TypeId.Name -eq 'ParameterAttribute' }).Mandatory | + Should -Contain $true + } + + It 'Accepts only the documented severity thresholds' { + $command = Get-Command -Name 'Test-PSBuildScriptAnalysis' + $validateSet = $command.Parameters['SeverityThreshold'].Attributes.Where({ + $_.TypeId.Name -eq 'ValidateSetAttribute' + })[0] + ($validateSet.ValidValues | Sort-Object) -join ',' | Should -Be 'Error,Information,None,Warning' + } + } + + Context 'Severity gating' { + + # Regression coverage for the gate that never fired: the severity counts were computed + # from $_Severity (an undefined variable) rather than $_.Severity, so every count was + # zero and the Error, Warning, and Information thresholds could never fail a build. + # + # Each mock returns a single record, which PowerShell unrolls to a scalar. That also + # guards the collection semantics the function depends on: Windows PowerShell 5.1 does + # not expose .Where() or .Count on every scalar type, so the analyzer result has to be + # wrapped in @() before it is inspected. + + It 'Fails when a finding meets the threshold' -ForEach @( + @{ FindingSeverity = 'Error'; Threshold = 'Error' } + @{ FindingSeverity = 'Error'; Threshold = 'Warning' } + @{ FindingSeverity = 'Error'; Threshold = 'Information' } + @{ FindingSeverity = 'Warning'; Threshold = 'Warning' } + @{ FindingSeverity = 'Warning'; Threshold = 'Information' } + @{ FindingSeverity = 'Information'; Threshold = 'Information' } + ) { + $mockFindings = @( + [PSCustomObject]@{ + Severity = $FindingSeverity + RuleName = "Fake${FindingSeverity}Rule" + ScriptName = 'Fake.ps1' + Message = "A fake $FindingSeverity record" + } + ) + Mock -CommandName 'Invoke-ScriptAnalyzer' -ModuleName 'PowerShellBuild' -MockWith { + $mockFindings + }.GetNewClosure() + + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = $Threshold + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Throw + } + + It 'Passes when a finding is below the threshold' -ForEach @( + @{ FindingSeverity = 'Warning'; Threshold = 'Error' } + @{ FindingSeverity = 'Information'; Threshold = 'Error' } + @{ FindingSeverity = 'Information'; Threshold = 'Warning' } + ) { + $mockFindings = @( + [PSCustomObject]@{ + Severity = $FindingSeverity + RuleName = "Fake${FindingSeverity}Rule" + ScriptName = 'Fake.ps1' + Message = "A fake $FindingSeverity record" + } + ) + Mock -CommandName 'Invoke-ScriptAnalyzer' -ModuleName 'PowerShellBuild' -MockWith { + $mockFindings + }.GetNewClosure() + + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = $Threshold + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + It 'Reports but never fails at the None threshold for a finding' -ForEach @( + @{ FindingSeverity = 'Error' } + @{ FindingSeverity = 'Warning' } + @{ FindingSeverity = 'Information' } + ) { + $mockFindings = @( + [PSCustomObject]@{ + Severity = $FindingSeverity + RuleName = "Fake${FindingSeverity}Rule" + ScriptName = 'Fake.ps1' + Message = "A fake $FindingSeverity record" + } + ) + Mock -CommandName 'Invoke-ScriptAnalyzer' -ModuleName 'PowerShellBuild' -MockWith { + $mockFindings + }.GetNewClosure() + + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = 'None' + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + It 'Passes at the threshold when the analyzer reports nothing' -ForEach @( + @{ Threshold = 'None' } + @{ Threshold = 'Error' } + @{ Threshold = 'Warning' } + @{ Threshold = 'Information' } + ) { + Mock -CommandName 'Invoke-ScriptAnalyzer' -ModuleName 'PowerShellBuild' -MockWith { @() } + + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = $Threshold + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + It 'Fails on any finding when no severity threshold is supplied' { + $mockFindings = @( + [PSCustomObject]@{ + Severity = 'Information' + RuleName = 'FakeInformationRule' + ScriptName = 'Fake.ps1' + Message = 'A fake Information record' + } + ) + Mock -CommandName 'Invoke-ScriptAnalyzer' -ModuleName 'PowerShellBuild' -MockWith { + $mockFindings + }.GetNewClosure() + + $testParameters = @{ + Path = $script:cleanPath + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Throw + } + + It 'Analyzes the supplied path recursively' { + Mock -CommandName 'Invoke-ScriptAnalyzer' -ModuleName 'PowerShellBuild' -MockWith { @() } + + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = 'Error' + SettingsPath = $script:defaultSettingsPath + } + Test-PSBuildScriptAnalysis @testParameters + + $shouldInvokeParameters = @{ + CommandName = 'Invoke-ScriptAnalyzer' + ModuleName = 'PowerShellBuild' + Times = 1 + Exactly = $true + ParameterFilter = { + $Path -eq $script:cleanPath -and + $Settings -eq $script:defaultSettingsPath -and + $Recurse + } + } + Should -Invoke @shouldInvokeParameters + } + } + + Context 'End-to-end analysis' { + + It 'Fails a script with an Error-severity finding at the Error threshold' { + $testParameters = @{ + Path = $script:errorFindingPath + SeverityThreshold = 'Error' + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Throw + } + + It 'Passes a script with only a Warning-severity finding at the Error threshold' { + $testParameters = @{ + Path = $script:warningFindingPath + SeverityThreshold = 'Error' + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + It 'Fails a script with a Warning-severity finding at the Warning threshold' { + $testParameters = @{ + Path = $script:warningFindingPath + SeverityThreshold = 'Warning' + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Throw + } + + It 'Passes a script with no findings at the Error threshold' { + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = 'Error' + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + # Exercises the no-threshold branch against a real empty result. The analyzer returns + # nothing rather than an empty collection when it finds no issues, so this covers the + # $null case that a mocked empty array cannot reproduce. + It 'Passes a script with no findings when no severity threshold is supplied' { + $testParameters = @{ + Path = $script:cleanPath + SettingsPath = $script:defaultSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + } + + Context 'Settings file handling' { + + It 'Honors a settings file that excludes the triggered rule' { + $testParameters = @{ + Path = $script:warningFindingPath + SeverityThreshold = 'Warning' + SettingsPath = $script:excludeWriteHostSettingsPath + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + # Regression: an unsupplied SettingsPath was passed to the analyzer as -Settings '', + # which PSScriptAnalyzer resolved against the current directory and rejected, so the + # documented example (no -SettingsPath) failed with a path error instead of analyzing. + It 'Analyzes with the default rule set when no settings path is supplied' { + $testParameters = @{ + Path = $script:cleanPath + SeverityThreshold = 'Error' + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Not -Throw + } + + It 'Still detects findings when no settings path is supplied' { + $testParameters = @{ + Path = $script:errorFindingPath + SeverityThreshold = 'Error' + } + { Test-PSBuildScriptAnalysis @testParameters } | Should -Throw + } + } +}