diff --git a/appveyor.yml b/appveyor.yml index df77dc5a69..b2f56e6e17 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,9 +4,6 @@ configuration: "Debug" build_script: - ps: Set-Service wuauserv -StartupType Manual #otherwise, choco command exits with code 1058 - - ps: choco install dotnet-5.0-sdk | Out-String | Out-Null - - ps: choco install dotnetcore-sdk | Out-String | Out-Null -# - ps: Push-Location bin\projects\dbatools; dotnet build ;Pop-Location version: 2.1.{build} diff --git a/tests/Remove-DbaAvailabilityGroup.Tests.ps1 b/tests/Remove-DbaAvailabilityGroup.Tests.ps1 index bd354a860f..8c22fb51d9 100644 --- a/tests/Remove-DbaAvailabilityGroup.Tests.ps1 +++ b/tests/Remove-DbaAvailabilityGroup.Tests.ps1 @@ -16,11 +16,12 @@ Describe "$commandname Unit Tests" -Tag 'UnitTests' { Describe "$commandname Integration Tests" -Tag "IntegrationTests" { BeforeAll { $agname = "dbatoolsci_removewholegroup" - $null = New-DbaAvailabilityGroup -Primary $script:instance3 -Name $agname -ClusterType None -FailoverMode Manual -Confirm:$false + $null = New-DbaAvailabilityGroup -Primary $script:instance3 -Name $agname -ClusterType None -FailoverMode Manual -Certificate dbatoolsci_AGCert -Confirm:$false } Context "removes the newly created ag" { It "removes the ag" { - $results = Remove-DbaAvailabilityGroup -SqlInstance $script:instance3 -AvailabilityGroup $agname -Confirm:$false + $results = Remove-DbaAvailabilityGroup -SqlInstance $script:instance3 -AvailabilityGroup $agname -Confirm:$false -WarningVariable warn + $warn | Should -BeNullorEmpty $results.Status | Should -Be 'Removed' $results.AvailabilityGroup | Should -Be $agname } diff --git a/tests/appveyor.pester.ps1 b/tests/appveyor.pester.ps1 index b1ced2ba5b..94752baead 100644 --- a/tests/appveyor.pester.ps1 +++ b/tests/appveyor.pester.ps1 @@ -3,7 +3,7 @@ This script will invoke Pester tests, then serialize XML results and pull them in appveyor.yml .DESCRIPTION -Internal function that creates SMO server object. +Internal function that runs pester tests .PARAMETER Finalize If Finalize is specified, we collect XML output, upload tests, and indicate build errors @@ -159,23 +159,13 @@ function Get-CodecovReport($Results, $ModuleBase) { $newreport } -function Send-CodecovReport($CodecovReport) { - $params = @{ } - $params['branch'] = $env:APPVEYOR_REPO_BRANCH - $params['service'] = "appveyor" - $params['job'] = $env:APPVEYOR_ACCOUNT_NAME - if ($params['job']) { $params['job'] += '/' + $env:APPVEYOR_PROJECT_SLUG } - if ($params['job']) { $params['job'] += '/' + $env:APPVEYOR_BUILD_VERSION } - $params['build'] = $env:APPVEYOR_JOB_ID - $params['pr'] = $env:APPVEYOR_PULL_REQUEST_NUMBER - $params['slug'] = $env:APPVEYOR_REPO_NAME - $params['commit'] = $env:APPVEYOR_REPO_COMMIT - Add-Type -AssemblyName System.Web - $CodeCovParams = [System.Web.HttpUtility]::ParseQueryString([String]::Empty) - $params.GetEnumerator() | Where-Object Value | ForEach-Object { $CodeCovParams.Add($_.Name, $_.Value) } - $Request = [System.UriBuilder]('https://codecov.io/upload/v2') - $Request.Query = $CodeCovParams.ToString() - Invoke-RestMethod -Uri $Request.Uri -Method Post -InFile $CodecovReport -ContentType 'multipart/form-data' +function Get-PesterTestVersion($testFilePath) { + $testFileContent = Get-Content -Path $testFilePath -Raw + if ($testFileContent -Like '*#pester5*') + { + return '5' + } + return '4' } @@ -188,15 +178,19 @@ if (-not $Finalize) { #Run a test with the current version of PowerShell #Make things faster by removing most output if (-not $Finalize) { - Import-Module Pester - Write-Host -Object "appveyor.pester: Running with Pester Version $((Get-Command Invoke-Pester -ErrorAction SilentlyContinue).Version)" -ForegroundColor DarkGreen Set-Variable ProgressPreference -Value SilentlyContinue if ($AllScenarioTests.Count -eq 0) { Write-Host -ForegroundColor DarkGreen "Nothing to do in this scenario" return } + # Remove any previously loaded pester module + Remove-Module -Name pester -ErrorAction SilentlyContinue + # Import pester 4 + Import-Module pester -RequiredVersion 4.4.2 + Write-Host -Object "appveyor.pester: Running with Pester Version $((Get-Command Invoke-Pester -ErrorAction SilentlyContinue).Version)" -ForegroundColor DarkGreen # invoking a single invoke-pester consumes too much memory, let's go file by file $AllTestsWithinScenario = Get-ChildItem -File -Path $AllScenarioTests + #start the round for pester 4 tests $Counter = 0 foreach ($f in $AllTestsWithinScenario) { $Counter += 1 @@ -205,6 +199,13 @@ if (-not $Finalize) { 'Show' = 'None' 'PassThru' = $true } + #get if this test should run on pester 4 or pester 5 + $pesterVersionToUse = Get-PesterTestVersion -testFilePath $f.FullName + if ($pesterVersionToUse -eq '5') { + # we're in the "region" of pester 4, so skip + continue + } + #opt-in if ($IncludeCoverage) { $CoverFiles = Get-CoverageIndications -Path $f -ModuleBase $ModuleBase @@ -218,7 +219,7 @@ if (-not $Finalize) { if ($trialNo -eq 1) { $appvTestName = $f.Name } else { - $appvTestName = "$f.Name, attempt #$trialNo" + $appvTestName = "$($f.Name), attempt #$trialNo" } Add-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome Running $PesterRun = Invoke-Pester @PesterSplat @@ -233,6 +234,56 @@ if (-not $Finalize) { } } } + + #start the round for pester 5 tests + # Remove any previously loaded pester module + Remove-Module -Name pester -ErrorAction SilentlyContinue + # Import pester 4 + Import-Module pester -RequiredVersion 5.6.1 + Write-Host -Object "appveyor.pester: Running with Pester Version $((Get-Command Invoke-Pester -ErrorAction SilentlyContinue).Version)" -ForegroundColor DarkGreen + $Counter = 0 + foreach ($f in $AllTestsWithinScenario) { + $Counter += 1 + + #get if this test should run on pester 4 or pester 5 + $pesterVersionToUse = Get-PesterTestVersion -testFilePath $f.FullName + if ($pesterVersionToUse -eq '4') { + # we're in the "region" of pester 5, so skip + continue + } + $pester5Config = New-PesterConfiguration + $pester5Config.Run.Path = $f.FullName + $pester5config.Run.PassThru = $true + #opt-in + if ($IncludeCoverage) { + $CoverFiles = Get-CoverageIndications -Path $f -ModuleBase $ModuleBase + $pester5Config.CodeCoverage.Enabled = $true + $pester5Config.CodeCoverage.Path = $CoverFiles + $pester5Config.CodeCoverage.OutputFormat = 'JaCoCo' + $pester5Config.CodeCoverage.OutputPath = "$ModuleBase\Pester5Coverage$PSVersion$Counter.xml" + } + + $trialNo = 1 + while ($trialNo -le 3) { + if ($trialNo -eq 1) { + $appvTestName = $f.Name + } else { + $appvTestName = "$($f.Name), attempt #$trialNo" + } + Add-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome Running + $PesterRun = Invoke-Pester -Configuration $pester5config + $PesterRun | Export-Clixml -Path "$ModuleBase\Pester5Results$PSVersion$Counter.xml" + $outcome = "Passed" + if ($PesterRun.FailedCount -gt 0) { + $trialno += 1 + Update-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome "Failed" -Duration $PesterRun.Time.TotalMilliseconds + } else { + Update-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome "Passed" -Duration $PesterRun.Time.TotalMilliseconds + break + } + } + } + # Gather support package as an artifact # New-DbatoolsSupportPackage -Path $ModuleBase - turns out to be too heavy try { @@ -281,9 +332,10 @@ if (-not $Finalize) { #$totalcount = $results | Select-Object -ExpandProperty TotalCount | Measure-Object -Sum | Select-Object -ExpandProperty Sum $failedcount = $results | Select-Object -ExpandProperty FailedCount | Measure-Object -Sum | Select-Object -ExpandProperty Sum if ($failedcount -gt 0) { + # pester 4 output $faileditems = $results | Select-Object -ExpandProperty TestResult | Where-Object { $_.Passed -notlike $True } if ($faileditems) { - Write-Warning "Failed tests summary:" + Write-Warning "Failed tests summary (pester 4):" $faileditems | ForEach-Object { $name = $_.Name [pscustomobject]@{ @@ -297,8 +349,30 @@ if (-not $Finalize) { throw "$failedcount tests failed." } } + + + $results5 = @(Get-ChildItem -Path "$ModuleBase\Pester5Results*.xml" | Import-Clixml) + # pester 5 output + $faileditems = $results5 | Select-Object -ExpandProperty Tests | Where-Object { $_.Passed -notlike $True } + if ($faileditems) { + Write-Warning "Failed tests summary (pester 5):" + $faileditems | ForEach-Object { + $name = $_.Name + [pscustomobject]@{ + Path = $_.Path -Join '/' + Name = "It $name" + Result = $_.Result + Message = $_.ErrorRecord -Join "" + } + } | Sort-Object Path, Name, Result, Message | Format-List + throw "$failedcount tests failed." + } + #opt-in if ($IncludeCoverage) { + # for now, this manages recreating a codecov-ingestable format for pester 4. Pester 5 uses JaCoCo natively, which + # codecov accepts ... there's only the small matter that we generate one coverage per run, and there's a run per test file + # and there's no native-powershelly-way to merge JaCoCo reports. Let's start small, and complicate our lives farther down the line. $CodecovReport = Get-CodecovReport -Results $results -ModuleBase $ModuleBase $CodecovReport | ConvertTo-Json -Depth 4 -Compress | Out-File -FilePath "$ModuleBase\PesterResultsCoverage.json" -Encoding utf8 } diff --git a/tests/appveyor.post.ps1 b/tests/appveyor.post.ps1 index a25051f33c..7d695a4560 100644 --- a/tests/appveyor.post.ps1 +++ b/tests/appveyor.post.ps1 @@ -1,7 +1,18 @@ Add-AppveyorTest -Name "appveyor.post" -Framework NUnit -FileName "appveyor.post.ps1" -Outcome Running $sw = [system.diagnostics.stopwatch]::startNew() -Write-Host -Object "appveyor.post: Sending coverage data" -ForeGroundColor DarkGreen +Write-Host -Object "appveyor.post: Sending coverage data (pester 4)" -ForeGroundColor DarkGreen Push-AppveyorArtifact PesterResultsCoverage.json -FileName "PesterResultsCoverage" codecov -f PesterResultsCoverage.json --flag "ps,$($env:SCENARIO.ToLowerInvariant())" | Out-Null + +Write-Host -Object "appveyor.post: Sending coverage data (pester 5)" -ForeGroundColor DarkGreen +$ProjectRoot = $env:APPVEYOR_BUILD_FOLDER, +$ModuleBase = $ProjectRoot, +$pester5CoverageFiles = Get-ChildItem -Path "$ModuleBase\Pester5Coverage*.xml" +foreach($coverageFile in $pester5CoverageFiles) +{ + Push-AppveyorArtifact $coverageFile.FullName -FileName $coverageFile.Name + codecov -f $coverageFile.FullName --flag "ps,$($env:SCENARIO.ToLowerInvariant())" | Out-Null +} + $sw.Stop() Update-AppveyorTest -Name "appveyor.post" -Framework NUnit -FileName "appveyor.post.ps1" -Outcome Passed -Duration $sw.ElapsedMilliseconds \ No newline at end of file diff --git a/tests/appveyor.prep.ps1 b/tests/appveyor.prep.ps1 index 9363d20e3b..bd00f2e055 100644 --- a/tests/appveyor.prep.ps1 +++ b/tests/appveyor.prep.ps1 @@ -19,6 +19,7 @@ git clone -q --branch=master --depth=1 https://github.com/dataplat/appveyor-lab. #Get codecov (to upload coverage results) Write-Host -Object "appveyor.prep: Install codecov" -ForegroundColor DarkGreen choco install codecov | Out-Null +#FIXME : read about the new uploader https://docs.codecov.com/docs/codecov-uploader#using-the-uploader #Get PSScriptAnalyzer (to check warnings) Write-Host -Object "appveyor.prep: Install PSScriptAnalyzer" -ForegroundColor DarkGreen @@ -33,10 +34,14 @@ if (-not(Test-Path 'C:\Program Files\WindowsPowerShell\Modules\dbatools.library' } #Get Pester (to run tests) - choco isn't working onall scenarios, weird -Write-Host -Object "appveyor.prep: Install Pester" -ForegroundColor DarkGreen +Write-Host -Object "appveyor.prep: Install Pester4" -ForegroundColor DarkGreen if (-not(Test-Path 'C:\Program Files\WindowsPowerShell\Modules\Pester\4.4.2')) { Install-Module -Name Pester -Force -SkipPublisherCheck -MaximumVersion 4.4.2 | Out-Null } +Write-Host -Object "appveyor.prep: Install Pester5" -ForegroundColor DarkGreen +if (-not(Test-Path 'C:\Program Files\WindowsPowerShell\Modules\Pester\5.6.1')) { + Install-Module -Name Pester -Force -SkipPublisherCheck -RequiredVersion 5.6.1 | Out-Null +} #Setup DbatoolsConfig Path.DbatoolsExport path Write-Host -Object "appveyor.prep: Create Path.DbatoolsExport" -ForegroundColor DarkGreen @@ -45,10 +50,6 @@ if (-not(Test-Path 'C:\Users\appveyor\Documents\DbatoolsExport')) { } -#Get opencover.portable (to run DLL tests) -Write-Host -Object "appveyor.prep: Install opencover.portable" -ForegroundColor DarkGreen -choco install opencover.portable | Out-Null - Write-Host -Object "appveyor.prep: Trust SQL Server Cert (now required)" -ForegroundColor DarkGreen Import-Module dbatools.library Import-Module C:\github\dbatools\dbatools.psd1 diff --git a/tests/pester.groups.ps1 b/tests/pester.groups.ps1 index 5a373e9670..7815585e77 100644 --- a/tests/pester.groups.ps1 +++ b/tests/pester.groups.ps1 @@ -22,47 +22,32 @@ $TestsRunGroups = @{ ) # do not run on appveyor "appveyor_disabled" = @( - # takes too long - 'Install-DbaSqlWatch', - 'Uninstall-DbaSqlWatch', - 'Get-DbaExecutionPlan', - # Non-useful info from newly started sql servers - 'Get-DbaCpuRingBuffer', - 'Get-DbaLatchStatistic', - # fails on newer version of SMO + # tests that work locally against SQL Server 2022 instances without problems but fail on AppVeyor + 'ConvertTo-DbaXESession', + 'Export-DbaUser', + 'Get-DbaPermission', 'Get-DbaUserPermission', - 'Invoke-DbaBalanceDataFiles', - 'Invoke-DbaWhoisActive', # Works locally aganint a SQL Server 2022 instance without problems. 'Install-DbaDarlingData', - # previous tests that were failing on older versions too + 'Invoke-DbaWhoisActive', 'Remove-DbaAvailabilityGroup', - 'Read-DbaAuditFile', + 'Remove-DbaDatabaseSafely', 'Sync-DbaLoginPermission', - 'Read-DbaXEFile', - 'Stop-DbaXESession', - 'Test-DbaTempDbConfig', - #'New-DbaDbUser', - 'Stop-DbaXESession', + # tests that fail locally against SQL Server 2022 instances and fail on AppVeyor + 'Set-DbaAgentJobStep', 'New-DbaLogin', 'Watch-DbaDbLogin', - 'ConvertTo-DbaXESession', - 'Test-DbaInstanceName', - 'Test-DbaDeprecatedFeature', - 'Remove-DbaDatabaseSafely', - 'Get-DbaDbMasterKey', - 'Get-DbaPermission', - 'Test-DbaManagementObject', + # tests that fail because the command does not work + 'Copy-DbaDbCertificate', 'Export-DbaDacPackage', - 'New-DbaDbTransfer', - 'Get-DbaDbSynonym', - 'Get-DbaDbVirtualLogFile', - 'Get-DbaFile', - 'Get-DbaHelpIndex', - 'Get-DbaExternalProcess', - # just fails too often - 'Test-DbaMaxDop', - 'Test-DbaOptimizeForAdHoc', - 'New-DbaDbSnapshot' + 'Read-DbaAuditFile', + 'Read-DbaXEFile', + # takes too long + 'Install-DbaSqlWatch', + 'Uninstall-DbaSqlWatch', + 'Get-DbaExecutionPlan', + # Non-useful info from newly started sql servers + 'Get-DbaCpuRingBuffer', + 'Get-DbaLatchStatistic' ) # do not run everywhere "disabled" = @()