From a6898956d076628ecc635318f6cc19b4cc8f8dfe Mon Sep 17 00:00:00 2001 From: Richard Crutchfield Date: Wed, 26 Jul 2023 17:21:07 -0400 Subject: [PATCH] WIP --- .../Functional/Products/Products.Tests.ps1 | 90 +++++++- .../Products/TestPlans/aad.testplan.yaml | 212 +++++++++++++++++- 2 files changed, 281 insertions(+), 21 deletions(-) diff --git a/Testing/Functional/Products/Products.Tests.ps1 b/Testing/Functional/Products/Products.Tests.ps1 index a061f86232..9bec5959bd 100644 --- a/Testing/Functional/Products/Products.Tests.ps1 +++ b/Testing/Functional/Products/Products.Tests.ps1 @@ -31,7 +31,6 @@ $TestContainers = @() $TestContainers += New-PesterContainer -Path "Testing/Functional/Products" -Data @{ TenantDomain = "y2zj1.onmicrosoft.com"; TenantDisplayName = "y2zj1"; ProductName = "sharepoint"; M365Environment = "commercial" } Invoke-Pester -Container $TestContainers -Output Detailed - #> [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', 'Thumbprint', Justification = 'False positive as rule does not scan child scopes')] @@ -86,6 +85,7 @@ BeforeDiscovery{ $YamlString = Get-Content -Path $TestPlanPath | Out-String $ProductTestPlan = ConvertFrom-Yaml $YamlString $TestPlan = $ProductTestPlan.TestPlan.ToArray() + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'Tests', Justification = 'Variable is used in ScriptBlock')] $Tests = $TestPlan.Tests $ServicePrincipalParams = @{CertThumbprintParams = @{ @@ -122,31 +122,64 @@ BeforeAll{ } # Dot source utility functions - . Join-Path -Path $PSScriptRoot -ChildPath "FunctionalTestUtils.ps1" + . (Join-Path -Path $PSScriptRoot -ChildPath "FunctionalTestUtils.ps1") function SetConditions { + [CmdletBinding(DefaultParameterSetName = 'Actual')] param( - [Parameter(Mandatory = $true)] + [Parameter(Mandatory = $true, ParameterSetName = 'Actual')] + [Parameter(Mandatory = $true, ParameterSetName = 'Cached')] [AllowEmptyCollection()] [array] - $Conditions + $Conditions, + [Parameter(Mandatory = $true, ParameterSetName = 'Cached')] + [string] + $OutputFolder ) ForEach($Condition in $Conditions){ [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'Splat', Justification = 'Variable is used in ScriptBlock')] $Splat = $Condition.Splat + + if ('Cached' -eq $PSCmdlet.ParameterSetName){ + Write-Output "Setting output folder to: $OutputFolder" + $Splat.Add("OutputFolder", [string]$OutputFolder) + } + $ScriptBlock = [ScriptBlock]::Create("$($Condition.Command) @Splat") try { $ScriptBlock.Invoke() } - catch [Newtonsoft.Json.JsonReaderException]{ + catch { Write-Error $PSItem.ToString() } } } - function ExecuteScubagear() { + function UpdateProviderExport{ + param( + [Parameter(Mandatory = $true)] + [AllowNull()] + [object] + $Updates, + [Parameter(Mandatory = $true)] + [string] + $OutputFolder + ) + + $ProviderExport = LoadProviderExport($OutputFolder) + + foreach ($Update in $Updates.ToArray()){ + $Key = $Update.Key + $ProviderExport.$Key = $Update.Value + } + + PublishProviderExport -OutputFolder $OutputFolder -Export $ProviderExport + + } + + function RunScuba() { # Execute ScubaGear to extract the config data and produce the output JSON Invoke-SCuBA -CertificateThumbPrint $Thumbprint -AppId $AppId -Organization $TenantDomain -Productnames $ProductName -OutPath . -M365Environment $M365Environment -Quiet } @@ -155,6 +188,27 @@ BeforeAll{ $IntermediateTestResults = Get-Content "$OutputFolder/TestResults.json" -Raw | ConvertFrom-Json $IntermediateTestResults } + function LoadProviderExport($OutputFolder) { + if (-not (Test-Path -Path "$OutputFolder/ModifiedProviderSettingsExport.json" -PathType Leaf)){ + Copy-Item -Path "$OutputFolder/ProviderSettingsExport.json" -Destination "$OutputFolder/ModifiedProviderSettingsExport.json" + } + + $ProviderExport = Get-Content "$OutputFolder/ModifiedProviderSettingsExport.json" -Raw | ConvertFrom-Json + $ProviderExport + } + +function PublishProviderExport() { + param( + [Parameter(Mandatory = $true)] + [string] + $OutputFolder, + [Parameter(Mandatory = $true)] + [object] + $Export + ) + $Json = $Export | ConvertTo-Json -Depth 10 | Out-String + Set-Content -Path "$OutputFolder/ModifiedProviderSettingsExport.json" -Value $Json +} function RemoveConditionalAccessPolicyByName{ [CmdletBinding()] @@ -167,7 +221,7 @@ BeforeAll{ $Ids = Get-MgIdentityConditionalAccessPolicy | Where-Object {$_.DisplayName -match $DisplayName} | Select-Object -Property Id - foreach($Id in $Ids){ + foreach($Id in $Ids){ if (-not ([string]::IsNullOrEmpty($Id.Id))){ Write-Output "Removing $DisplayName with id of $($Id.Id)" Remove-MgIdentityConditionalAccessPolicy -ConditionalAccessPolicyId $Id.Id @@ -203,12 +257,26 @@ BeforeAll{ Describe "Policy Checks for "{ Context "Start tests for policy " -ForEach $TestPlan{ BeforeEach{ - SetConditions -Conditions $Preconditions.ToArray() - ExecuteScubagear + if ('RunScuba' -eq $TestDriver){ + SetConditions -Conditions $Preconditions.ToArray() + RunScuba + } + elseif ('RunCached' -eq $TestDriver){ + RunScuba + $ReportFolders = Get-ChildItem . -directory -Filter "M365BaselineConformance*" | Sort-Object -Property LastWriteTime -Descending + $OutputFolder = $ReportFolders[0].Name + SetConditions -Conditions $Preconditions.ToArray() -OutputFolder $OutputFolder + Invoke-RunCached -Productnames $ProductName -ExportProvider $false -OutPath $OutputFolder -OutProviderFileName 'ModifiedProviderSettingsExport' -Quiet + } + else { + Write-Error -Message "Invalid Test Driver: $TestDriver" + } + $ReportFolders = Get-ChildItem . -directory -Filter "M365BaselineConformance*" | Sort-Object -Property LastWriteTime -Descending $OutputFolder = $ReportFolders[0] $IntermediateTestResults = LoadTestResults($OutputFolder) # Search the results object for the specific requirement we are validating and ensure the results are what we expect + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'PolicyResultObj', Justification = 'Variable is used in ScriptBlock')] $PolicyResultObj = $IntermediateTestResults | Where-Object { $_.PolicyId -eq $PolicyId } $BaselineReports = Join-Path -Path $OutputFolder -ChildPath 'BaselineReports.html' $script:url = (Get-Item $BaselineReports).FullName @@ -218,7 +286,7 @@ Describe "Policy Checks for "{ Context "Execute test, " -ForEach $Tests { It "Check test case results" { - # Check intermediate output + #Check intermediate output $PolicyResultObj.RequirementMet | Should -Be $ExpectedResult $Details = $PolicyResultObj.ReportDetails @@ -318,7 +386,7 @@ Describe "Policy Checks for "{ $RowData[4].GetAttribute("innerHTML") | FromInnerHtml | Should -BeExactly $PolicyResultObj.ReportDetails } else { - $false | Should -BeTrue -Because "policy should be custom, not checked, or have and expected result. [$Msg]" + $false | Should -BeTrue -Because "policy should be custom, not checked, or have and expected result. [$Msg]" } } } diff --git a/Testing/Functional/Products/TestPlans/aad.testplan.yaml b/Testing/Functional/Products/TestPlans/aad.testplan.yaml index 821e7ae89b..64db0ba6c6 100644 --- a/Testing/Functional/Products/TestPlans/aad.testplan.yaml +++ b/Testing/Functional/Products/TestPlans/aad.testplan.yaml @@ -1,6 +1,7 @@ ProductName: aad TestPlan: - PolicyId: MS.AAD.1.1v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.1.1v1 ReportOnly Preconditions: @@ -21,6 +22,7 @@ TestPlan: Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.2.1v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.2.1v1 Report Only Preconditions: @@ -41,6 +43,7 @@ TestPlan: Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.2.2v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.2.2v1 Not Checked Preconditions: [] @@ -48,18 +51,67 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.2.3v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.2.3v1 Default Preconditions: [] Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.3.1v1 + TestDriver: RunCached Tests: + - TestDescription: MS.AAD.3.1v1 Bad Authentication Strength + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "conditional_access_policies" + Value: + - Conditions: + Applications: + IncludeApplications: + - "All" + ExcludeApplications: [] + Users: + IncludeUsers: + - "All" + GrantControls: + AuthenticationStrength: + AllowedCombinations: + - "windowsHelloForBusiness" + - "Super Strength" + - "x509CertificateMultiFactor" + State: "enabled" + DisplayName: "MS.AAD.3.1v1 Bad Authentication Strength" + Postconditions: [] + ExpectedResult: false - TestDescription: MS.AAD.3.1v1 Default - Preconditions: [] + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "conditional_access_policies" + Value: + - Conditions: + Applications: + IncludeApplications: + - "All" + ExcludeApplications: [] + Users: + IncludeUsers: + - "All" + GrantControls: + AuthenticationStrength: + AllowedCombinations: + - "windowsHelloForBusiness" + - "fido2" + - "x509CertificateMultiFactor" + State: "enabled" + DisplayName: "Test Policy - MS.AAD.3.1v1 Default" Postconditions: [] - ExpectedResult: false + ExpectedResult: true - PolicyId: MS.AAD.3.2v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.2v1 MFA Not Required Preconditions: @@ -80,6 +132,7 @@ TestPlan: Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.3.3v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.3v1 Not Checked Preconditions: [] @@ -87,6 +140,7 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.3.4v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.4v1 Not Checked Preconditions: [] @@ -94,6 +148,7 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.3.5v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.5v1 Not Checked Preconditions: [] @@ -101,6 +156,7 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.3.6v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.6v1 Report Only Preconditions: @@ -121,18 +177,21 @@ TestPlan: Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.3.7v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.7v1 Default Preconditions: [] Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.3.8v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.3.8v1 Default Preconditions: [] Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.4.1v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.4.1v1 Not Checked Preconditions: [] @@ -140,6 +199,7 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.5.1v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.5.1v1 Allow Preconditions: @@ -160,34 +220,46 @@ TestPlan: Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.5.2v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.5.2v1 Default Preconditions: [] Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.5.3v1 + TestDriver: RunCached Tests: - TestDescription: MS.AAD.5.3v1 No Admin Consent Request Policy Enabled Preconditions: - - Command: Update-MgPolicyAdminConsentRequestPolicy + - Command: UpdateProviderExport Splat: - IsEnabled: false + Updates: + - Key: "admin_consent_policies" + Value: + - Id: "Test Policy - MS.AAD.5.3v1 No Admin Consent Request Policy Enabled" + IsEnabled: false Postconditions: [] ExpectedResult: false - TestDescription: MS.AAD.5.3v1 Default Preconditions: - - Command: Update-MgPolicyAdminConsentRequestPolicy + - Command: UpdateProviderExport Splat: - IsEnabled: true + Updates: + - Key: "admin_consent_policies" + Value: + - Id: "Test Policy - MS.AAD.5.3v1 Default" + IsEnabled: true Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.5.4v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.5.4v1 Default Preconditions: [] Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.6.1v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.6.1v1 Not Checked Preconditions: [] @@ -195,12 +267,91 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.7.1v1 + TestDriver: RunCached Tests: - - TestDescription: MS.AAD.7.1v1 Default - Preconditions: [] + - TestDescription: MS.AAD.7.1v1 To Many Admin + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "privileged_users" + Value: + User1: + DisplayName: "Test User1" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + User2: + DisplayName: "Test User2" + roles: + - "Global Administrator" + User3: + DisplayName: "Test User3" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + User4: + DisplayName: "Test User4" + roles: + - "Global Administrator" + User5: + DisplayName: "Test User5" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + User6: + DisplayName: "Test User6" + roles: + - "Global Administrator" + User7: + DisplayName: "Test User7" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + User8: + DisplayName: "Test User8" + roles: + - "Global Administrator" + User9: + DisplayName: "Test User9" + roles: + - "Global Administrator" Postconditions: [] ExpectedResult: false + - TestDescription: MS.AAD.7.1v1 To Few Admin + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "privileged_users" + Value: + User1: + DisplayName: "Test User1" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + Postconditions: [] + ExpectedResult: false + - TestDescription: MS.AAD.7.1v1 Default + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "privileged_users" + Value: + User1: + DisplayName: "Test User1" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + User2: + DisplayName: "Test User2" + roles: + - "Global Administrator" + Postconditions: [] + ExpectedResult: true - PolicyId: MS.AAD.7.2v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.7.2v1 Not Checked Preconditions: [] @@ -208,48 +359,87 @@ TestPlan: IsNotChecked: true ExpectedResult: false - PolicyId: MS.AAD.7.3v1 + TestDriver: RunCached Tests: + - TestDescription: MS.AAD.7.3v1 On Prem Admin + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "privileged_users" + Value: + User1: + DisplayName: "Test User1" + OnPremisesImmutableId: "Test On-Prem User1" + roles: + - "Privileged Role Administrator" + - "Global Administrator" + Postconditions: [] + ExpectedResult: false - TestDescription: MS.AAD.7.3v1 Default - Preconditions: [] + Preconditions: + - Command: UpdateProviderExport + Splat: + Updates: + - Key: "privileged_users" + Value: + User1: + DisplayName: "Test User1" + OnPremisesImmutableId: null + roles: + - "Privileged Role Administrator" + - "Global Administrator" + User2: + DisplayName: "Test User2" + OnPremisesImmutableId: null + roles: + - "Global Administrator" Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.7.4v1 + TestDriver: RunCached Tests: - TestDescription: MS.AAD.7.4v1 Default Preconditions: [] Postconditions: [] - ExpectedResult: false + ExpectedResult: true - PolicyId: MS.AAD.7.5v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.7.5v1 Default Preconditions: [] Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.7.6v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.7.6v1 Default Preconditions: [] Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.7.7v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.7.7v1 Default Preconditions: [] Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.7.8v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.7.8v1 Default Preconditions: [] Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.7.9v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.7.9v1 Default Preconditions: [] Postconditions: [] ExpectedResult: false - PolicyId: MS.AAD.8.1v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.8.1v1 Restricted access Preconditions: @@ -276,6 +466,7 @@ TestPlan: Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.8.2v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.8.2v1 Everyone Preconditions: @@ -294,6 +485,7 @@ TestPlan: Postconditions: [] ExpectedResult: true - PolicyId: MS.AAD.8.3v1 + TestDriver: RunScuba Tests: - TestDescription: MS.AAD.8.3v1 Not Checked Preconditions: []