diff --git a/README.md b/README.md index fa8b9cef..0bf2ea31 100644 --- a/README.md +++ b/README.md @@ -59,17 +59,9 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch ## Release history -__Changes__ (2022-Nov-29 / Major) +__Changes__ (2022-Dec-02 / Major) -* Network analysis - fix TenantSummary info if feature is disabled (`-NoNetwork`) -* Use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.1.55 -* Updated TenantSummary screenshot issue #148 - -__Changes__ (2022-Nov-28 / Major) - -* Network analysis - fix Private Endpoints feature - * Handle manual manualPrivateLinkServiceConnections -* Update parameter `-ExcludedResourceTypesDiagnosticsCapable` default with `microsoft.chaos/chaosexperiments` +* PSRule for Azure fix | Get resources using ARM API inside Foreach-Object -parallel loop Passed tests: Powershell Core 7.2.6 on Windows Passed tests: Powershell Core 7.2.7 Azure DevOps hosted agent ubuntu-22.04 diff --git a/history.md b/history.md index 7d1a5118..5ce63377 100644 --- a/history.md +++ b/history.md @@ -4,6 +4,10 @@ ### AzGovViz version 6 +__Changes__ (2022-Dec-02 / Major) + +* PSRule for Azure fix | Get resources using ARM API inside Foreach-Object -parallel loop + __Changes__ (2022-Nov-29 / Major) * Network analysis - fix TenantSummary info if feature is disabled (`-NoNetwork`) diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1 index ae6f95cf..88755565 100644 --- a/pwsh/AzGovVizParallel.ps1 +++ b/pwsh/AzGovVizParallel.ps1 @@ -362,7 +362,7 @@ Param $AzAPICallVersion = '1.1.55', [string] - $ProductVersion = 'v6_major_20221129_1', + $ProductVersion = 'v6_major_20221202_3', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -4953,6 +4953,7 @@ function processDataCollection { $htDefenderEmailContacts = $using:htDefenderEmailContacts $arrayVNets = $using:arrayVNets $arrayPrivateEndPoints = $using:arrayPrivateEndPoints + $htResourceProvidersRef = $using:htResourceProvidersRef #other $function:addRowToTable = $using:funcAddRowToTable $function:namingValidation = $using:funcNamingValidation @@ -28260,14 +28261,62 @@ function dataCollectionResources { $subscriptionQuotaId ) - $currentTask = "Getting ResourceTypes for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime&api-version=2021-04-01" + $currentTask = "Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime,properties&api-version=2021-04-01" $method = 'GET' $resourcesSubscriptionResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' + #Write-Host 'arm resList count:'$resourcesSubscriptionResult.Count #region PSRule if ($azAPICallConf['htParameters'].DoPSRule -eq $true) { if ($resourcesSubscriptionResult.Count -gt 0) { + + $arrayResourcesWithProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) + $resourcesSubscriptionResult | ForEach-Object -Parallel { + $resource = $_ + $resourceId = $resource.id + $arrayResourcesWithProperties = $using:arrayResourcesWithProperties + $azAPICallConf = $using:azAPICallConf + $htResourceProvidersRef = $using:htResourceProvidersRef + + if ($htResourceProvidersRef.($resource.type)) { + $apiVersionToUse = $htResourceProvidersRef.($resource.type).APIFirst + $currentTask = "Getting Resource for PSRule API-version: '{0}'; ResourceId: '{1}'" -f $apiVersionToUse, $resourceId + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($resourceId)?api-version=$apiVersionToUse" + $method = 'GET' + $resource = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' -listenOn Content -unhandledErrorAction Continue + $null = $script:arrayResourcesWithProperties.Add($resource) + } + else { + Write-Host 'Please report at AzGovViz Repo ... No API-version matches!' + Write-Host 'ResourceType:' $resource.type + Write-Host 'ResourceId:' $resourceId + } + + } -ThrottleLimit 20 + #Write-Host 'arm resGet count:' $arrayResourcesWithProperties.Count + + # $currentTask = "Getting Resources (ARG) for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" + # $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01" + # $method = 'POST' + + # $body = @" + # { + # "query": "Resources", + # "subscriptions":[ + # "$($scopeId)" + # ], + # "options": { + # "`$top": 100 + # } + # } + # "@ + # $resourcesSubscriptionResultARG = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -body $body -listenOn Content + + if ($resourcesSubscriptionResult.Count -ne $arrayResourcesWithProperties.Count) { + Write-Host " FYI: Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId'] - ARM list count: $($resourcesSubscriptionResult.Count); ARG get count: $($arrayResourcesWithProperties.Count)" + } + $startPSRule = Get-Date try { <# @@ -28276,10 +28325,10 @@ function dataCollectionResources { Import-Module (Join-Path $path -ChildPath 'PSRule.Rules.Azure-nodeps.psd1') #> if ($azAPICallConf['htParameters'].PSRuleFailedOnly -eq $true) { - $psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error + $psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error } else { - $psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue + $psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue } } catch { @@ -32512,6 +32561,26 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { Write-Host 'Collecting custom data' $startDataCollection = Get-Date + $startGetRPs = Get-Date + $currentTask = 'Getting RPs' + Write-Host $currentTask + $uri = 'https://management.azure.com/providers?api-version=2021-04-01' + $method = 'GET' + $resourceProviders = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' + + $htResourceProvidersRef = @{} + foreach ($resourceProvider in $resourceProviders) { + foreach ($resourceProviderResourceType in $resourceProvider.resourceTypes) { + $APIs = $resourceProviderResourceType.apiVersions | Sort-Object -Descending + $htResourceProvidersRef.("$($resourceProvider.nameSpace)/$($resourceProviderResourceType.resourceType)") = @{ + APIFirst = $APIs | Select-Object -First 1 + APIs = $APIs + } + } + } + $endGetRPs = Get-Date + Write-Host "Getting RPs duration: $((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalSeconds) seconds)" + processDataCollection -mgId $ManagementGroupId showMemoryUsage diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1 index b92d254a..712dc5c1 100644 --- a/pwsh/dev/devAzGovVizParallel.ps1 +++ b/pwsh/dev/devAzGovVizParallel.ps1 @@ -362,7 +362,7 @@ Param $AzAPICallVersion = '1.1.55', [string] - $ProductVersion = 'v6_major_20221129_1', + $ProductVersion = 'v6_major_20221202_3', [string] $GithubRepository = 'aka.ms/AzGovViz', @@ -977,6 +977,26 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) { Write-Host 'Collecting custom data' $startDataCollection = Get-Date + $startGetRPs = Get-Date + $currentTask = 'Getting RPs' + Write-Host $currentTask + $uri = 'https://management.azure.com/providers?api-version=2021-04-01' + $method = 'GET' + $resourceProviders = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' + + $htResourceProvidersRef = @{} + foreach ($resourceProvider in $resourceProviders) { + foreach ($resourceProviderResourceType in $resourceProvider.resourceTypes) { + $APIs = $resourceProviderResourceType.apiVersions | Sort-Object -Descending + $htResourceProvidersRef.("$($resourceProvider.nameSpace)/$($resourceProviderResourceType.resourceType)") = @{ + APIFirst = $APIs | Select-Object -First 1 + APIs = $APIs + } + } + } + $endGetRPs = Get-Date + Write-Host "Getting RPs duration: $((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalSeconds) seconds)" + processDataCollection -mgId $ManagementGroupId showMemoryUsage diff --git a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 index c06738e0..a72f5d10 100644 --- a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 +++ b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 @@ -437,14 +437,62 @@ function dataCollectionResources { $subscriptionQuotaId ) - $currentTask = "Getting ResourceTypes for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" - $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime&api-version=2021-04-01" + $currentTask = "Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime,properties&api-version=2021-04-01" $method = 'GET' $resourcesSubscriptionResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' + #Write-Host 'arm resList count:'$resourcesSubscriptionResult.Count #region PSRule if ($azAPICallConf['htParameters'].DoPSRule -eq $true) { if ($resourcesSubscriptionResult.Count -gt 0) { + + $arrayResourcesWithProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) + $resourcesSubscriptionResult | ForEach-Object -Parallel { + $resource = $_ + $resourceId = $resource.id + $arrayResourcesWithProperties = $using:arrayResourcesWithProperties + $azAPICallConf = $using:azAPICallConf + $htResourceProvidersRef = $using:htResourceProvidersRef + + if ($htResourceProvidersRef.($resource.type)) { + $apiVersionToUse = $htResourceProvidersRef.($resource.type).APIFirst + $currentTask = "Getting Resource for PSRule API-version: '{0}'; ResourceId: '{1}'" -f $apiVersionToUse, $resourceId + $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($resourceId)?api-version=$apiVersionToUse" + $method = 'GET' + $resource = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' -listenOn Content -unhandledErrorAction Continue + $null = $script:arrayResourcesWithProperties.Add($resource) + } + else { + Write-Host 'Please report at AzGovViz Repo ... No API-version matches!' + Write-Host 'ResourceType:' $resource.type + Write-Host 'ResourceId:' $resourceId + } + + } -ThrottleLimit 20 + #Write-Host 'arm resGet count:' $arrayResourcesWithProperties.Count + + # $currentTask = "Getting Resources (ARG) for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']" + # $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01" + # $method = 'POST' + + # $body = @" + # { + # "query": "Resources", + # "subscriptions":[ + # "$($scopeId)" + # ], + # "options": { + # "`$top": 100 + # } + # } + # "@ + # $resourcesSubscriptionResultARG = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -body $body -listenOn Content + + if ($resourcesSubscriptionResult.Count -ne $arrayResourcesWithProperties.Count) { + Write-Host " FYI: Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId'] - ARM list count: $($resourcesSubscriptionResult.Count); ARG get count: $($arrayResourcesWithProperties.Count)" + } + $startPSRule = Get-Date try { <# @@ -453,10 +501,10 @@ function dataCollectionResources { Import-Module (Join-Path $path -ChildPath 'PSRule.Rules.Azure-nodeps.psd1') #> if ($azAPICallConf['htParameters'].PSRuleFailedOnly -eq $true) { - $psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error + $psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error } else { - $psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue + $psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue } } catch { diff --git a/pwsh/dev/functions/processDataCollection.ps1 b/pwsh/dev/functions/processDataCollection.ps1 index daa77466..e18ae329 100644 --- a/pwsh/dev/functions/processDataCollection.ps1 +++ b/pwsh/dev/functions/processDataCollection.ps1 @@ -355,6 +355,7 @@ function processDataCollection { $htDefenderEmailContacts = $using:htDefenderEmailContacts $arrayVNets = $using:arrayVNets $arrayPrivateEndPoints = $using:arrayPrivateEndPoints + $htResourceProvidersRef = $using:htResourceProvidersRef #other $function:addRowToTable = $using:funcAddRowToTable $function:namingValidation = $using:funcNamingValidation diff --git a/version.txt b/version.txt index a204506c..127780a7 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v6_major_20221129_1 \ No newline at end of file +v6_major_20221202_3 \ No newline at end of file