Skip to content

Commit

Permalink
v6_major_20221202_3
Browse files Browse the repository at this point in the history
  • Loading branch information
@JulianHayward
JulianHayward committed Dec 2, 2022
1 parent 74363de commit 92cb043
Show file tree
Hide file tree
Showing 7 changed files with 155 additions and 21 deletions.
12 changes: 2 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,17 +59,9 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch

## Release history

__Changes__ (2022-Nov-29 / Major)
__Changes__ (2022-Dec-02 / Major)

* Network analysis - fix TenantSummary info if feature is disabled (`-NoNetwork`)
* Use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.1.55
* Updated TenantSummary screenshot issue #148

__Changes__ (2022-Nov-28 / Major)

* Network analysis - fix Private Endpoints feature
* Handle manual manualPrivateLinkServiceConnections
* Update parameter `-ExcludedResourceTypesDiagnosticsCapable` default with `microsoft.chaos/chaosexperiments`
* PSRule for Azure fix | Get resources using ARM API inside Foreach-Object -parallel loop

Passed tests: Powershell Core 7.2.6 on Windows
Passed tests: Powershell Core 7.2.7 Azure DevOps hosted agent ubuntu-22.04
Expand Down
4 changes: 4 additions & 0 deletions history.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@

### AzGovViz version 6

__Changes__ (2022-Dec-02 / Major)

* PSRule for Azure fix | Get resources using ARM API inside Foreach-Object -parallel loop

__Changes__ (2022-Nov-29 / Major)

* Network analysis - fix TenantSummary info if feature is disabled (`-NoNetwork`)
Expand Down
79 changes: 74 additions & 5 deletions pwsh/AzGovVizParallel.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ Param
$AzAPICallVersion = '1.1.55',

[string]
$ProductVersion = 'v6_major_20221129_1',
$ProductVersion = 'v6_major_20221202_3',

[string]
$GithubRepository = 'aka.ms/AzGovViz',
Expand Down Expand Up @@ -4953,6 +4953,7 @@ function processDataCollection {
$htDefenderEmailContacts = $using:htDefenderEmailContacts
$arrayVNets = $using:arrayVNets
$arrayPrivateEndPoints = $using:arrayPrivateEndPoints
$htResourceProvidersRef = $using:htResourceProvidersRef
#other
$function:addRowToTable = $using:funcAddRowToTable
$function:namingValidation = $using:funcNamingValidation
Expand Down Expand Up @@ -28260,14 +28261,62 @@ function dataCollectionResources {
$subscriptionQuotaId
)

$currentTask = "Getting ResourceTypes for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
$uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime&api-version=2021-04-01"
$currentTask = "Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
$uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime,properties&api-version=2021-04-01"
$method = 'GET'
$resourcesSubscriptionResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'
#Write-Host 'arm resList count:'$resourcesSubscriptionResult.Count

#region PSRule
if ($azAPICallConf['htParameters'].DoPSRule -eq $true) {
if ($resourcesSubscriptionResult.Count -gt 0) {

$arrayResourcesWithProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
$resourcesSubscriptionResult | ForEach-Object -Parallel {
$resource = $_
$resourceId = $resource.id
$arrayResourcesWithProperties = $using:arrayResourcesWithProperties
$azAPICallConf = $using:azAPICallConf
$htResourceProvidersRef = $using:htResourceProvidersRef

if ($htResourceProvidersRef.($resource.type)) {
$apiVersionToUse = $htResourceProvidersRef.($resource.type).APIFirst
$currentTask = "Getting Resource for PSRule API-version: '{0}'; ResourceId: '{1}'" -f $apiVersionToUse, $resourceId
$uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($resourceId)?api-version=$apiVersionToUse"
$method = 'GET'
$resource = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' -listenOn Content -unhandledErrorAction Continue
$null = $script:arrayResourcesWithProperties.Add($resource)
}
else {
Write-Host 'Please report at AzGovViz Repo ... No API-version matches!'
Write-Host 'ResourceType:' $resource.type
Write-Host 'ResourceId:' $resourceId
}

} -ThrottleLimit 20
#Write-Host 'arm resGet count:' $arrayResourcesWithProperties.Count

# $currentTask = "Getting Resources (ARG) for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
# $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01"
# $method = 'POST'

# $body = @"
# {
# "query": "Resources",
# "subscriptions":[
# "$($scopeId)"
# ],
# "options": {
# "`$top": 100
# }
# }
# "@
# $resourcesSubscriptionResultARG = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -body $body -listenOn Content

if ($resourcesSubscriptionResult.Count -ne $arrayResourcesWithProperties.Count) {
Write-Host " FYI: Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId'] - ARM list count: $($resourcesSubscriptionResult.Count); ARG get count: $($arrayResourcesWithProperties.Count)"
}

$startPSRule = Get-Date
try {
<#
Expand All @@ -28276,10 +28325,10 @@ function dataCollectionResources {
Import-Module (Join-Path $path -ChildPath 'PSRule.Rules.Azure-nodeps.psd1')
#>
if ($azAPICallConf['htParameters'].PSRuleFailedOnly -eq $true) {
$psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error
$psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error
}
else {
$psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue
$psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue
}
}
catch {
Expand Down Expand Up @@ -32512,6 +32561,26 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
Write-Host 'Collecting custom data'
$startDataCollection = Get-Date

$startGetRPs = Get-Date
$currentTask = 'Getting RPs'
Write-Host $currentTask
$uri = 'https://management.azure.com/providers?api-version=2021-04-01'
$method = 'GET'
$resourceProviders = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'

$htResourceProvidersRef = @{}
foreach ($resourceProvider in $resourceProviders) {
foreach ($resourceProviderResourceType in $resourceProvider.resourceTypes) {
$APIs = $resourceProviderResourceType.apiVersions | Sort-Object -Descending
$htResourceProvidersRef.("$($resourceProvider.nameSpace)/$($resourceProviderResourceType.resourceType)") = @{
APIFirst = $APIs | Select-Object -First 1
APIs = $APIs
}
}
}
$endGetRPs = Get-Date
Write-Host "Getting RPs duration: $((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalSeconds) seconds)"

processDataCollection -mgId $ManagementGroupId

showMemoryUsage
Expand Down
22 changes: 21 additions & 1 deletion pwsh/dev/devAzGovVizParallel.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ Param
$AzAPICallVersion = '1.1.55',

[string]
$ProductVersion = 'v6_major_20221129_1',
$ProductVersion = 'v6_major_20221202_3',

[string]
$GithubRepository = 'aka.ms/AzGovViz',
Expand Down Expand Up @@ -977,6 +977,26 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
Write-Host 'Collecting custom data'
$startDataCollection = Get-Date

$startGetRPs = Get-Date
$currentTask = 'Getting RPs'
Write-Host $currentTask
$uri = 'https://management.azure.com/providers?api-version=2021-04-01'
$method = 'GET'
$resourceProviders = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'

$htResourceProvidersRef = @{}
foreach ($resourceProvider in $resourceProviders) {
foreach ($resourceProviderResourceType in $resourceProvider.resourceTypes) {
$APIs = $resourceProviderResourceType.apiVersions | Sort-Object -Descending
$htResourceProvidersRef.("$($resourceProvider.nameSpace)/$($resourceProviderResourceType.resourceType)") = @{
APIFirst = $APIs | Select-Object -First 1
APIs = $APIs
}
}
}
$endGetRPs = Get-Date
Write-Host "Getting RPs duration: $((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalSeconds) seconds)"

processDataCollection -mgId $ManagementGroupId

showMemoryUsage
Expand Down
56 changes: 52 additions & 4 deletions pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,14 +437,62 @@ function dataCollectionResources {
$subscriptionQuotaId
)

$currentTask = "Getting ResourceTypes for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
$uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime&api-version=2021-04-01"
$currentTask = "Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
$uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime,properties&api-version=2021-04-01"
$method = 'GET'
$resourcesSubscriptionResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'
#Write-Host 'arm resList count:'$resourcesSubscriptionResult.Count

#region PSRule
if ($azAPICallConf['htParameters'].DoPSRule -eq $true) {
if ($resourcesSubscriptionResult.Count -gt 0) {

$arrayResourcesWithProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
$resourcesSubscriptionResult | ForEach-Object -Parallel {
$resource = $_
$resourceId = $resource.id
$arrayResourcesWithProperties = $using:arrayResourcesWithProperties
$azAPICallConf = $using:azAPICallConf
$htResourceProvidersRef = $using:htResourceProvidersRef

if ($htResourceProvidersRef.($resource.type)) {
$apiVersionToUse = $htResourceProvidersRef.($resource.type).APIFirst
$currentTask = "Getting Resource for PSRule API-version: '{0}'; ResourceId: '{1}'" -f $apiVersionToUse, $resourceId
$uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($resourceId)?api-version=$apiVersionToUse"
$method = 'GET'
$resource = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' -listenOn Content -unhandledErrorAction Continue
$null = $script:arrayResourcesWithProperties.Add($resource)
}
else {
Write-Host 'Please report at AzGovViz Repo ... No API-version matches!'
Write-Host 'ResourceType:' $resource.type
Write-Host 'ResourceId:' $resourceId
}

} -ThrottleLimit 20
#Write-Host 'arm resGet count:' $arrayResourcesWithProperties.Count

# $currentTask = "Getting Resources (ARG) for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
# $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01"
# $method = 'POST'

# $body = @"
# {
# "query": "Resources",
# "subscriptions":[
# "$($scopeId)"
# ],
# "options": {
# "`$top": 100
# }
# }
# "@
# $resourcesSubscriptionResultARG = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -body $body -listenOn Content

if ($resourcesSubscriptionResult.Count -ne $arrayResourcesWithProperties.Count) {
Write-Host " FYI: Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId'] - ARM list count: $($resourcesSubscriptionResult.Count); ARG get count: $($arrayResourcesWithProperties.Count)"
}

$startPSRule = Get-Date
try {
<#
Expand All @@ -453,10 +501,10 @@ function dataCollectionResources {
Import-Module (Join-Path $path -ChildPath 'PSRule.Rules.Azure-nodeps.psd1')
#>
if ($azAPICallConf['htParameters'].PSRuleFailedOnly -eq $true) {
$psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error
$psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error
}
else {
$psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue
$psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue
}
}
catch {
Expand Down
1 change: 1 addition & 0 deletions pwsh/dev/functions/processDataCollection.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ function processDataCollection {
$htDefenderEmailContacts = $using:htDefenderEmailContacts
$arrayVNets = $using:arrayVNets
$arrayPrivateEndPoints = $using:arrayPrivateEndPoints
$htResourceProvidersRef = $using:htResourceProvidersRef
#other
$function:addRowToTable = $using:funcAddRowToTable
$function:namingValidation = $using:funcNamingValidation
Expand Down
2 changes: 1 addition & 1 deletion version.txt
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v6_major_20221129_1
v6_major_20221202_3

0 comments on commit 92cb043

Please sign in to comment.