Merge pull request #151 from JulianHayward/Fix

v6_major_20221204_1

.azuredevops/pipelines/AzGovViz.variables.yml

@@ -1,4 +1,4 @@
-# AzGovViz v6_major_20221023_1
+# AzGovViz v6_major_20221202_4
 # First things first:
 # 1. Replace <YourServiceConnection> with the name of your service connection
 # 2. Replace <YourManagementGroupId> with the your ManagementGroupId
@@ -257,7 +257,7 @@ variables:
 
   # Define the number of script blocks running in parallel. Leveraging PowerShell Core´s parallel capability you can define the ThrottleLimit (default=5)
   - name: ThrottleLimit
-    # Integer | Default = 5 | example: value: 10
+    # Integer | Default = 10 | example: value: 11
     value:
 
   # Will show memory usage

README.md

@@ -59,19 +59,18 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch
 
 ## Release history
 
-__Changes__ (2022-Nov-29 / Major)
-
-* Network analysis - fix TenantSummary info if feature is disabled (`-NoNetwork`)
-* Use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.1.55
-* Updated TenantSummary screenshot issue #148
-
-__Changes__ (2022-Nov-28 / Major)
-
-* Network analysis - fix Private Endpoints feature
-  * Handle manual manualPrivateLinkServiceConnections
-* Update parameter `-ExcludedResourceTypesDiagnosticsCapable` default with `microsoft.chaos/chaosexperiments`
-
-Passed tests: Powershell Core 7.2.6 on Windows  
+__Changes__ (2022-Dec-04 / Major)
+
+* PSRule for Azure fix | Get resources using ARM API inside Foreach-Object -parallel loop
+* Private Endpoints 
+  * fix resource identification
+  * add cross tenant detection
+* Storage Account Access Analysis - add insights on 'Allowed Copy Scope' and 'Allow Cross Tenant Replication'
+* Updated [API reference](#api-reference)
+* Cosmetics
+* Bugfixes
+
+Passed tests: Powershell Core 7.3.0 on Windows  
 Passed tests: Powershell Core 7.2.7 Azure DevOps hosted agent ubuntu-22.04  
 Passed tests: Powershell Core 7.2.7 Github Actions hosted agent ubuntu-latest  
 Passed tests: Powershell Core 7.2.6 GitHub Codespaces mcr.microsoft.com/powershell:latest  
@@ -523,6 +522,7 @@ AzGovViz polls the following APIs
 | MS Graph | v1.0 | /servicePrincipals |
 | ARM | 2021-05-01-preview | /`resourceId`/providers/Microsoft.Insights/diagnosticSettingsCategories |
 | ARM | 2018-11-01-preview | /`scopeId`/providers/Microsoft.Blueprint/blueprints/`blueprintName` |
+| ARM | 2021-04-01 | /providers |
 | ARM | 2021-06-01 | /providers/Microsoft.Authorization/policyDefinitions |
 | ARM | 2021-06-01 | /providers/Microsoft.Authorization/policySetDefinitions |
 | ARM | 2020-02-01 | /providers/Microsoft.Management/getEntities |

history.md

@@ -4,6 +4,17 @@
 
 ### AzGovViz version 6
 
+__Changes__ (2022-Dec-04 / Major)
+
+* PSRule for Azure fix | Get resources using ARM API inside Foreach-Object -parallel loop
+* Private Endpoints 
+  * fix resource identification
+  * add cross tenant detection
+* Storage Account Access Analysis - add insights on 'Allowed Copy Scope' and 'Allow Cross Tenant Replication'
+* Updated [API reference](#api-reference)
+* Cosmetics
+* Bugfixes
+
 __Changes__ (2022-Nov-29 / Major)
 
 * Network analysis - fix TenantSummary info if feature is disabled (`-NoNetwork`)

pwsh/dev/devAzGovVizParallel.ps1

@@ -359,10 +359,10 @@ Param
     $Product = 'AzGovViz',
 
     [string]
-    $AzAPICallVersion = '1.1.55',
+    $AzAPICallVersion = '1.1.58',
 
     [string]
-    $ProductVersion = 'v6_major_20221129_1',
+    $ProductVersion = 'v6_major_20221204_1',
 
     [string]
     $GithubRepository = 'aka.ms/AzGovViz',
@@ -914,6 +914,8 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
     $htDefenderEmailContacts = [System.Collections.Hashtable]::Synchronized((New-Object System.Collections.Hashtable))
     $arrayVNets = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
     $arrayPrivateEndPoints = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
+    $arrayPrivateEndPointsFromResourceProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
+    $htUnknownTenantsForSubscription = @{}
 }
 
 if (-not $HierarchyMapOnly) {
@@ -977,6 +979,26 @@ if ($azAPICallConf['htParameters'].HierarchyMapOnly -eq $false) {
     Write-Host 'Collecting custom data'
     $startDataCollection = Get-Date
 
+    $startGetRPs = Get-Date
+    $currentTask = 'Getting Tenant Resource Providers'
+    Write-Host $currentTask
+    $uri = 'https://management.azure.com/providers?api-version=2021-04-01'
+    $method = 'GET'
+    $resourceProviders = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'
+
+    $htResourceProvidersRef = @{}
+    foreach ($resourceProvider in $resourceProviders) {
+        foreach ($resourceProviderResourceType in $resourceProvider.resourceTypes) {
+            $APIs = $resourceProviderResourceType.apiVersions | Sort-Object -Descending
+            $htResourceProvidersRef.("$($resourceProvider.nameSpace)/$($resourceProviderResourceType.resourceType)") = @{
+                APIFirst = $APIs | Select-Object -First 1
+                APIs     = $APIs
+            }
+        }
+    }
+    $endGetRPs = Get-Date
+    Write-Host " Getting Tenant Resource Providers duration: $((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalSeconds) seconds)"
+
     processDataCollection -mgId $ManagementGroupId
 
     showMemoryUsage
@@ -1454,7 +1476,7 @@ $html = @"
         link.media = "screen,print";
         document.getElementsByTagName( "head" )[0].appendChild( link );
     </script>
-    <link rel="stylesheet" type="text/css" href="https://www.azadvertizer.net/azgovvizv4/css/azgovvizmain_004_048.css">
+    <link rel="stylesheet" type="text/css" href="https://www.azadvertizer.net/azgovvizv4/css/azgovvizmain_004_052.css">
     <script src="https://www.azadvertizer.net/azgovvizv4/js/jquery-3.6.0.min.js"></script>
     <script src="https://www.azadvertizer.net/azgovvizv4/js/jquery-ui-1.13.0.min.js"></script>
     <script type="text/javascript" src="https://www.azadvertizer.net/azgovvizv4/js/highlight_v004_002.js"></script>

pwsh/dev/functions/addHtParameters.ps1

@@ -38,6 +38,7 @@ function addHtParameters {
         NoStorageAccountAccessAnalysis               = [bool]$NoStorageAccountAccessAnalysis
         GitHubActionsOIDC                            = [bool]$GitHubActionsOIDC
         NoNetwork                                    = [bool]$NoNetwork
+        ThrottleLimit                                = $ThrottleLimit
     }
     Write-Host 'htParameters:'
     $azAPICallConf['htParameters'] | Format-Table -AutoSize | Out-String

pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1

@@ -437,14 +437,75 @@ function dataCollectionResources {
         $subscriptionQuotaId
     )
 
-    $currentTask = "Getting ResourceTypes for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
-    $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime&api-version=2021-04-01"
+    #region resources LIST
+    $currentTask = "Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']"
+    $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/resources?`$expand=createdTime,changedTime,properties&api-version=2021-04-01"
     $method = 'GET'
     $resourcesSubscriptionResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'
+    #Write-Host 'arm resList count:'$resourcesSubscriptionResult.Count
+    #endregion resources LIST
+
+    #region resources GET
+    if ($resourcesSubscriptionResult.Count -gt 0) {
+        $arrayResourcesWithProperties = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList))
+        $resourcesSubscriptionResult | ForEach-Object -Parallel {
+            $resource = $_
+
+            #region using
+            $arrayResourcesWithProperties = $using:arrayResourcesWithProperties
+            $htResourceProvidersRef = $using:htResourceProvidersRef
+            $arrayPrivateEndPointsFromResourceProperties = $using:arrayPrivateEndPointsFromResourceProperties
+            $scopeId = $using:scopeId
+            $scopeDisplayName = $using:scopeDisplayName
+            $ChildMgParentNameChainDelimited = $using:ChildMgParentNameChainDelimited
+            $azAPICallConf = $using:azAPICallConf
+            #endregion using
+
+            if ($htResourceProvidersRef.($resource.type)) {
+                $apiVersionToUse = $htResourceProvidersRef.($resource.type).APIFirst
+                $currentTask = "Getting Resource for PSRule API-version: '$apiVersionToUse'; ResourceType: '$($resource.type)'; ResourceId: '$($resource.id)'"
+                $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)$($resource.id)?api-version=$apiVersionToUse"
+                $method = 'GET'
+                $resourceResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection' -listenOn Content -unhandledErrorAction Continue
+
+                if ($resourceResult -ne 'ResourceOrResourcegroupNotFound') {
+                    $null = $script:arrayResourcesWithProperties.Add($resourceResult)
+
+                    if ($resourceResult.properties.privateEndpointConnections.Count -gt 0) {
+                        foreach ($privateEndpointConnection in $resourceResult.properties.privateEndpointConnections) {
+                            $resourceResultIdSplit = $resourceResult.id -split '/'
+                            $null = $script:arrayPrivateEndPointsFromResourceProperties.Add([PSCustomObject]@{
+                                    ResourceName              = $resourceResult.name
+                                    ResourceType              = $resourceResult.type
+                                    ResourceId                = $resourceResult.id
+                                    ResourceResourceGroup     = $resourceResultIdSplit[4]
+                                    ResourceSubscriptionId    = $scopeId
+                                    ResourceSubscriptionName  = $scopeDisplayName
+                                    ResourceMGPath            = $ChildMgParentNameChainDelimited
+                                    privateEndpointConnection = $privateEndpointConnection
+                                })
+                        }
+                    }
+
+                }
+            }
+            else {
+                Write-Host "Please report at AzGovViz Repo ... No API-version matches! ResourceType: '$($resource.type)'; ResourceId: '$($resource.id)'"
+            }
+
+        } -ThrottleLimit $azAPICallConf['htParameters'].ThrottleLimit
+    }
+    #Write-Host 'arm resGet count:' $arrayResourcesWithProperties.Count
+    #endregion resources GET
+
+    if ($resourcesSubscriptionResult.Count -ne $arrayResourcesWithProperties.Count) {
+        Write-Host " FYI: Getting Resources for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$subscriptionQuotaId']  - ARM list count: $($resourcesSubscriptionResult.Count); ARG get count: $($arrayResourcesWithProperties.Count)"
+    }
 
     #region PSRule
     if ($azAPICallConf['htParameters'].DoPSRule -eq $true) {
         if ($resourcesSubscriptionResult.Count -gt 0) {
+
             $startPSRule = Get-Date
             try {
                 <#
@@ -453,10 +514,10 @@ function dataCollectionResources {
                 Import-Module (Join-Path $path -ChildPath 'PSRule.Rules.Azure-nodeps.psd1')
                 #>
                 if ($azAPICallConf['htParameters'].PSRuleFailedOnly -eq $true) {
-                    $psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error
+                    $psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue -Outcome Fail, Error
                 }
                 else {
-                    $psruleResults = $resourcesSubscriptionResult | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue
+                    $psruleResults = $arrayResourcesWithProperties | Invoke-PSRule -Module psrule.rules.Azure -As Detail -Culture en-us -WarningAction Ignore -ErrorAction SilentlyContinue
                 }
             }
             catch {

pwsh/dev/functions/getOrphanedResources.ps1

@@ -84,8 +84,11 @@ function getOrphanedResources {
             $subscriptions = '"{0}"' -f ($batch.Group.subscriptionId -join '","')
             $body = @"
 {
-"query": "$($queryDetail.query)",
-"subscriptions": [$($subscriptions)]
+    "query": "$($queryDetail.query)",
+    "subscriptions": [$($subscriptions)],
+    "options": {
+        "`$top": 100
+    }
 }
 "@
 

pwsh/dev/functions/html/htmlFunctions.ps1

@@ -224,17 +224,17 @@ function processScopeInsights($mgChild, $mgChildOf) {
         }
 
         if ($mgName -eq $mgId) {
-            $mgNameAndOrId = "<b>$($mgName -replace '<', '&lt;' -replace '>', '&gt;')</b>"
+            $mgNameAndOrId = "$($mgName -replace '<', '&lt;' -replace '>', '&gt;')"
         }
         else {
-            $mgNameAndOrId = "<b>$($mgName -replace '<', '&lt;' -replace '>', '&gt;')</b> ($mgId)"
+            $mgNameAndOrId = "$($mgName -replace '<', '&lt;' -replace '>', '&gt;') ($mgId)"
         }
 
         $script:html += @"
 <button type="button" class="collapsible" id="table_$mgId">$levelSpacing<img class="imgMg $($classDefaultMG)" src="https://www.azadvertizer.net/azgovvizv4/icon/Icon-general-11-Management-Groups.svg"> <span class="valignMiddle">$mgNameAndOrId $subInfo</span></button>
 <div class="content">
 <table class="bottomrow">
-<tr><td class="detailstd"><p><a href="#hierarchy_$mgId"><i class="fa fa-eye" aria-hidden="true" style="color: #0078df"></i> <i>Highlight Management Group in HierarchyMap</i></a></p></td></tr>
+<tr><td class="detailstd"><a href="#hierarchy_$mgId"><i class="fa fa-eye" aria-hidden="true" style="color: #0078df"></i> <i><span>Highlight Management Group in HierarchyMap</span></i></a></td></tr>
 "@
         if ($mgId -eq $defaultManagementGroupId) {
             $script:html += @'
@@ -275,7 +275,7 @@ function processScopeInsightsMGSubs($mgChild) {
             $script:html += @"
     <tr>
         <td class="detailstd">
-            <button type="button" class="collapsible"><p><i class="fa fa-check-circle blue" aria-hidden="true"></i> $subscriptionLinkedCount Subscriptions linked $subscriptionsOutOfScopelinkedDetail</p></button>
+            <button type="button" class="collapsible"><i class="fa fa-check-circle blue" aria-hidden="true"></i> <span>$subscriptionLinkedCount Subscriptions linked $subscriptionsOutOfScopelinkedDetail</span></button>
             <div class="content"><!--collapsible-->
 "@
         }
@@ -284,7 +284,7 @@ function processScopeInsightsMGSubs($mgChild) {
             if ($subscriptionLinkedCount -gt 1) {
                 if (-not $NoScopeInsights) {
                     $script:html += @"
-                <button type="button" class="collapsible"> <img class="imgSub" src="https://www.azadvertizer.net/azgovvizv4/icon/Icon-general-2-Subscriptions.svg"> <span class="valignMiddle"><b>$($subEntry.subscription -replace '<', '&lt;' -replace '>', '&gt;')</b> ($($subEntry.subscriptionId))</span></button>
+                <button type="button" class="collapsible"> <img class="imgSub" src="https://www.azadvertizer.net/azgovvizv4/icon/Icon-general-2-Subscriptions.svg"> <span class="valignMiddle">$($subEntry.subscription -replace '<', '&lt;' -replace '>', '&gt;') ($($subEntry.subscriptionId))</span></button>
                 <div class="contentSub"><!--collapsiblePerSub-->
 "@
                 }
@@ -293,14 +293,14 @@ function processScopeInsightsMGSubs($mgChild) {
             else {
                 if (-not $NoScopeInsights) {
                     $script:html += @"
-                <img class="imgSub" src="https://www.azadvertizer.net/azgovvizv4/icon/Icon-general-2-Subscriptions.svg"> <span class="valignMiddle"><b>$($subEntry.subscription -replace '<', '&lt;' -replace '>', '&gt;')</b> ($($subEntry.subscriptionId))</span></button>
+                <img class="imgSub" src="https://www.azadvertizer.net/azgovvizv4/icon/Icon-general-2-Subscriptions.svg"> <span class="valignMiddle">$($subEntry.subscription -replace '<', '&lt;' -replace '>', '&gt;') ($($subEntry.subscriptionId))</span></button>
 "@
                 }
             }
             if (-not $NoScopeInsights) {
                 $script:html += @"
                 <table class="subTable">
-                <tr><td class="detailstd"><p><a href="#hierarchySub_$mgChild"><i class="fa fa-eye" aria-hidden="true" style="color: #0078df"></i> <i>Highlight Subscription in HierarchyMap</i></a></p></td></tr>
+                <tr><td class="detailstd"><a href="#hierarchySub_$mgChild"><i class="fa fa-eye" aria-hidden="true" style="color: #0078df"></i> <i><span>Highlight Subscription in HierarchyMap</span></i></a></td></tr>
 "@
             }
 
@@ -333,7 +333,7 @@ function processScopeInsightsMGSubs($mgChild) {
             $script:html += @"
     <tr>
         <td class="detailstd">
-            <p><i class="fa fa-ban" aria-hidden="true"></i> $subscriptionLinkedCount Subscriptions linked $subscriptionsOutOfScopelinkedDetail</p>
+            <i class="fa fa-ban" aria-hidden="true"></i> <span>$subscriptionLinkedCount Subscriptions linked $subscriptionsOutOfScopelinkedDetail</span>
 "@
         }
     }

pwsh/dev/functions/processDataCollection.ps1

@@ -355,6 +355,8 @@ function processDataCollection {
                 $htDefenderEmailContacts = $using:htDefenderEmailContacts
                 $arrayVNets = $using:arrayVNets
                 $arrayPrivateEndPoints = $using:arrayPrivateEndPoints
+                $htResourceProvidersRef = $using:htResourceProvidersRef
+                $arrayPrivateEndPointsFromResourceProperties = $using:arrayPrivateEndPointsFromResourceProperties
                 #other
                 $function:addRowToTable = $using:funcAddRowToTable
                 $function:namingValidation = $using:funcNamingValidation

pwsh/dev/functions/processNetwork.ps1

@@ -10,7 +10,6 @@ function processNetwork {
     $script:htSubnets = @{}
     $script:arrayVirtualNetworks = [System.Collections.ArrayList]@()
     $script:arraySubnets = [System.Collections.ArrayList]@()
-    $htUnknownTenantsForSubscription = @{}
 
     foreach ($vnet in $arrayVNets) {
 
@@ -77,9 +76,9 @@ function processNetwork {
                                     $peeringXTenant = 'true'
                                 }
                             }
-                            $htUnknownTenantsForSubscription.($remotesubscriptionId) = @{}
-                            $htUnknownTenantsForSubscription.($remotesubscriptionId).TenantId = $arrayRemoteMGPath -join ', '
-                            $remoteMGPath = $arrayRemoteMGPath -join ', '
+                            $script:htUnknownTenantsForSubscription.($remotesubscriptionId) = @{}
+                            $script:htUnknownTenantsForSubscription.($remotesubscriptionId).TenantId = $arrayRemoteMGPath -join ', '
+                            $remoteMGPath = $arrayRemoteMGPath -join ' or '
                         }
                     }
                 }