From 7c22189bf196c67676ceeb1d62bd3436ab1d28ed Mon Sep 17 00:00:00 2001
From: sushmith <6890568+bsushmith@users.noreply.github.com>
Date: Fri, 17 Nov 2023 14:11:17 +0530
Subject: [PATCH] perf(bigquery): fetch tables concurrently from bq (#112)

---
 plugins/providers/bigquery/provider.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/plugins/providers/bigquery/provider.go b/plugins/providers/bigquery/provider.go
index d1ffbd4d2..9f19df4d4 100644
--- a/plugins/providers/bigquery/provider.go
+++ b/plugins/providers/bigquery/provider.go
@@ -148,20 +148,24 @@ func (p *Provider) GetResources(ctx context.Context, pc *domain.ProviderConfig)
 			//setting fetchTables to true here so that even if we try to get only tables we can do that,
 			//previously it wasn't possible as fetchTables was getting set to true only from within the dataset code block below.
 			if containsString(resourceTypes, ResourceTypeDataset) {
-				mu.Lock()
 				fetchTables = false
-				defer mu.Unlock()
 				if datasetFilter != "" {
 					v, err := evaluator.Expression(datasetFilter).EvaluateWithStruct(dataset)
 					if err != nil {
 						p.logger.Error(ctx, fmt.Sprintf("evaluating filter expression %q for dataset %q: %v", datasetFilter, dataset.URN, err))
 					}
 					if !reflect.ValueOf(v).IsZero() {
+						// resources list is not concurrent safe since we are appending to it from multiple goroutines concurrently,
+						// so we need to lock it before appending
+						mu.Lock()
 						resources = append(resources, dataset)
+						mu.Unlock()
 						fetchTables = true
 					}
 				} else {
+					mu.Lock()
 					resources = append(resources, dataset)
+					mu.Unlock()
 					fetchTables = true
 				}
 			}
@@ -181,7 +185,9 @@ func (p *Provider) GetResources(ctx context.Context, pc *domain.ProviderConfig)
 				if containsString(resourceTypes, ResourceTypeDataset) {
 					dataset.Children = children
 				} else {
+					mu.Lock()
 					resources = append(resources, children...)
+					mu.Unlock()
 				}
 			}
 			return nil