Add support for tags (#227)

* Add tag support

* Add tag support

* Add tag support

* Review changes/suggestions

* Add credentials

* Add chunking to tag fetching

* Update rate limiter

* Remove imports

* Remove imports

* Fix tag fetching

* Changeset

* Changeset

.changeset/weak-actors-pretend.md

@@ -0,0 +1,5 @@
+---
+'@mirohq/cloud-data-import': minor
+---
+
+Add tag support

package.json

@@ -85,6 +85,7 @@
 		"@aws-sdk/client-rds": "^3.658.1",
 		"@aws-sdk/client-redshift": "3.654.0",
 		"@aws-sdk/client-resource-explorer-2": "^3.662.0",
+		"@aws-sdk/client-resource-groups-tagging-api": "3.669.0",
 		"@aws-sdk/client-route-53": "^3.624.0",
 		"@aws-sdk/client-s3": "^3.621.0",
 		"@aws-sdk/client-sns": "^3.621.0",

src/aws-app/main.ts

@@ -52,6 +52,11 @@ export default async () => {
 		return {...acc, ...resources}
 	}, {})
 
+	// aggregate tags
+	const tags = result.reduce((acc, {tags}) => {
+		return {...acc, ...tags}
+	}, {})
+
 	// aggregate errors
 	const errors = result.reduce((acc, {errors}) => {
 		return [...acc, ...errors]
@@ -62,7 +67,8 @@ export default async () => {
 		provider: 'aws',
 		docVersion: '0.1.0',
 		resources: config.raw ? resources : {},
-		processed: await getProcessedData(resources),
+		processed: await getProcessedData(resources, tags),
+		tags: tags,
 		errors,
 		metadata: {
 			account: await getAwsAccountId(credentials),

src/aws-app/process/getPlacementData.ts

@@ -27,6 +27,7 @@ export const getResourcePlacementData = (arnInfo: ARNInfo, resource: ResourceDes
 		availabilityZones: [],
 		subnets: [],
 		securityGroups: [],
+		tags: {},
 	}
 
 	// Athena Named Queries
@@ -319,9 +320,7 @@ export const getPlacementData = (resources: Resources): PlacementData => {
 			continue
 		}
 
-		const data = getResourcePlacementData(arnInfo, resource)
-
-		placementData[arn] = data
+		placementData[arn] = getResourcePlacementData(arnInfo, resource)
 	}
 
 	return placementData

src/aws-app/process/index.ts

@@ -1,27 +1,42 @@
-import type {ProcessedData, Resources} from '@/types'
+import type {ProcessedData, Resources, ResourceTags} from '@/types'
 
 import {getPlacementData} from './getPlacementData'
 import {getProcessedResources} from './resources'
 import {getProcessedContainers} from './containers'
 import {ProcessingErrorManager} from './utils/ProcessingErrorManager'
 
-export const getProcessedData = (resources: Resources): ProcessedData => {
+export const getProcessedData = (resources: Resources, resourceTags: ResourceTags): ProcessedData => {
 	const processingErrorsManager = new ProcessingErrorManager()
 
 	// Get the placement data which is a simplified version of the resources focusing on the location of the resources
 	const placementData = getPlacementData(resources)
 
 	// Calculate the containers, connections and resources
-	const processedResources = getProcessedResources(placementData)
+	const processedResources = getProcessedResources(placementData, resourceTags)
 	const containers = getProcessedContainers(placementData, resources, processingErrorsManager)
 
 	// Log all collected errors
 	processingErrorsManager.render()
 
+	// Tag values
+	let possibleTagValues: {[key: string]: string[]} = {}
+	for (const [_arn, tags] of Object.entries(resourceTags)) {
+		for (const [key, value] of Object.entries(tags)) {
+			if (!possibleTagValues[key]) {
+				possibleTagValues[key] = []
+			}
+
+			if (value && !possibleTagValues[key].includes(value)) {
+				possibleTagValues[key].push(value)
+			}
+		}
+	}
+
 	// Return the processed data
 	return {
 		resources: processedResources,
 		connections: [],
 		containers,
+		tags: possibleTagValues,
 	}
 }

src/aws-app/process/resources/processResources.ts

@@ -1,6 +1,7 @@
 import {PlacementData, ProcessedResources} from '../types'
+import {ResourceTags} from '@/types'
 
-export const getProcessedResources = (placementData: PlacementData): ProcessedResources => {
+export const getProcessedResources = (placementData: PlacementData, tags: ResourceTags): ProcessedResources => {
 	const processedResources = {} as ProcessedResources
 
 	for (const [arn, {name, service, type, variant}] of Object.entries(placementData)) {
@@ -9,6 +10,7 @@ export const getProcessedResources = (placementData: PlacementData): ProcessedRe
 		processedResources[arn] = {
 			name,
 			type: unifiedType,
+			tags: tags[arn] ?? {},
 		}
 	}
 

src/aws-app/process/types.ts

@@ -11,6 +11,7 @@ export type ResourcePlacementData = {
 	availabilityZones: string[]
 	subnets: string[]
 	securityGroups: string[]
+	tags: {[key: string]: number}
 }
 
 export type PlacementData = {

src/scanners/common/createGlobalScanner.ts

@@ -5,11 +5,14 @@ import {
 	Credentials,
 	ScannerLifecycleHook,
 	RateLimiter,
+	ResourceTags,
 } from '@/types'
 import {CreateGlobalScannerFunction, GetRateLimiterFunction} from '@/scanners/types'
+import {fetchTags} from '@/scanners/common/fetchTags'
 
 type GlobalScanResult<T extends ResourceDescription> = {
 	resources: Resources<T>
+	tags: ResourceTags
 	error: Error | null
 }
 
@@ -18,6 +21,7 @@ async function performGlobalScan<T extends ResourceDescription>(
 	scanFunction: GlobalScanFunction<T>,
 	credentials: Credentials,
 	rateLimiter: RateLimiter,
+	tagsRateLimiter: RateLimiter,
 	hooks: ScannerLifecycleHook[],
 ): Promise<GlobalScanResult<T>> {
 	try {
@@ -27,17 +31,20 @@ async function performGlobalScan<T extends ResourceDescription>(
 		// Perform scan
 		const resources = await scanFunction(credentials, rateLimiter)
 
+		// Fetch tags
+		const tags = await fetchTags(credentials, tagsRateLimiter)
+
 		// onComplete hook
 		hooks.forEach((hook) => hook.onComplete?.(resources, service))
 
 		// Return resources
-		return {resources, error: null}
+		return {resources, tags, error: null}
 	} catch (error) {
 		// onError hook
 		hooks.forEach((hook) => hook.onError?.(error as Error, service))
 
 		// Return error
-		return {resources: {} as Resources<never>, error: error as Error}
+		return {resources: {} as Resources<never>, tags: {}, error: error as Error}
 	}
 }
 
@@ -47,19 +54,28 @@ export const createGlobalScanner: CreateGlobalScannerFunction = <T extends Resou
 	options: {
 		credentials: Credentials
 		getRateLimiter: GetRateLimiterFunction
+		tagsRateLimiter: RateLimiter
 		hooks: ScannerLifecycleHook[]
 	},
 ) => {
 	return async () => {
-		const {credentials, getRateLimiter, hooks} = options
+		const {credentials, getRateLimiter, tagsRateLimiter, hooks} = options
 
 		// Perform global scan
 		const rateLimiter = getRateLimiter(service)
-		const {resources, error} = await performGlobalScan(service, scanFunction, credentials, rateLimiter, hooks)
+		const {resources, tags, error} = await performGlobalScan(
+			service,
+			scanFunction,
+			credentials,
+			rateLimiter,
+			tagsRateLimiter,
+			hooks,
+		)
 
 		// Return resources and errors
 		return {
 			resources,
+			tags,
 			errors: error ? [{service, message: error.message}] : [],
 		}
 	}

src/scanners/common/createRegionalScanner.ts

@@ -6,12 +6,15 @@ import {
 	Credentials,
 	ScannerLifecycleHook,
 	RateLimiter,
+	ResourceTags,
 } from '@/types'
 import {CreateRegionalScannerFunction, GetRateLimiterFunction} from '@/scanners/types'
+import {fetchTags} from '@/scanners/common/fetchTags'
 
 type RegionScanResult<T extends ResourceDescription> = {
 	region: string
 	resources: Resources<T> | null
+	tags?: ResourceTags
 	error: Error | null
 }
 
@@ -21,6 +24,7 @@ async function scanRegion<T extends ResourceDescription>(
 	region: string,
 	credentials: Credentials,
 	rateLimiter: RateLimiter,
+	tagsRateLimiter: RateLimiter,
 	hooks: ScannerLifecycleHook[],
 ): Promise<RegionScanResult<T>> {
 	try {
@@ -30,11 +34,14 @@ async function scanRegion<T extends ResourceDescription>(
 		// Perform scan
 		const resources = await scanFunction(credentials, rateLimiter, region)
 
+		// Fetch tags
+		const tags = await fetchTags(credentials, tagsRateLimiter)
+
 		// onComplete hook
 		hooks.forEach((hook) => hook.onComplete?.(resources, service, region))
 
 		// Return resources
-		return {region, resources, error: null}
+		return {region, resources, tags, error: null}
 	} catch (error) {
 		// onError hook
 		hooks.forEach((hook) => hook.onError?.(error as Error, service, region))
@@ -51,17 +58,18 @@ export const createRegionalScanner: CreateRegionalScannerFunction = <T extends R
 	options: {
 		credentials: Credentials
 		getRateLimiter: GetRateLimiterFunction
+		tagsRateLimiter: RateLimiter
 		hooks: ScannerLifecycleHook[]
 	},
 ) => {
 	return async () => {
-		const {credentials, getRateLimiter, hooks} = options
+		const {credentials, getRateLimiter, tagsRateLimiter, hooks} = options
 
 		// Scan each region in parallel
 		const scanResults = await Promise.all(
 			regions.map((region) => {
 				const rateLimiter = getRateLimiter(service, region)
-				return scanRegion(service, scanFunction, region, credentials, rateLimiter, hooks)
+				return scanRegion(service, scanFunction, region, credentials, rateLimiter, tagsRateLimiter, hooks)
 			}),
 		)
 
@@ -73,6 +81,14 @@ export const createRegionalScanner: CreateRegionalScannerFunction = <T extends R
 			return acc
 		}, {} as Resources<T>)
 
+		// Aggregate resource tags
+		const tags = scanResults.reduce((acc, {tags}) => {
+			if (tags) {
+				Object.assign(acc, tags)
+			}
+			return acc
+		}, {} as ResourceTags)
+
 		// Extract errors
 		const errors: ScannerError[] = scanResults
 			.map(({region, error}) => {
@@ -81,6 +97,6 @@ export const createRegionalScanner: CreateRegionalScannerFunction = <T extends R
 			.filter(Boolean) as ScannerError[]
 
 		// Return the combined resources and errors
-		return {resources, errors}
+		return {resources, tags, errors}
 	}
 }

src/scanners/common/fetchTags.ts

@@ -0,0 +1,38 @@
+import {Credentials, RateLimiter, ResourceTags} from '@/types'
+import {GetResourcesCommand, ResourceGroupsTaggingAPIClient} from '@aws-sdk/client-resource-groups-tagging-api'
+
+export const fetchTags = async (credentials: Credentials, rateLimiter: RateLimiter): Promise<ResourceTags> => {
+	const client = new ResourceGroupsTaggingAPIClient({credentials})
+	const tagResult: Record<string, Record<string, string | undefined>> = {}
+
+	try {
+		let paginationToken: string | undefined = undefined
+
+		do {
+			const command: GetResourcesCommand = new GetResourcesCommand({
+				PaginationToken: paginationToken,
+			})
+
+			const response = await rateLimiter.throttle(() => client.send(command))
+			paginationToken = response.PaginationToken
+
+			for (const resourceData of response.ResourceTagMappingList ?? []) {
+				for (const tagData of resourceData.Tags ?? []) {
+					if (!resourceData.ResourceARN || !tagData.Key) {
+						continue
+					}
+
+					if (!tagResult[resourceData.ResourceARN]) {
+						tagResult[resourceData.ResourceARN] = {}
+					}
+
+					tagResult[resourceData.ResourceARN][tagData.Key] = tagData.Value
+				}
+			}
+		} while (paginationToken)
+	} catch (error) {
+		console.error('Error fetching tag resources:', error)
+	}
+
+	return tagResult
+}

src/scanners/getAwsScanners.ts

@@ -53,9 +53,12 @@ export const getAwsScanners = ({
 	regions,
 	shouldIncludeGlobalServices,
 }: GetAwsScannersArguments): Scanner[] => {
+	const tagsRateLimiter = getRateLimiter('resource-groups-tagging')
+
 	const options = {
 		credentials,
 		getRateLimiter,
+		tagsRateLimiter,
 		hooks: hooks || [],
 	}
 

src/scanners/types.ts

@@ -16,6 +16,7 @@ interface CreateScannerOptions {
 	credentials: Credentials
 	hooks: ScannerLifecycleHook[]
 	getRateLimiter: GetRateLimiterFunction
+	tagsRateLimiter: RateLimiter
 }
 
 export type CreateRegionalScannerFunction = <T extends ResourceDescription>(

src/types.ts

@@ -74,6 +74,10 @@ export type Resources<T extends ResourceDescription = ResourceDescription> = {
 	[arn: string]: T
 }
 
+export type Tags = Record<string, string | undefined>
+
+export type ResourceTags = Record<string, Tags>
+
 export type Credentials = AwsCredentialIdentity | undefined
 
 export interface RateLimiter {
@@ -106,6 +110,7 @@ export type ScannerError = {
 
 export type ScannerResult<T extends ResourceDescription = ResourceDescription> = {
 	resources: Resources<T>
+	tags?: ResourceTags
 	errors: ScannerError[]
 }
 
@@ -174,6 +179,7 @@ export interface ProcessedData {
 		[arn: string]: {
 			name: string
 			type: string
+			tags: {[key: string]: string | undefined}
 		}
 	}
 	connections: {
@@ -189,13 +195,15 @@ export interface ProcessedData {
 		securityGroups: {[arn: string]: SecurityGroupContainer}
 		subnets: {[arn: string]: SubnetContainer}
 	}
+	tags: {[key: string]: string[]}
 }
 
 export interface StandardOutputSchema {
 	provider: 'aws'
 	docVersion: string
 	resources: Resources
 	processed?: ProcessedData
+	tags: ResourceTags
 	errors: ScannerError[]
 	metadata: {
 		account: string