Decoupled tags scanning from resource scanning (#253)

* refactor: removed tags from global and regional scanner system

* feat: added tags scanning

* feat: tags are fetched and combined

* Create gentle-paws-give.md

* Update gentle-paws-give.md

.changeset/gentle-paws-give.md

@@ -0,0 +1,6 @@
+---
+"@mirohq/cloud-data-import": minor
+---
+
+- Separated tags scanning from main resource discovery pipeline
+- Exposed new `getTagsScanner` function from the module

src/aws-app/main.ts

@@ -1,6 +1,6 @@
 import path from 'path'
 import {Logger} from './hooks/Logger'
-import {getAllAwsScanners} from '@/scanners'
+import {getAllAwsScanners, getTagsScanner} from '@/scanners'
 import {StandardOutputSchema, AwsScannerError} from '@/types'
 import {saveAsJson} from './utils/saveAsJson'
 import * as cliMessages from './cliMessages'
@@ -10,6 +10,7 @@ import {getConfig} from './config'
 import {createRateLimiterFactory} from './utils/createRateLimiterFactory'
 import {getAwsAccountId} from '@/scanners/scan-functions/aws/common/getAwsAccountId'
 import {AWSRateLimitExhaustionRetryStrategy} from './utils/AWSRateLimitExhaustionRetryStrategy'
+import {AwsServices} from '@/constants'
 
 export default async () => {
 	console.log(cliMessages.getIntro())
@@ -29,36 +30,49 @@ export default async () => {
 	const credentials = undefined
 
 	// prepare scanners
-	const scanners = getAllAwsScanners({
+	const hooks = [
+		new Logger(), // log scanning progress
+	]
+
+	const commonScannerOptions = {
 		credentials,
-		regions: config.regions,
 		getRateLimiter,
+		hooks,
+	}
+
+	const resourceScanners = getAllAwsScanners({
+		...commonScannerOptions,
+		regions: config.regions,
 		shouldIncludeGlobalServices: !config['regional-only'],
-		hooks: [
-			new Logger(), // log scanning progress
-		],
+	})
+
+	const scanResources = () => Promise.all(resourceScanners.map((scanner) => scanner()))
+	const scanTags = getTagsScanner({
+		...commonScannerOptions,
+		services: Object.values(AwsServices),
 	})
 
 	// run scanners
 	const startedAt = new Date()
-	const result = await Promise.all(scanners.map((scanner) => scanner()))
+	const [discoveredResources, discoveredTags] = await Promise.all([
+		scanResources(), // scan resources
+		scanTags(), // scan tags
+	])
 	const finishedAt = new Date()
 
 	// calculate duration
 	const duration = parseFloat(((finishedAt.getTime() - startedAt.getTime()) / 1000).toFixed(2))
 
 	// aggregate resources
-	const resources = result.reduce((acc, {resources}) => {
-		return {...acc, ...resources}
+	const resources = discoveredResources.reduce((acc, {results}) => {
+		return {...acc, ...results}
 	}, {})
 
-	// aggregate tags
-	const tags = result.reduce((acc, {tags}) => {
-		return {...acc, ...tags}
-	}, {})
+	// prepare tags
+	const tags = discoveredTags.results
 
 	// aggregate errors
-	const errors = result.reduce((acc, {errors}) => {
+	const errors = discoveredResources.reduce((acc, {errors}) => {
 		return [...acc, ...errors]
 	}, [] as AwsScannerError[])
 

src/constants.ts

@@ -34,42 +34,49 @@ export const awsRegionIds = [
 	'us-gov-west-1',
 ] as const
 
+/**
+ * When adding a new AWS service resource type to the AwsServices enum:
+ *
+ * 1. Refer to the AWS Resource Groups Tagging API documentation for the list of supported resource types.
+ * 2. Use the exact 'service-code:resource-type' string from the documentation.
+ * 3. Add the new service to the enum using a descriptive key.
+ */
 export enum AwsServices {
-	ATHENA_NAMED_QUERIES = 'athena/named-queries',
-	AUTOSCALING_GROUPS = 'autoscaling/groups',
-	CLOUDTRAIL_TRAILS = 'cloudtrail/trails',
-	CLOUDWATCH_METRIC_ALARMS = 'cloudwatch/metric-alarms',
-	CLOUDWATCH_METRIC_STREAMS = 'cloudwatch/metric-streams',
-	DYNAMODB_TABLES = 'dynamodb/tables',
-	EC2_INSTANCES = 'ec2/instances',
-	EC2_VPCS = 'ec2/vpcs',
-	EC2_VPC_ENDPOINTS = 'ec2/vpc-endpoints',
-	EC2_SUBNETS = 'ec2/subnets',
-	EC2_ROUTE_TABLES = 'ec2/route-tables',
-	EC2_INTERNET_GATEWAYS = 'ec2/internet-gateways',
-	EC2_NAT_GATEWAYS = 'ec2/nat-gateways',
-	EC2_TRANSIT_GATEWAYS = 'ec2/transit-gateways',
-	EC2_VOLUMES = 'ec2/volumes',
-	EC2_NETWORK_ACLS = 'ec2/network-acls',
-	EC2_VPN_GATEWAYS = 'ec2/vpn-gateways',
-	EC2_NETWORK_INTERFACES = 'ec2/network-interfaces',
-	ECS_CLUSTERS = 'ecs/clusters',
-	ECS_SERVICES = 'ecs/services',
-	ECS_TASKS = 'ecs/tasks',
-	EFS_FILE_SYSTEMS = 'efs/file-systems',
-	ELASTICACHE_CLUSTERS = 'elasticache/clusters',
-	ELBV2_LOAD_BALANCERS = 'elbv2/load-balancers',
-	ELBV2_TARGET_GROUPS = 'elbv2/target-groups',
-	ELBV1_LOAD_BALANCERS = 'elbv1/load-balancers',
-	EKS_CLUSTERS = 'eks/clusters',
-	LAMBDA_FUNCTIONS = 'lambda/functions',
-	REDSHIFT_CLUSTERS = 'redshift/clusters',
-	RDS_INSTANCES = 'rds/instances',
-	RDS_CLUSTERS = 'rds/clusters',
-	RDS_PROXIES = 'rds/proxies',
-	S3_BUCKETS = 's3/buckets',
-	SNS_TOPICS = 'sns/topics',
-	SQS_QUEUES = 'sqs/queues',
-	ROUTE53_HOSTED_ZONES = 'route53/hosted-zones',
-	CLOUDFRONT_DISTRIBUTIONS = 'cloudfront/distributions',
+	ATHENA_NAMED_QUERIES = 'athena:named-query',
+	AUTOSCALING_GROUPS = 'autoscaling:group',
+	CLOUDTRAIL_TRAILS = 'cloudtrail:trail',
+	CLOUDWATCH_METRIC_ALARMS = 'cloudwatch:metric-alarm',
+	CLOUDWATCH_METRIC_STREAMS = 'cloudwatch:metric-stream',
+	DYNAMODB_TABLES = 'dynamodb:table',
+	EC2_INSTANCES = 'ec2:instance',
+	EC2_VPCS = 'ec2:vpc',
+	EC2_VPC_ENDPOINTS = 'ec2:vpc-endpoint',
+	EC2_SUBNETS = 'ec2:subnet',
+	EC2_ROUTE_TABLES = 'ec2:route-table',
+	EC2_INTERNET_GATEWAYS = 'ec2:internet-gateway',
+	EC2_NAT_GATEWAYS = 'ec2:nat-gateway',
+	EC2_TRANSIT_GATEWAYS = 'ec2:transit-gateway',
+	EC2_VOLUMES = 'ec2:volume',
+	EC2_NETWORK_ACLS = 'ec2:network-acl',
+	EC2_VPN_GATEWAYS = 'ec2:vpn-gateway',
+	EC2_NETWORK_INTERFACES = 'ec2:network-interface',
+	ECS_CLUSTERS = 'ecs:cluster',
+	ECS_SERVICES = 'ecs:service',
+	ECS_TASKS = 'ecs:task',
+	EFS_FILE_SYSTEMS = 'efs:file-system',
+	ELASTICACHE_CLUSTERS = 'elasticache:cluster',
+	ELBV2_LOAD_BALANCERS = 'elasticloadbalancingv2:loadbalancer',
+	ELBV2_TARGET_GROUPS = 'elasticloadbalancingv2:targetgroup',
+	ELBV1_LOAD_BALANCERS = 'elbv1:load-balancer',
+	EKS_CLUSTERS = 'eks:cluster',
+	LAMBDA_FUNCTIONS = 'lambda:function',
+	REDSHIFT_CLUSTERS = 'redshift:cluster',
+	RDS_INSTANCES = 'rds:instance',
+	RDS_CLUSTERS = 'rds:cluster',
+	RDS_PROXIES = 'rds:proxy',
+	S3_BUCKETS = 's3:bucket',
+	SNS_TOPICS = 'sns:topic',
+	SQS_QUEUES = 'sqs:queue',
+	ROUTE53_HOSTED_ZONES = 'route53:hosted-zone',
+	CLOUDFRONT_DISTRIBUTIONS = 'cloudfront:distribution',
 }

src/lib/index.ts

@@ -1,7 +1,7 @@
 import {getAwsProcessedData} from '@/aws-app/process'
-import {createRateLimiter, getAllAwsScanners, getAwsScanner} from '@/scanners'
+import {createRateLimiter, getAllAwsScanners, getAwsScanner, getTagsScanner} from '@/scanners'
 
-export type {GetAwsScannerArguments, GetAllAwsScannersArguments} from '@/scanners'
+export type {GetAwsScannerArguments, GetAllAwsScannersArguments, GetAwsTagsScannerArguments} from '@/scanners'
 export type * from '@/types'
 
 export {awsRegionIds} from '@/constants'
@@ -50,6 +50,20 @@ export const experimental_getAllAwsScanners = getAllAwsScanners
  */
 export const experimental_getAwsScanner = getAwsScanner
 
+/**
+ * @public
+ * @experimental This export is experimental and may change or be removed in future versions.
+ * Use with caution.
+ * @remarks
+ * WARNING: This is an experimental API. It may undergo significant changes or be removed entirely in non-major version updates.
+ * Before using this in production, please consider the following:
+ * - This API is not covered by semantic versioning guarantees.
+ * - Breaking changes may occur in minor or patch releases.
+ * - Always check the changelog before updating, even for non-major versions.
+ * - If you depend on this feature, consider pinning your package version to avoid unexpected breaks.
+ */
+export const experimental_getTagsScanner = getTagsScanner
+
 /**
  * @public
  * @experimental This export is experimental and may change or be removed in future versions.

src/scanners/common/createGlobalScanner.ts

@@ -1,18 +1,9 @@
-import {
-	AwsGlobalScanFunction,
-	AwsCredentials,
-	AwsScannerLifecycleHook,
-	RateLimiter,
-	AwsTags,
-	AwsResources,
-} from '@/types'
+import {AwsGlobalScanFunction, AwsCredentials, AwsScannerLifecycleHook, RateLimiter, AwsResources} from '@/types'
 import {CreateGlobalScannerFunction, CreateScannerOptions, GetRateLimiterFunction} from '@/scanners/types'
-import {fetchTags} from '@/scanners/common/fetchTags'
 import {AwsServices} from '@/constants'
 
 type GlobalScanResult<T extends AwsServices> = {
 	resources: AwsResources<T>
-	tags: AwsTags
 	error: Error | null
 }
 
@@ -21,7 +12,6 @@ async function performGlobalScan<T extends AwsServices>(
 	scanFunction: AwsGlobalScanFunction<T>,
 	credentials: AwsCredentials,
 	rateLimiter: RateLimiter,
-	tagsRateLimiter: RateLimiter,
 	hooks: AwsScannerLifecycleHook[],
 ): Promise<GlobalScanResult<T>> {
 	try {
@@ -31,20 +21,17 @@ async function performGlobalScan<T extends AwsServices>(
 		// Perform scan
 		const resources = await scanFunction(credentials, rateLimiter)
 
-		// Fetch tags
-		const tags = await fetchTags(Object.keys(resources), credentials, tagsRateLimiter)
-
 		// onComplete hook
 		hooks.forEach((hook) => hook.onComplete?.(resources, service))
 
 		// Return resources
-		return {resources, tags, error: null}
+		return {resources, error: null}
 	} catch (error) {
 		// onError hook
 		hooks.forEach((hook) => hook.onError?.(error as Error, service))
 
 		// Return error
-		return {resources: {}, tags: {}, error: error as Error}
+		return {resources: {}, error: error as Error}
 	}
 }
 
@@ -54,23 +41,15 @@ export const createGlobalScanner: CreateGlobalScannerFunction = <T extends AwsSe
 	options: CreateScannerOptions,
 ) => {
 	return async () => {
-		const {credentials, getRateLimiter, tagsRateLimiter, hooks} = options
+		const {credentials, getRateLimiter, hooks} = options
 
 		// Perform global scan
 		const rateLimiter = getRateLimiter(service)
-		const {resources, tags, error} = await performGlobalScan(
-			service,
-			scanFunction,
-			credentials,
-			rateLimiter,
-			tagsRateLimiter,
-			hooks,
-		)
+		const {resources, error} = await performGlobalScan(service, scanFunction, credentials, rateLimiter, hooks)
 
 		// Return resources and errors
 		return {
-			resources,
-			tags,
+			results: resources,
 			errors: error ? [{service, message: error.message}] : [],
 		}
 	}

src/scanners/common/createRegionalScanner.ts

@@ -4,17 +4,14 @@ import {
 	AwsCredentials,
 	AwsScannerLifecycleHook,
 	RateLimiter,
-	AwsTags,
 	AwsResources,
 } from '@/types'
 import {CreateRegionalScannerFunction, CreateScannerOptions} from '@/scanners/types'
-import {fetchTags} from '@/scanners/common/fetchTags'
 import {AwsServices} from '@/constants'
 
 type RegionalScanResult<T extends AwsServices> = {
 	region: string
 	resources: AwsResources<T>
-	tags: AwsTags
 	error: Error | null
 }
 
@@ -24,7 +21,6 @@ async function scanRegion<T extends AwsServices>(
 	region: string,
 	credentials: AwsCredentials,
 	rateLimiter: RateLimiter,
-	tagsRateLimiter: RateLimiter,
 	hooks: AwsScannerLifecycleHook[],
 ): Promise<RegionalScanResult<T>> {
 	try {
@@ -34,20 +30,17 @@ async function scanRegion<T extends AwsServices>(
 		// Perform scan
 		const resources = await scanFunction(credentials, rateLimiter, region)
 
-		// Fetch tags
-		const tags = await fetchTags(Object.keys(resources), credentials, tagsRateLimiter)
-
 		// onComplete hook
 		hooks.forEach((hook) => hook.onComplete?.(resources, service, region))
 
 		// Return resources
-		return {region, resources, tags, error: null}
+		return {region, resources, error: null}
 	} catch (error) {
 		// onError hook
 		hooks.forEach((hook) => hook.onError?.(error as Error, service, region))
 
 		// Return error
-		return {region, resources: {}, tags: {}, error: error as Error}
+		return {region, resources: {}, error: error as Error}
 	}
 }
 
@@ -58,13 +51,13 @@ export const createRegionalScanner: CreateRegionalScannerFunction = <T extends A
 	options: CreateScannerOptions,
 ) => {
 	return async () => {
-		const {credentials, getRateLimiter, tagsRateLimiter, hooks} = options
+		const {credentials, getRateLimiter, hooks} = options
 
 		// Scan each region in parallel
 		const scanResults = await Promise.all(
 			regions.map((region) => {
 				const rateLimiter = getRateLimiter(service, region)
-				return scanRegion(service, scanFunction, region, credentials, rateLimiter, tagsRateLimiter, hooks)
+				return scanRegion(service, scanFunction, region, credentials, rateLimiter, hooks)
 			}),
 		)
 
@@ -76,14 +69,6 @@ export const createRegionalScanner: CreateRegionalScannerFunction = <T extends A
 			return acc
 		}, {} as AwsResources<T>)
 
-		// Aggregate resource tags
-		const tags = scanResults.reduce((acc, {tags}) => {
-			if (tags) {
-				Object.assign(acc, tags)
-			}
-			return acc
-		}, {} as AwsTags)
-
 		// Extract errors
 		const errors: AwsScannerError[] = scanResults
 			.map(({region, error}) => {
@@ -92,6 +77,6 @@ export const createRegionalScanner: CreateRegionalScannerFunction = <T extends A
 			.filter(Boolean) as AwsScannerError[]
 
 		// Return the combined resources and errors
-		return {resources, tags, errors}
+		return {results: resources, errors}
 	}
 }

src/scanners/index.ts

@@ -2,5 +2,5 @@ export {createRateLimiter} from './common/RateLimiter'
 export {createGlobalScanner} from './common/createGlobalScanner'
 export {createRegionalScanner} from './common/createRegionalScanner'
 
-export {getAllAwsScanners, getAwsScanner} from './scanner-factory'
-export type {GetAwsScannerArguments, GetAllAwsScannersArguments} from './types'
+export {getAllAwsScanners, getAwsScanner, getTagsScanner} from './scanner-factory'
+export type {GetAwsScannerArguments, GetAllAwsScannersArguments, GetAwsTagsScannerArguments} from './types'