Skip to content

Commit

Permalink
fix: lambda-promtail, update s3 filename regex to allow finding of lo…
Browse files Browse the repository at this point in the history
…g files from AWS GovCloud regions (#12482)
  • Loading branch information
Overflow0xFFFF authored Apr 15, 2024
1 parent be03884 commit 7a81d26
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 2 deletions.
4 changes: 2 additions & 2 deletions tools/lambda-promtail/lambda-promtail/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ var (
// source: https://docs.aws.amazon.com/waf/latest/developerguide/logging-s3.html
// format: aws-waf-logs-suffix[/prefix]/AWSLogs/aws-account-id/WAFLogs/region/webacl-name/year/month/day/hour/minute/aws-account-id_waflogs_region_webacl-name_timestamp_hash.log.gz
// example: aws-waf-logs-test/AWSLogs/11111111111/WAFLogs/us-east-1/TEST-WEBACL/2021/10/28/19/50/11111111111_waflogs_us-east-1_TEST-WEBACL_20211028T1950Z_e0ca43b5.log.gz
defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:elasticloadbalancing|vpcflowlogs)\_\w+-\w+-\d_(?:(?P<lb_type>app|net)\.*?)?(?P<src>[a-zA-Z0-9\-]+)`)
defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?P<lb_type>app|net)\.*?)?(?P<src>[a-zA-Z0-9\-]+)`)
defaultTimestampRegex = regexp.MustCompile(`(?P<timestamp>\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+Z)?)`)
cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P<organization_id>o-[a-z0-9]{10,32})?\/?(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)\_\w+-\w+-\d_(?:(?:app|nlb|net)\.*?)?.+_(?P<src>[a-zA-Z0-9\-]+)`)
cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P<organization_id>o-[a-z0-9]{10,32})?\/?(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?:app|nlb|net)\.*?)?.+_(?P<src>[a-zA-Z0-9\-]+)`)
cloudfrontFilenameRegex = regexp.MustCompile(`(?P<prefix>.*)\/(?P<src>[A-Z0-9]+)\.(?P<year>\d+)-(?P<month>\d+)-(?P<day>\d+)-(.+)`)
cloudfrontTimestampRegex = regexp.MustCompile(`(?P<timestamp>\d+-\d+-\d+\s\d+:\d+:\d+)`)
wafFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>WAFLogs)\/(?P<region>[\w-]+)\/(?P<src>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/(?P<hour>\d+)\/(?P<minute>\d+)\/\d+\_waflogs\_[\w-]+_[\w-]+_\d+T\d+Z_\w+`)
Expand Down
101 changes: 101 additions & 0 deletions tools/lambda-promtail/lambda-promtail/s3_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,39 @@ func Test_getLabels(t *testing.T) {
},
wantErr: false,
},
{
name: "s3_govcloud_flow_logs",
args: args{
record: events.S3EventRecord{
AWSRegion: "us-gov-east-1",
S3: events.S3Entity{
Bucket: events.S3Bucket{
Name: "vpc_logs_test",
OwnerIdentity: events.S3UserIdentity{
PrincipalID: "test",
},
},
Object: events.S3Object{
Key: "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-gov-east-1/2022/01/24/123456789012_vpcflowlogs_us-gov-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz",
},
},
},
},
want: map[string]string{
"account_id": "123456789012",
"bucket": "vpc_logs_test",
"bucket_owner": "test",
"bucket_region": "us-gov-east-1",
"day": "24",
"key": "my-bucket/AWSLogs/123456789012/vpcflowlogs/us-gov-east-1/2022/01/24/123456789012_vpcflowlogs_us-gov-east-1_fl-1234abcd_20180620T1620Z_fe123456.log.gz",
"month": "01",
"region": "us-gov-east-1",
"src": "fl-1234abcd",
"type": FLOW_LOG_TYPE,
"year": "2022",
},
wantErr: false,
},
{
name: "cloudtrail_digest_logs",
args: args{
Expand Down Expand Up @@ -192,6 +225,39 @@ func Test_getLabels(t *testing.T) {
},
wantErr: false,
},
{
name: "cloudtrail_govcloud_logs",
args: args{
record: events.S3EventRecord{
AWSRegion: "us-gov-east-1",
S3: events.S3Entity{
Bucket: events.S3Bucket{
Name: "cloudtrail_logs_test",
OwnerIdentity: events.S3UserIdentity{
PrincipalID: "test",
},
},
Object: events.S3Object{
Key: "my-bucket/AWSLogs/123456789012/CloudTrail/us-gov-east-1/2022/01/24/123456789012_CloudTrail_us-gov-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz",
},
},
},
},
want: map[string]string{
"account_id": "123456789012",
"bucket": "cloudtrail_logs_test",
"bucket_owner": "test",
"bucket_region": "us-gov-east-1",
"day": "24",
"key": "my-bucket/AWSLogs/123456789012/CloudTrail/us-gov-east-1/2022/01/24/123456789012_CloudTrail_us-gov-east-1_20220124T0000Z_4jhzXFO2Jlvu2b3y.json.gz",
"month": "01",
"region": "us-gov-east-1",
"src": "4jhzXFO2Jlvu2b3y",
"type": CLOUDTRAIL_LOG_TYPE,
"year": "2022",
},
wantErr: false,
},
{
name: "organization_cloudtrail_logs",
args: args{
Expand Down Expand Up @@ -293,6 +359,41 @@ func Test_getLabels(t *testing.T) {
},
wantErr: false,
},
{
name: "s3_govcloud_waf",
args: args{
record: events.S3EventRecord{
AWSRegion: "us-gov-east-1",
S3: events.S3Entity{
Bucket: events.S3Bucket{
Name: "waf_logs_test",
OwnerIdentity: events.S3UserIdentity{
PrincipalID: "test",
},
},
Object: events.S3Object{
Key: "prefix/AWSLogs/11111111111/WAFLogs/us-gov-east-1/TEST-WEBACL/2021/10/28/19/50/11111111111_waflogs_us-gov-east-1_TEST-WEBACL_20211028T1950Z_e0ca43b5.log.gz",
},
},
},
},
want: map[string]string{
"account_id": "11111111111",
"bucket_owner": "test",
"bucket_region": "us-gov-east-1",
"bucket": "waf_logs_test",
"day": "28",
"hour": "19",
"key": "prefix/AWSLogs/11111111111/WAFLogs/us-gov-east-1/TEST-WEBACL/2021/10/28/19/50/11111111111_waflogs_us-gov-east-1_TEST-WEBACL_20211028T1950Z_e0ca43b5.log.gz",
"minute": "50",
"month": "10",
"region": "us-gov-east-1",
"src": "TEST-WEBACL",
"type": WAF_LOG_TYPE,
"year": "2021",
},
wantErr: false,
},
{
name: "missing_type",
args: args{
Expand Down

0 comments on commit 7a81d26

Please sign in to comment.