Skip to content

Commit

Permalink
lambda-promtail: cloudwatch: add '__aws_log_type' label (grafana#11335)
Browse files Browse the repository at this point in the history
**What this PR does / why we need it**:

Adds the `__aws_log_type` label to AWS CloudWatch logs in
`lambda-promtail`.

**Which issue(s) this PR fixes**:

N/A

**Special notes for your reviewer**:

AWS S3 & Kinesis log types already have this label. The
`lambda-promtail` documentation
[here](https://github.com/grafana/loki/blob/main/docs/sources/send-data/lambda-promtail/_index.md?plain=1#L154)
suggests that CloudWatch logs has this label added as well, but in
practice it does not AFAICT.

**Checklist**
- [x] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [x] Tests updated
- [ ] `CHANGELOG.md` updated
- [ ] If the change is worth mentioning in the release notes, add
`add-to-release-notes` label
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/setup/upgrade/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](grafana@d10549e)
- [ ] If the change is deprecating or removing a configuration option,
update the `deprecated-config.yaml` and `deleted-config.yaml` files
respectively in the `tools/deprecated-config-checker` directory.
[Example
PR](grafana@0d4416a)
  • Loading branch information
elliotdobson authored and rhnasc committed Apr 12, 2024
1 parent 18f4865 commit 368271c
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 0 deletions.
1 change: 1 addition & 0 deletions tools/lambda-promtail/lambda-promtail/cw.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ func parseCWEvent(ctx context.Context, b *batch, ev *events.CloudwatchLogsEvent)
}

labels := model.LabelSet{
model.LabelName("__aws_log_type"): model.LabelValue("cloudwatch"),
model.LabelName("__aws_cloudwatch_log_group"): model.LabelValue(data.LogGroup),
model.LabelName("__aws_cloudwatch_owner"): model.LabelValue(data.Owner),
}
Expand Down
60 changes: 60 additions & 0 deletions tools/lambda-promtail/lambda-promtail/cw_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package main

import (
"context"
"testing"

"github.com/aws/aws-lambda-go/events"
"github.com/stretchr/testify/require"

"github.com/grafana/loki/pkg/logproto"
)

func Test_parseCWEvent(t *testing.T) {
tests := []struct {
name string
b *batch
expectedStream string
keepStream bool
}{
{
name: "cloudwatch",
b: &batch{
streams: map[string]*logproto.Stream{},
},
expectedStream: `{__aws_cloudwatch_log_group="testLogGroup", __aws_cloudwatch_owner="123456789123", __aws_log_type="cloudwatch"}`,
keepStream: false,
},
{
name: "cloudwatch_keepStream",
b: &batch{
streams: map[string]*logproto.Stream{},
},
expectedStream: `{__aws_cloudwatch_log_group="testLogGroup", __aws_cloudwatch_log_stream="testLogStream", __aws_cloudwatch_owner="123456789123", __aws_log_type="cloudwatch"}`,
keepStream: true,
},
}

for _, tt := range tests {
// Docs: https://docs.aws.amazon.com/lambda/latest/dg/services-cloudwatchlogs.html
// Example CloudWatchLogEvent copied from https://github.com/aws/aws-lambda-go/blob/main/events/cloudwatch_logs_test.go
cwevent := &events.CloudwatchLogsEvent{
AWSLogs: events.CloudwatchLogsRawData{
Data: "H4sIAAAAAAAAAHWPwQqCQBCGX0Xm7EFtK+smZBEUgXoLCdMhFtKV3akI8d0bLYmibvPPN3wz00CJxmQnTO41whwWQRIctmEcB6sQbFC3CjW3XW8kxpOpP+OC22d1Wml1qZkQGtoMsScxaczKN3plG8zlaHIta5KqWsozoTYw3/djzwhpLwivWFGHGpAFe7DL68JlBUk+l7KSN7tCOEJ4M3/qOI49vMHj+zCKdlFqLaU2ZHV2a4Ct/an0/ivdX8oYc1UVX860fQDQiMdxRQEAAA==",
},
}

t.Run(tt.name, func(t *testing.T) {
batchSize = 131072 // Set large enough we don't send to promtail
keepStream = tt.keepStream
err := parseCWEvent(context.Background(), tt.b, cwevent)
if err != nil {
t.Error(err)
}
require.Len(t, tt.b.streams, 1)
stream, ok := tt.b.streams[tt.expectedStream]
require.True(t, ok, "batch does not contain stream: %s", tt.expectedStream)
require.NotNil(t, stream)
})
}
}

0 comments on commit 368271c

Please sign in to comment.