From c9a3ff53e4b60330c53fe8659d5632fd4d12d734 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 31 May 2023 18:49:24 +0200 Subject: [PATCH] Set unique timestamp in lambda-promtail. (#9560) --- tools/lambda-promtail/lambda-promtail/cloudtrail_test.go | 2 +- tools/lambda-promtail/lambda-promtail/cw.go | 3 ++- tools/lambda-promtail/lambda-promtail/json_stream.go | 2 +- tools/lambda-promtail/lambda-promtail/kinesis.go | 5 +++-- tools/lambda-promtail/lambda-promtail/kinesis_test.go | 3 ++- tools/lambda-promtail/lambda-promtail/main_test.go | 2 +- tools/lambda-promtail/lambda-promtail/promtail.go | 3 ++- tools/lambda-promtail/lambda-promtail/s3.go | 5 +++-- tools/lambda-promtail/lambda-promtail/s3_test.go | 3 ++- 9 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tools/lambda-promtail/lambda-promtail/cloudtrail_test.go b/tools/lambda-promtail/lambda-promtail/cloudtrail_test.go index e96c7b5bbac0..9ebf9a2d738c 100644 --- a/tools/lambda-promtail/lambda-promtail/cloudtrail_test.go +++ b/tools/lambda-promtail/lambda-promtail/cloudtrail_test.go @@ -13,7 +13,7 @@ func TestParseJson(t *testing.T) { if err != nil { t.Error(err) } - gzipReader,err := gzip.NewReader(file) + gzipReader, err := gzip.NewReader(file) if err != nil { t.Error(err) } diff --git a/tools/lambda-promtail/lambda-promtail/cw.go b/tools/lambda-promtail/lambda-promtail/cw.go index 8dba341b7370..4238e09051ee 100644 --- a/tools/lambda-promtail/lambda-promtail/cw.go +++ b/tools/lambda-promtail/lambda-promtail/cw.go @@ -6,8 +6,9 @@ import ( "time" "github.com/aws/aws-lambda-go/events" - "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" + + "github.com/grafana/loki/pkg/logproto" ) func parseCWEvent(ctx context.Context, b *batch, ev *events.CloudwatchLogsEvent) error { diff --git a/tools/lambda-promtail/lambda-promtail/json_stream.go b/tools/lambda-promtail/lambda-promtail/json_stream.go index 140ebd338dfb..bce105cf86fe 100644 --- a/tools/lambda-promtail/lambda-promtail/json_stream.go +++ b/tools/lambda-promtail/lambda-promtail/json_stream.go @@ -6,7 +6,7 @@ import ( "io" ) -// Stream helps transmit each recordss withing a channel. +// Stream helps transmit each recordss within a channel. type Stream struct { records chan Record } diff --git a/tools/lambda-promtail/lambda-promtail/kinesis.go b/tools/lambda-promtail/lambda-promtail/kinesis.go index 9b4b3fa31749..110af1ee150a 100644 --- a/tools/lambda-promtail/lambda-promtail/kinesis.go +++ b/tools/lambda-promtail/lambda-promtail/kinesis.go @@ -5,8 +5,9 @@ import ( "time" "github.com/aws/aws-lambda-go/events" - "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" + + "github.com/grafana/loki/pkg/logproto" ) func parseKinesisEvent(ctx context.Context, b batchIf, ev *events.KinesisEvent) error { @@ -15,7 +16,7 @@ func parseKinesisEvent(ctx context.Context, b batchIf, ev *events.KinesisEvent) } for _, record := range ev.Records { - timestamp := time.Unix(record.Kinesis.ApproximateArrivalTimestamp.Unix(),0) + timestamp := time.Unix(record.Kinesis.ApproximateArrivalTimestamp.Unix(), 0) labels := model.LabelSet{ model.LabelName("__aws_log_type"): model.LabelValue("kinesis"), diff --git a/tools/lambda-promtail/lambda-promtail/kinesis_test.go b/tools/lambda-promtail/lambda-promtail/kinesis_test.go index 4b342110ee76..927f357a4897 100644 --- a/tools/lambda-promtail/lambda-promtail/kinesis_test.go +++ b/tools/lambda-promtail/lambda-promtail/kinesis_test.go @@ -7,8 +7,9 @@ import ( "testing" "github.com/aws/aws-lambda-go/events" - "github.com/grafana/loki/pkg/logproto" "github.com/stretchr/testify/require" + + "github.com/grafana/loki/pkg/logproto" ) type MockBatch struct { diff --git a/tools/lambda-promtail/lambda-promtail/main_test.go b/tools/lambda-promtail/lambda-promtail/main_test.go index fbd47571f341..03cc81d48fb7 100644 --- a/tools/lambda-promtail/lambda-promtail/main_test.go +++ b/tools/lambda-promtail/lambda-promtail/main_test.go @@ -18,7 +18,7 @@ func TestLambdaPromtail_ExtraLabelsValid(t *testing.T) { } func TestLambdaPromtail_ExtraLabelsMissingValue(t *testing.T) { - extraLabels, err := parseExtraLabels("A,a,B,b,C,c,D",false) + extraLabels, err := parseExtraLabels("A,a,B,b,C,c,D", false) require.Nil(t, extraLabels) require.Errorf(t, err, invalidExtraLabelsError) } diff --git a/tools/lambda-promtail/lambda-promtail/promtail.go b/tools/lambda-promtail/lambda-promtail/promtail.go index 46fc3f9f0f1e..56020c30ea81 100644 --- a/tools/lambda-promtail/lambda-promtail/promtail.go +++ b/tools/lambda-promtail/lambda-promtail/promtail.go @@ -15,8 +15,9 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" "github.com/grafana/dskit/backoff" - "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" + + "github.com/grafana/loki/pkg/logproto" ) const ( diff --git a/tools/lambda-promtail/lambda-promtail/s3.go b/tools/lambda-promtail/lambda-promtail/s3.go index 60eb3a13b7cf..93f3d8e9d89c 100644 --- a/tools/lambda-promtail/lambda-promtail/s3.go +++ b/tools/lambda-promtail/lambda-promtail/s3.go @@ -12,9 +12,10 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/grafana/loki/pkg/logproto" "github.com/prometheus/common/model" + "github.com/grafana/loki/pkg/logproto" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -97,7 +98,6 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io. ls = applyExtraLabels(ls) - timestamp := time.Now() // extract the timestamp of the nested event and sends the rest as raw json if labels["type"] == CLOUDTRAIL_LOG_TYPE { records := make(chan Record) @@ -131,6 +131,7 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io. } match := timestampRegex.FindStringSubmatch(log_line) + timestamp := time.Now() if len(match) > 0 { timestamp, err = time.Parse(time.RFC3339, match[1]) if err != nil { diff --git a/tools/lambda-promtail/lambda-promtail/s3_test.go b/tools/lambda-promtail/lambda-promtail/s3_test.go index ad778c67574a..83f1161b6d95 100644 --- a/tools/lambda-promtail/lambda-promtail/s3_test.go +++ b/tools/lambda-promtail/lambda-promtail/s3_test.go @@ -8,8 +8,9 @@ import ( "testing" "github.com/aws/aws-lambda-go/events" - "github.com/grafana/loki/pkg/logproto" "github.com/stretchr/testify/require" + + "github.com/grafana/loki/pkg/logproto" ) func Test_getLabels(t *testing.T) {