diff --git a/component/loki/process/process.go b/component/loki/process/process.go index 0d1e9dd9b54d..8047b7052d0a 100644 --- a/component/loki/process/process.go +++ b/component/loki/process/process.go @@ -1,3 +1,6 @@ +// The code in this package is adapted/ported over from grafana/loki/clients/pkg/logentry. +// +// The last Loki commit scanned for upstream changes was 7d5475541c66a819f6f456a45f8c143a084e6831. package process import ( diff --git a/component/loki/process/stages/decolorize.go b/component/loki/process/stages/decolorize.go index e20864ba1761..26356ef1552c 100644 --- a/component/loki/process/stages/decolorize.go +++ b/component/loki/process/stages/decolorize.go @@ -1,7 +1,5 @@ package stages -// NOTE: This code is copied from Promtail (07cbef92268aecc0f20d1791a6df390c2df5c072) with changes kept to the minimum. - import ( "github.com/grafana/loki/pkg/logql/log" ) diff --git a/component/loki/process/stages/decolorize_test.go b/component/loki/process/stages/decolorize_test.go index 2df016885156..ad4669ecf090 100644 --- a/component/loki/process/stages/decolorize_test.go +++ b/component/loki/process/stages/decolorize_test.go @@ -1,7 +1,5 @@ package stages -// NOTE: This code is copied from Promtail (07cbef92268aecc0f20d1791a6df390c2df5c072) with changes kept to the minimum. - import ( "testing" "time" diff --git a/component/loki/process/stages/drop.go b/component/loki/process/stages/drop.go index e5e1b981c8b8..222885919bb2 100644 --- a/component/loki/process/stages/drop.go +++ b/component/loki/process/stages/drop.go @@ -1,8 +1,5 @@ package stages -// NOTE: This package is ported over from grafana/loki/clients/pkg/logentry/stages -// (07cbef92268aecc0f20d1791a6df390c2df5c072) with changes kept to the minimum. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/drop_test.go b/component/loki/process/stages/drop_test.go index 0dd182d45ffa..00f6169ad5e3 100644 --- a/component/loki/process/stages/drop_test.go +++ b/component/loki/process/stages/drop_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/extensions.go b/component/loki/process/stages/extensions.go index dc9cc5f95509..58e2eac36b8b 100644 --- a/component/loki/process/stages/extensions.go +++ b/component/loki/process/stages/extensions.go @@ -1,8 +1,5 @@ package stages -// NOTE: This package is ported over from grafana/loki/clients/pkg/logentry/stages -// (07cbef92268aecc0f20d1791a6df390c2df5c072) with changes kept to the minimum. - import ( "fmt" "strings" diff --git a/component/loki/process/stages/extensions_test.go b/component/loki/process/stages/extensions_test.go index 5cf71add03f2..02cdd572f596 100644 --- a/component/loki/process/stages/extensions_test.go +++ b/component/loki/process/stages/extensions_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "testing" "time" diff --git a/component/loki/process/stages/geoip.go b/component/loki/process/stages/geoip.go index 47c86a7b7a9b..0b4787bd0e2a 100644 --- a/component/loki/process/stages/geoip.go +++ b/component/loki/process/stages/geoip.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/inspector.go b/component/loki/process/stages/inspector.go index 55375685f810..105af419eae8 100644 --- a/component/loki/process/stages/inspector.go +++ b/component/loki/process/stages/inspector.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "fmt" "io" diff --git a/component/loki/process/stages/json.go b/component/loki/process/stages/json.go index 838a46b4e21a..b4aa5ae0bb1d 100644 --- a/component/loki/process/stages/json.go +++ b/component/loki/process/stages/json.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/json_test.go b/component/loki/process/stages/json_test.go index a930946b25b2..3cbdde3f16bd 100644 --- a/component/loki/process/stages/json_test.go +++ b/component/loki/process/stages/json_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/label_drop.go b/component/loki/process/stages/label_drop.go index 2fa9f15e2afb..ae117220cef5 100644 --- a/component/loki/process/stages/label_drop.go +++ b/component/loki/process/stages/label_drop.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "time" diff --git a/component/loki/process/stages/label_drop_test.go b/component/loki/process/stages/label_drop_test.go index f8d4689ce186..97e2ef7a8487 100644 --- a/component/loki/process/stages/label_drop_test.go +++ b/component/loki/process/stages/label_drop_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "testing" "time" diff --git a/component/loki/process/stages/label_keep.go b/component/loki/process/stages/label_keep.go index a0edbae51c13..c06baa31f03c 100644 --- a/component/loki/process/stages/label_keep.go +++ b/component/loki/process/stages/label_keep.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "time" diff --git a/component/loki/process/stages/label_keep_test.go b/component/loki/process/stages/label_keep_test.go index e67b4c21e628..74dd4c4c555b 100644 --- a/component/loki/process/stages/label_keep_test.go +++ b/component/loki/process/stages/label_keep_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "testing" "time" diff --git a/component/loki/process/stages/labels.go b/component/loki/process/stages/labels.go index b77b182315f6..7ccaedb8a1e4 100644 --- a/component/loki/process/stages/labels.go +++ b/component/loki/process/stages/labels.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/labels_test.go b/component/loki/process/stages/labels_test.go index d622843cb6e7..e86c837ea24b 100644 --- a/component/loki/process/stages/labels_test.go +++ b/component/loki/process/stages/labels_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "bytes" "errors" diff --git a/component/loki/process/stages/logfmt.go b/component/loki/process/stages/logfmt.go index 6f290516b7e5..a8129d9815c5 100644 --- a/component/loki/process/stages/logfmt.go +++ b/component/loki/process/stages/logfmt.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/logfmt_test.go b/component/loki/process/stages/logfmt_test.go index 4e4844da7e17..e32dc279a8e8 100644 --- a/component/loki/process/stages/logfmt_test.go +++ b/component/loki/process/stages/logfmt_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "testing" "time" diff --git a/component/loki/process/stages/match.go b/component/loki/process/stages/match.go index 436fba8e7d57..4b84bb99a90a 100644 --- a/component/loki/process/stages/match.go +++ b/component/loki/process/stages/match.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/match_test.go b/component/loki/process/stages/match_test.go index 10217b595cde..12bff8187735 100644 --- a/component/loki/process/stages/match_test.go +++ b/component/loki/process/stages/match_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "fmt" "testing" @@ -137,6 +133,7 @@ func TestMatcher(t *testing.T) { {`{foo="bar",bar!~"te.*"}`, map[string]string{"foo": "bar", "bar": "test"}, MatchActionDrop, false, false, false}, {`{foo=""}`, map[string]string{}, MatchActionKeep, false, true, false}, + {`{foo="bar"} |= "foo" | status >= 200`, map[string]string{"foo": "bar"}, MatchActionKeep, false, false, true}, } for _, tt := range tests { diff --git a/component/loki/process/stages/multiline.go b/component/loki/process/stages/multiline.go index 45a271aaf98a..9c2ab940e7dc 100644 --- a/component/loki/process/stages/multiline.go +++ b/component/loki/process/stages/multiline.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "bytes" "errors" @@ -81,7 +77,7 @@ type multilineState struct { currentLines uint64 // The number of lines of the current multiline block. } -// newMulitlineStage creates a MulitlineStage from config +// newMultilineStage creates a MulitlineStage from config func newMultilineStage(logger log.Logger, config MultilineConfig) (Stage, error) { err := validateMultilineConfig(&config) if err != nil { diff --git a/component/loki/process/stages/multiline_test.go b/component/loki/process/stages/multiline_test.go index e60354aa0ae0..d876df508d43 100644 --- a/component/loki/process/stages/multiline_test.go +++ b/component/loki/process/stages/multiline_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "sort" "sync" diff --git a/component/loki/process/stages/output.go b/component/loki/process/stages/output.go index efa2f2555cba..06b3a6152151 100644 --- a/component/loki/process/stages/output.go +++ b/component/loki/process/stages/output.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "reflect" diff --git a/component/loki/process/stages/output_test.go b/component/loki/process/stages/output_test.go index ef0816cf1853..5ec1dd251bf0 100644 --- a/component/loki/process/stages/output_test.go +++ b/component/loki/process/stages/output_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "bytes" "strings" diff --git a/component/loki/process/stages/pipeline.go b/component/loki/process/stages/pipeline.go index fb7d30cce637..194a57ccd0ac 100644 --- a/component/loki/process/stages/pipeline.go +++ b/component/loki/process/stages/pipeline.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "context" "fmt" diff --git a/component/loki/process/stages/pipeline_test.go b/component/loki/process/stages/pipeline_test.go index 8e3b8047b037..9d797c4c76b4 100644 --- a/component/loki/process/stages/pipeline_test.go +++ b/component/loki/process/stages/pipeline_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "fmt" "sync" diff --git a/component/loki/process/stages/regex.go b/component/loki/process/stages/regex.go index 3575e0d11980..cf1482d20ad9 100644 --- a/component/loki/process/stages/regex.go +++ b/component/loki/process/stages/regex.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/regex_test.go b/component/loki/process/stages/regex_test.go index 0513c4870a93..26a7ba4654e8 100644 --- a/component/loki/process/stages/regex_test.go +++ b/component/loki/process/stages/regex_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "bytes" "errors" diff --git a/component/loki/process/stages/replace.go b/component/loki/process/stages/replace.go index 027a04c41b94..cda06a020377 100644 --- a/component/loki/process/stages/replace.go +++ b/component/loki/process/stages/replace.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "bytes" "errors" diff --git a/component/loki/process/stages/replace_test.go b/component/loki/process/stages/replace_test.go index 9bb69c687478..aa189f06d0dd 100644 --- a/component/loki/process/stages/replace_test.go +++ b/component/loki/process/stages/replace_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/sampling.go b/component/loki/process/stages/sampling.go index ebc4f0c6d129..53bb6ad2e579 100644 --- a/component/loki/process/stages/sampling.go +++ b/component/loki/process/stages/sampling.go @@ -1,7 +1,5 @@ package stages -// NOTE: This code is copied from Promtail (07cbef92268aecc0f20d1791a6df390c2df5c072) with changes kept to the minimum. - import ( "fmt" "math" diff --git a/component/loki/process/stages/sampling_test.go b/component/loki/process/stages/sampling_test.go index 414a2e7ddea1..916125b01995 100644 --- a/component/loki/process/stages/sampling_test.go +++ b/component/loki/process/stages/sampling_test.go @@ -1,7 +1,5 @@ package stages -// NOTE: This code is copied from Promtail (07cbef92268aecc0f20d1791a6df390c2df5c072) with changes kept to the minimum. - import ( "fmt" "testing" diff --git a/component/loki/process/stages/stage.go b/component/loki/process/stages/stage.go index 1a27830ae735..055467c6cfaf 100644 --- a/component/loki/process/stages/stage.go +++ b/component/loki/process/stages/stage.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "fmt" "os" diff --git a/component/loki/process/stages/static_labels.go b/component/loki/process/stages/static_labels.go index 8ca9baeb27c0..4ccb22538de8 100644 --- a/component/loki/process/stages/static_labels.go +++ b/component/loki/process/stages/static_labels.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/static_labels_test.go b/component/loki/process/stages/static_labels_test.go index 0a41c163db3f..21759617b18c 100644 --- a/component/loki/process/stages/static_labels_test.go +++ b/component/loki/process/stages/static_labels_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "testing" "time" diff --git a/component/loki/process/stages/timestamp.go b/component/loki/process/stages/timestamp.go index 83ea240931bc..1e9ac10bc542 100644 --- a/component/loki/process/stages/timestamp.go +++ b/component/loki/process/stages/timestamp.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "errors" "fmt" diff --git a/component/loki/process/stages/timestamp_test.go b/component/loki/process/stages/timestamp_test.go index b9f59422e6cf..e36471548edf 100644 --- a/component/loki/process/stages/timestamp_test.go +++ b/component/loki/process/stages/timestamp_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "bytes" "fmt" diff --git a/component/loki/process/stages/util.go b/component/loki/process/stages/util.go index 72c00931c013..652c2e1edd51 100644 --- a/component/loki/process/stages/util.go +++ b/component/loki/process/stages/util.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "fmt" "math" diff --git a/component/loki/process/stages/util_test.go b/component/loki/process/stages/util_test.go index d8ef1560e0cb..721814c0c241 100644 --- a/component/loki/process/stages/util_test.go +++ b/component/loki/process/stages/util_test.go @@ -1,9 +1,5 @@ package stages -// This package is ported over from grafana/loki/clients/pkg/logentry/stages. -// We aim to port the stages in steps, to avoid introducing huge amounts of -// new code without being able to slowly review, examine and test them. - import ( "fmt" "testing" diff --git a/docs/sources/flow/reference/components/loki.process.md b/docs/sources/flow/reference/components/loki.process.md index 2dceb23c686c..f21104263026 100644 --- a/docs/sources/flow/reference/components/loki.process.md +++ b/docs/sources/flow/reference/components/loki.process.md @@ -561,11 +561,15 @@ The following arguments are supported: | Name | Type | Description | Default | Required | | --------------------- | -------- | ----------------------------------------------------------------------------------------------------- | --------------- | -------- | -| `selector` | `string` | The LogQL stream selector and filter expressions to use. | | yes | +| `selector` | `string` | The LogQL stream selector and line filter expressions to use. | | yes | | `pipeline_name` | `string` | A custom name to use for the nested pipeline. | `""` | no | | `action` | `string` | The action to take when the selector matches the log line. Supported values are `"keep"` and `"drop"` | `"keep"` | no | | `drop_counter_reason` | `string` | A custom reason to report for dropped lines. | `"match_stage"` | no | +{{% admonition type="note" %}} +The filters do not include label filter expressions such as `| label == "foobar"`. +{{% /admonition %}} + The `stage.match` block supports a number of `stage.*` inner blocks, like the top-level block. These are used to construct the nested set of stages to run if the selector matches the labels and content of the log entries. It supports all the @@ -1577,7 +1581,7 @@ The extracted data from the IP used in this example: - geoip_city_name: Kansas City - geoip_country_name: United States -- geoip_continet_name: North America +- geoip_continent_name: North America - geoip_continent_code: NA - geoip_location_latitude: 39.1027 - geoip_location_longitude: -94.5778