Merge branch 'main' into Jayclifford345-patch-2

grafana · Apr 24, 2024 · 197c43f · 197c43f
2 parents 1c2dcda + af09f53
commit 197c43f
Show file tree

Hide file tree

Showing 62 changed files with 1,492 additions and 339 deletions.
diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -139,7 +139,7 @@ steps:
   depends_on:
   - clone
   environment: {}
-  image: grafana/loki-build-image:0.33.2
+  image: grafana/loki-build-image:0.33.3
   name: documentation-helm-reference-check
 trigger:
   ref:
@@ -1085,7 +1085,7 @@ steps:
       from_secret: docker_password
     DOCKER_USERNAME:
       from_secret: docker_username
-  image: grafana/loki-build-image:0.33.2
+  image: grafana/loki-build-image:0.33.3
   name: build and push
   privileged: true
   volumes:
@@ -1308,6 +1308,6 @@ kind: secret
 name: gpg_private_key
 ---
 kind: signature
-hmac: 87480bff973003712122d81a1575e2a62cff6fd4a42b163487cae6c6a67d8e7c
+hmac: 33b9d2962b6dfcf1136ef7602d29e3f32f03b0d90dfd579652cbaf0a4ef2de4b
 
 ...
diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json
@@ -8,7 +8,7 @@
           "subdir": "workflows"
         }
       },
-      "version": "main"
+      "version": "124c4d996f9625478a79f1884465e29ea082d224"
     }
   ],
   "legacyImports": true

diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json
@@ -8,8 +8,8 @@
           "subdir": "workflows"
         }
       },
-      "version": "634945b73e8eed4f5161ec08810178ddeca7505b",
-      "sum": "BOnwSjzyOjWwv9ikwJSAgPBNnYHTU2PEDJ0PWY6nr7I="
+      "version": "124c4d996f9625478a79f1884465e29ea082d224",
+      "sum": "8wrJURq48ZBAtZcReO1W7AiXmvUyLqb932Q9sXyfFVo="
     }
   ],
   "legacyImports": false

diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet
@@ -1,10 +1,7 @@
 local lokiRelease = import 'workflows/main.jsonnet';
 local build = lokiRelease.build;
 
-local releaseLibRef = std.filter(
-  function(dep) dep.source.git.remote == 'https://github.com/grafana/loki-release.git',
-  (import 'jsonnetfile.json').dependencies
-)[0].version;
+local releaseLibRef = 'main';
 
 local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef;
 
@@ -20,7 +17,8 @@ local imageJobs = {
   querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']),
 };
 
-local buildImage = 'grafana/loki-build-image:0.33.1';
+local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION');
+local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion;
 local golangCiLintVersion = 'v1.55.1';
 
 local imageBuildTimeoutMin = 40;

diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -2,7 +2,7 @@
   "check":
     "uses": "grafana/loki-release/.github/workflows/check.yml@main"
     "with":
-      "build_image": "grafana/loki-build-image:0.33.1"
+      "build_image": "grafana/loki-build-image:0.33.3"
       "golang_ci_lint_version": "v1.55.1"
       "release_lib_ref": "main"
       "skip_validation": false
@@ -12,4 +12,4 @@
   "pull_request": {}
   "push":
     "branches":
-    - "main"
+    - "main"
diff --git a/.github/workflows/metrics-collector.yml b/.github/workflows/metrics-collector.yml
@@ -5,6 +5,7 @@ on:
 
 jobs:
   main:
+    if: github.owner == "grafana"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Actions

diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml
@@ -16,7 +16,7 @@ jobs:
   check:
     uses: "grafana/loki-release/.github/workflows/check.yml@main"
     with:
-      build_image: "grafana/loki-build-image:0.33.1"
+      build_image: "grafana/loki-build-image:0.33.3"
       golang_ci_lint_version: "v1.55.1"
       release_lib_ref: "main"
       skip_validation: false
@@ -58,7 +58,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -141,7 +141,7 @@ jobs:
           --env SKIP_ARM \
           --volume .:/src/loki \
           --workdir /src/loki \
-          --entrypoint /bin/sh "grafana/loki-build-image:0.33.1"
+          --entrypoint /bin/sh "grafana/loki-build-image:0.33.3"
           git config --global --add safe.directory /src/loki
           echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE
           make dist packages
@@ -753,7 +753,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -826,4 +826,4 @@ name: "Prepare Minor Release PR from Weekly"
 permissions:
   contents: "write"
   id-token: "write"
-  pull-requests: "write"
+  pull-requests: "write"
diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml
@@ -16,7 +16,7 @@ jobs:
   check:
     uses: "grafana/loki-release/.github/workflows/check.yml@main"
     with:
-      build_image: "grafana/loki-build-image:0.33.1"
+      build_image: "grafana/loki-build-image:0.33.3"
       golang_ci_lint_version: "v1.55.1"
       release_lib_ref: "main"
       skip_validation: false
@@ -58,7 +58,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -141,7 +141,7 @@ jobs:
           --env SKIP_ARM \
           --volume .:/src/loki \
           --workdir /src/loki \
-          --entrypoint /bin/sh "grafana/loki-build-image:0.33.1"
+          --entrypoint /bin/sh "grafana/loki-build-image:0.33.3"
           git config --global --add safe.directory /src/loki
           echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE
           make dist packages
@@ -753,7 +753,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -826,4 +826,4 @@ name: "Prepare Patch Release PR"
 permissions:
   contents: "write"
   id-token: "write"
-  pull-requests: "write"
+  pull-requests: "write"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -46,7 +46,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -158,7 +158,7 @@ jobs:
     - id: "get_github_app_token"
       if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}"
       name: "get github app token"
-      uses: "actions/github-app-token@v1"
+      uses: "actions/create-github-app-token@v1"
       with:
         app-id: "${{ secrets.APP_ID }}"
         owner: "${{ github.repository_owner }}"
@@ -219,4 +219,4 @@ name: "create release"
 permissions:
   contents: "write"
   id-token: "write"
-  pull-requests: "write"
+  pull-requests: "write"
diff --git a/.github/workflows/verify-release-workflow.yaml b/.github/workflows/verify-release-workflow.yaml
@@ -0,0 +1,18 @@
+name: Verify release workflow updates
+on: [pull_request]
+jobs:
+  check-release-changes:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: setup go
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.2'
+    - name: setup jsonnet
+      run: |
+        go install github.com/google/go-jsonnet/cmd/[email protected]
+        go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
+    - name: Check that the release workflows have been updated properly
+      run: |
+          make BUILD_IN_CONTAINER=false release-workflows-check
diff --git a/Makefile b/Makefile
@@ -36,8 +36,8 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
 # or you can override this with an environment variable
 BUILD_IN_CONTAINER ?= true
 
-# ensure you run `make drone` after changing this
-BUILD_IMAGE_VERSION ?= 0.33.2
+# ensure you run `make drone` and `make release-workflows` after changing this
+BUILD_IMAGE_VERSION ?= 0.33.3
 
 # Docker image info
 IMAGE_PREFIX ?= grafana
@@ -668,7 +668,8 @@ ifneq (,$(findstring WIP,$(IMAGE_TAG)))
 	false;
 endif
 	echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USERNAME} --password-stdin
-	$(SUDO) $(BUILD_OCI) -o type=registry -t $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) ./loki-build-image
+	$(SUDO) DOCKER_BUILDKIT=$(DOCKER_BUILDKIT) docker buildx build $(OCI_PLATFORMS) \
+		-o type=registry -t $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) ./loki-build-image
 
 # loki-operator
 loki-operator-image:
@@ -899,4 +900,10 @@ scan-vulnerabilities: trivy snyk
 .PHONY: release-workflows
 release-workflows:
 	pushd $(CURDIR)/.github && jb update && popd
-	jsonnet -SJ .github/vendor -m .github/workflows .github/release-workflows.jsonnet
+	jsonnet -SJ .github/vendor -m .github/workflows -V BUILD_IMAGE_VERSION=$(BUILD_IMAGE_VERSION) .github/release-workflows.jsonnet
+
+.PHONY: release-workflows-check
+release-workflows-check:
+	@$(MAKE) release-workflows
+	@echo "Checking diff"
+	@git diff --exit-code -- ".github/workflows/*release*" || (echo "Please build release workflows by running 'make release-workflows'" && false)
diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
@@ -16,6 +16,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/grafana/loki/v3/pkg/logcli/client"
+	"github.com/grafana/loki/v3/pkg/logcli/detected"
 	"github.com/grafana/loki/v3/pkg/logcli/index"
 	"github.com/grafana/loki/v3/pkg/logcli/labelquery"
 	"github.com/grafana/loki/v3/pkg/logcli/output"
@@ -253,6 +254,39 @@ Example:
 	   'my-query'
   `)
 	volumeRangeQuery = newVolumeQuery(true, volumeRangeCmd)
+
+	detectedFieldsCmd = app.Command("detected-fields", `Run a query for detected fields..
+
+The "detected-fields" command will return information about fields detected using either
+the "logfmt" or "json" parser against the log lines returned by the provided query for the
+provided time range.
+
+The "detected-fields" command will output extra information about the query
+and its results, such as the API URL, set of common labels, and set
+of excluded labels. This extra information can be suppressed with the
+--quiet flag.
+
+By default we look over the last hour of data; use --since to modify
+or provide specific start and end times with --from and --to respectively.
+
+Notice that when using --from and --to then ensure to use RFC3339Nano
+time format, but without timezone at the end. The local timezone will be added
+automatically or if using  --timezone flag.
+
+Example:
+
+	logcli detected-fields
+	   --timezone=UTC
+	   --from="2021-01-19T10:00:00Z"
+	   --to="2021-01-19T20:00:00Z"
+	   --output=jsonl
+	   'my-query'
+
+The output is limited to 100 fields by default; use --field-limit to increase.
+The query is limited to processing 1000 lines per subquery; use --line-limit to increase.
+`)
+
+	detectedFieldsQuery = newDetectedFieldsQuery(detectedFieldsCmd)
 )
 
 func main() {
@@ -388,6 +422,8 @@ func main() {
 		} else {
 			index.GetVolume(volumeQuery, queryClient, out, *statistics)
 		}
+	case detectedFieldsCmd.FullCommand():
+		detectedFieldsQuery.Do(queryClient, *outputMode)
 	}
 }
 
@@ -652,3 +688,41 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query {
 
 	return q
 }
+
+func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
+	// calculate query range from cli params
+	var from, to string
+	var since time.Duration
+
+	q := &detected.FieldsQuery{}
+
+	// executed after all command flags are parsed
+	cmd.Action(func(c *kingpin.ParseContext) error {
+		defaultEnd := time.Now()
+		defaultStart := defaultEnd.Add(-since)
+
+		q.Start = mustParse(from, defaultStart)
+		q.End = mustParse(to, defaultEnd)
+
+		q.Quiet = *quiet
+
+		return nil
+	})
+
+	cmd.Flag("field-limit", "Limit on number of fields to return.").
+		Default("100").
+		IntVar(&q.FieldLimit)
+	cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process.").
+		Default("1000").
+		IntVar(&q.LineLimit)
+	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").
+		Required().
+		StringVar(&q.QueryString)
+	cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
+	cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
+	cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
+	cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range.").
+		DurationVar(&q.Step)
+
+	return q
+}