diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json new file mode 100644 index 000000000000..7b19ed138f99 --- /dev/null +++ b/.github/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/loki-release.git", + "subdir": "workflows" + } + }, + "version": "v1.11.5" + } + ], + "legacyImports": true +} diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json new file mode 100644 index 000000000000..840c06de3d46 --- /dev/null +++ b/.github/jsonnetfile.lock.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/loki-release.git", + "subdir": "workflows" + } + }, + "version": "09374df9ca39fa58ec93d9e3fad4da1593186039", + "sum": "eMlN1tvu1jxKTdWvNfmXESn7JI+Wfu2C1wCabo7P2VQ=" + } + ], + "legacyImports": false +} diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet new file mode 100644 index 000000000000..37616b7ea56d --- /dev/null +++ b/.github/release-workflows.jsonnet @@ -0,0 +1,91 @@ +local lokiRelease = import 'workflows/main.jsonnet'; +local build = lokiRelease.build; +local job = lokiRelease.job; + +local releaseLibRef = std.filter( + function(dep) dep.source.git.remote == 'https://github.com/grafana/loki-release.git', + (import 'jsonnetfile.json').dependencies +)[0].version; + +local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef; + +local imageJobs = { + loki: build.image('loki', 'cmd/loki'), + fluentd: build.image('fluent-plugin-loki', 'clients/cmd/fluentd', platform=['linux/amd64']), + 'fluent-bit': build.image('fluent-bit-plugin-loki', 'clients/cmd/fluent-bit', platform=['linux/amd64']), + logstash: build.image('logstash-output-loki', 'clients/cmd/logstash', platform=['linux/amd64']), + logcli: build.image('logcli', 'cmd/logcli'), + 'loki-canary': build.image('loki-canary', 'cmd/loki-canary'), + 'loki-operator': build.image('loki-operator', 'operator', context='release/operator', platform=['linux/amd64']), + promtail: build.image('promtail', 'clients/cmd/promtail'), + querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']), +}; + +local buildImage = 'grafana/loki-build-image:0.30.1'; +local golangCiLintVersion = 'v1.51.2'; + +{ + 'patch-release-pr.yml': std.manifestYamlDoc( + lokiRelease.releasePRWorkflow( + imageJobs=imageJobs, + buildImage=buildImage, + branches=['release-[0-9]+.[0-9]+.x'], + checkTemplate=checkTemplate, + golangCiLintVersion=golangCiLintVersion, + imagePrefix='grafana', + releaseLibRef=releaseLibRef, + releaseRepo='grafana/loki', + skipArm=false, + skipValidation=false, + versioningStrategy='always-bump-patch', + useGitHubAppToken=true, + ), false, false + ), + 'minor-release-pr.yml': std.manifestYamlDoc( + lokiRelease.releasePRWorkflow( + imageJobs=imageJobs, + buildImage=buildImage, + branches=['k[0-9]+'], + checkTemplate=checkTemplate, + golangCiLintVersion=golangCiLintVersion, + imagePrefix='grafana', + releaseLibRef=releaseLibRef, + releaseRepo='grafana/loki', + skipArm=false, + skipValidation=false, + versioningStrategy='always-bump-minor', + useGitHubAppToken=true, + ), false, false + ), + 'release.yml': std.manifestYamlDoc( + lokiRelease.releaseWorkflow( + branches=['release-[0-9]+.[0-9]+.x', 'k[0-9]+'], + getDockerCredsFromVault=true, + imagePrefix='grafana', + releaseLibRef=releaseLibRef, + releaseRepo='grafana/loki', + useGitHubAppToken=false, + ), false, false + ), + 'check.yml': std.manifestYamlDoc({ + name: 'check', + on: { + pull_request: {}, + push: { + branches: ['main'], + }, + }, + jobs: { + check: { + uses: 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef, + with: { + build_image: buildImage, + golang_ci_lint_version: golangCiLintVersion, + release_lib_ref: releaseLibRef, + skip_validation: false, + use_github_app_token: true, + }, + }, + }, + }), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet new file mode 100644 index 000000000000..1165f1917ed8 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -0,0 +1,164 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; +local releaseStep = common.releaseStep; +local releaseLibStep = common.releaseLibStep; + +{ + image: function( + name, + path, + context='release', + platform=[ + 'linux/amd64', + 'linux/arm64', + 'linux/arm', + ] + ) + job.new() + + job.withStrategy({ + 'fail-fast': true, + matrix: { + platform: platform, + }, + }) + + job.withSteps([ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.setupNode, + common.googleAuth, + + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + + releaseStep('parse image platform') + + step.withId('platform') + + step.withRun(||| + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + |||), + + step.new('Build and export', 'docker/build-push-action@v5') + + step.withTimeoutMinutes(25) + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.with({ + context: context, + file: 'release/%s/Dockerfile' % path, + platforms: '${{ matrix.platform }}', + tags: '${{ env.IMAGE_PREFIX }}/%s:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}' % [name], + outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + }), + step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.with({ + path: 'release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + destination: 'loki-build-artifacts/${{ github.sha }}/images', //TODO: make bucket configurable + process_gcloudignore: false, + }), + ]), + + version: + job.new() + + job.withSteps([ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.setupNode, + common.extractBranchName, + common.githubAppToken, + common.setToken, + releaseLibStep('get release version') + + step.withId('version') + + step.withRun(||| + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --dry-run \ + --dry-run-output release.json \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --manifest-file .release-please-manifest.json \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + cat release.json + + if [[ `jq length release.json` -gt 1 ]]; then + echo 'release-please would create more than 1 PR, so cannot determine correct version' + echo "pr_created=false" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ `jq length release.json` -eq 0 ]]; then + echo "pr_created=false" >> $GITHUB_OUTPUT + else + version="$(npm run --silent get-version)" + echo "Parsed version: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + echo "pr_created=true" >> $GITHUB_OUTPUT + fi + |||), + ]) + + job.withOutputs({ + version: '${{ steps.version.outputs.version }}', + pr_created: '${{ steps.version.outputs.pr_created }}', + }), + + dist: function(buildImage, skipArm=true) + job.new() + + job.withSteps([ + common.fetchReleaseRepo, + common.googleAuth, + step.new('get nfpm signing keys', 'grafana/shared-workflows/actions/get-vault-secrets@main') + + step.withId('get-secrets') + + step.with({ + common_secrets: ||| + NFPM_SIGNING_KEY=packages-gpg:private-key + NFPM_PASSPHRASE=packages-gpg:passphrase + |||, + }), + + releaseStep('build artifacts') + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.withEnv({ + BUILD_IN_CONTAINER: false, + DRONE_TAG: '${{ needs.version.outputs.version }}', + IMAGE_TAG: '${{ needs.version.outputs.version }}', + NFPM_SIGNING_KEY_FILE: 'nfpm-private-key.key', + SKIP_ARM: skipArm, + }) + //TODO: the workdir here is loki specific + + step.withRun(||| + cat < $NFPM_SIGNING_KEY_FILE + make dist packages + EOF + ||| % buildImage), + + step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.with({ + path: 'release/dist', + destination: 'loki-build-artifacts/${{ github.sha }}', //TODO: make bucket configurable + process_gcloudignore: false, + }), + ]), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet new file mode 100644 index 000000000000..fcbc7d0db4f6 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet @@ -0,0 +1,145 @@ +{ + step: { + new: function(name, uses=null) { + name: name, + } + if uses != null then { + uses: uses, + } else {}, + with: function(with) { + with+: with, + }, + withRun: function(run) { + run: run, + }, + withId: function(id) { + id: id, + }, + withWorkingDirectory: function(workingDirectory) { + 'working-directory': workingDirectory, + }, + withIf: function(_if) { + 'if': _if, + }, + withEnv: function(env) { + env: env, + }, + withSecrets: function(env) { + secrets: env, + }, + withTimeoutMinutes: function(timeout) { + 'timeout-minutes': timeout, + }, + }, + job: { + new: function(runsOn='ubuntu-latest') { + 'runs-on': runsOn, + }, + with: function(with) { + with+: with, + }, + withUses: function(uses) { + uses: uses, + }, + withSteps: function(steps) { + steps: steps, + }, + withStrategy: function(strategy) { + strategy: strategy, + }, + withNeeds: function(needs) { + needs: needs, + }, + withIf: function(_if) { + 'if': _if, + }, + withOutputs: function(outputs) { + outputs: outputs, + }, + withContainer: function(container) { + container: container, + }, + withEnv: function(env) { + env: env, + }, + withSecrets: function(env) { + secrets: env, + }, + }, + + releaseStep: function(name, uses=null) $.step.new(name, uses) + + $.step.withWorkingDirectory('release'), + + releaseLibStep: function(name, uses=null) $.step.new(name, uses) + + $.step.withWorkingDirectory('lib'), + + checkout: + $.step.new('checkout', 'actions/checkout@v4'), + + fetchReleaseRepo: + $.step.new('pull code to release', 'actions/checkout@v4') + + $.step.with({ + repository: '${{ env.RELEASE_REPO }}', + path: 'release', + }), + fetchReleaseLib: + $.step.new('pull release library code', 'actions/checkout@v4') + + $.step.with({ + repository: 'grafana/loki-release', + path: 'lib', + ref: '${{ env.RELEASE_LIB_REF }}', + }), + + setupNode: $.step.new('setup node', 'actions/setup-node@v4') + + $.step.with({ + 'node-version': 20, + }), + + makeTarget: function(target) 'make %s' % target, + + alwaysGreen: { + steps: [ + $.step.new('always green') + + $.step.withRun('echo "always green"'), + ], + }, + + googleAuth: $.step.new('auth gcs', 'google-github-actions/auth@v2') + + $.step.with({ + credentials_json: '${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}', + }), + setupGoogleCloudSdk: $.step.new('Set up Cloud SDK', 'google-github-actions/setup-gcloud@v2') + + $.step.with({ + version: '>= 452.0.0', + }), + + extractBranchName: $.releaseStep('extract branch name') + + $.step.withId('extract_branch') + + $.step.withRun(||| + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + |||), + + fixDubiousOwnership: $.step.new('fix git dubious ownership') + + $.step.withRun(||| + git config --global --add safe.directory "$GITHUB_WORKSPACE" + |||), + + githubAppToken: $.step.new('get github app token', 'actions/github-app-token@v1') + + $.step.withId('get_github_app_token') + + $.step.withIf('${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}') + + $.step.with({ + 'app-id': '${{ secrets.APP_ID }}', + 'private-key': '${{ secrets.APP_PRIVATE_KEY }}', + // By setting owner, we should get access to all repositories in current owner's installation: https://github.com/marketplace/actions/create-github-app-token#create-a-token-for-all-repositories-in-the-current-owners-installation + owner: '${{ github.repository_owner }}', + }), + + setToken: $.step.new('set github token') + + $.step.withId('github_app_token') + + $.step.withRun(||| + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + |||), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet new file mode 100644 index 000000000000..0f676c63f1fb --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet @@ -0,0 +1,150 @@ +{ + common: import 'common.libsonnet', + job: $.common.job, + step: $.common.step, + build: import 'build.libsonnet', + release: import 'release.libsonnet', + validate: import 'validate.libsonnet', + releasePRWorkflow: function( + branches=['release-[0-9]+.[0-9]+.x', 'k[0-9]+'], + buildImage='grafana/loki-build-image:0.33.0', + checkTemplate='./.github/workflows/check.yml', + dockerUsername='grafana', + golangCiLintVersion='v1.55.1', + imageJobs={}, + imagePrefix='grafana', + releaseLibRef='main', + releaseRepo='grafana/loki-release', + skipArm=false, + skipValidation=false, + versioningStrategy='always-bump-patch', + useGitHubAppToken=true, + ) { + name: 'create release PR', + on: { + push: { + branches: branches, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'create-release-pr-${{ github.sha }}', + }, + env: { + RELEASE_REPO: releaseRepo, + DOCKER_USERNAME: dockerUsername, + IMAGE_PREFIX: imagePrefix, + SKIP_VALIDATION: skipValidation, + VERSIONING_STRATEGY: versioningStrategy, + RELEASE_LIB_REF: releaseLibRef, + USE_GITHUB_APP_TOKEN: useGitHubAppToken, + }, + local validationSteps = ['check'], + jobs: { + check: {} + $.job.withUses(checkTemplate) + + $.job.with({ + skip_validation: skipValidation, + build_image: buildImage, + golang_ci_lint_version: golangCiLintVersion, + release_lib_ref: releaseLibRef, + use_github_app_token: useGitHubAppToken, + }), + version: $.build.version + $.common.job.withNeeds(validationSteps), + dist: $.build.dist(buildImage, skipArm) + $.common.job.withNeeds(['version']), + } + std.mapWithKey(function(name, job) job + $.common.job.withNeeds(['version']), imageJobs) + { + local buildImageSteps = ['dist'] + std.objectFields(imageJobs), + 'create-release-pr': $.release.createReleasePR + $.common.job.withNeeds(buildImageSteps), + }, + }, + releaseWorkflow: function( + branches=['release-[0-9].[0-9].x', 'k[0-9]*'], + dockerUsername='grafana', + getDockerCredsFromVault=false, + imagePrefix='grafana', + releaseLibRef='main', + releaseRepo='grafana/loki-release', + useGitHubAppToken=true, + ) { + name: 'create release', + on: { + push: { + branches: branches, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'create-release-${{ github.sha }}', + }, + env: { + RELEASE_REPO: releaseRepo, + IMAGE_PREFIX: imagePrefix, + RELEASE_LIB_REF: releaseLibRef, + USE_GITHUB_APP_TOKEN: useGitHubAppToken, + }, + jobs: { + shouldRelease: $.release.shouldRelease, + createRelease: $.release.createRelease, + publishImages: $.release.publishImages(getDockerCredsFromVault, dockerUsername), + publishRelease: $.release.publishRelease, + }, + }, + check: { + name: 'check', + on: { + workflow_call: { + inputs: { + build_image: { + description: 'loki build image to use', + required: true, + type: 'string', + }, + skip_validation: { + default: false, + description: 'skip validation steps', + required: false, + type: 'boolean', + }, + golang_ci_lint_version: { + default: 'v1.55.1', + description: 'version of golangci-lint to use', + required: false, + type: 'string', + }, + release_lib_ref: { + default: 'main', + description: 'git ref of release library to use', + required: false, + type: 'string', + }, + use_github_app_token: { + default: true, + description: 'whether to use the GitHub App token for GH_TOKEN secret', + required: false, + type: 'boolean', + }, + }, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'check-${{ github.sha }}', + }, + env: { + RELEASE_LIB_REF: '${{ inputs.release_lib_ref }}', + USE_GITHUB_APP_TOKEN: '${{ inputs.use_github_app_token }}', + }, + jobs: $.validate, + }, +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet new file mode 100644 index 000000000000..de1cceb42453 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet @@ -0,0 +1,189 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; +local releaseStep = common.releaseStep; +local releaseLibStep = common.releaseLibStep; + +// DO NOT MODIFY THIS FOOTER TEMPLATE +// This template is matched by the should-release action to detect the correct +// sha to release and pull aritfacts from. If you need to change this, make sure +// to change it in both places. +//TODO: make bucket configurable +local pullRequestFooter = 'Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}'; + +{ + createReleasePR: + job.new() + + job.withSteps([ + common.fetchReleaseRepo, + common.fetchReleaseLib, + common.setupNode, + common.extractBranchName, + common.githubAppToken, + common.setToken, + + releaseLibStep('release please') + + step.withId('release') + + step.withEnv({ + SHA: '${{ github.sha }}', + }) + //TODO make bucket configurable + //TODO make a type/release in the backport action + //TODO backport action should not bring over autorelease: pending label + + step.withRun(||| + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --label "backport main,autorelease: pending,product-approved" \ + --manifest-file .release-please-manifest.json \ + --pull-request-footer "%s" \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + ||| % pullRequestFooter), + ]), + + shouldRelease: job.new() + + job.withSteps([ + common.fetchReleaseRepo, + common.fetchReleaseLib, + common.extractBranchName, + + step.new('should a release be created?', './lib/actions/should-release') + + step.withId('should_release') + + step.with({ + baseBranch: '${{ steps.extract_branch.outputs.branch }}', + }), + ]) + + job.withOutputs({ + shouldRelease: '${{ steps.should_release.outputs.shouldRelease }}', + sha: '${{ steps.should_release.outputs.sha }}', + name: '${{ steps.should_release.outputs.name }}', + branch: '${{ steps.extract_branch.outputs.branch }}', + + }), + createRelease: job.new() + + job.withNeeds(['shouldRelease']) + + job.withIf('${{ fromJSON(needs.shouldRelease.outputs.shouldRelease) }}') + + job.withSteps([ + common.fetchReleaseRepo, + common.fetchReleaseLib, + common.setupNode, + common.googleAuth, + common.setupGoogleCloudSdk, + common.githubAppToken, + common.setToken, + + // exits with code 1 if the url does not match + // meaning there are no artifacts for that sha + // we need to handle this if we're going to run this pipeline on every merge to main + releaseStep('download binaries') + + step.withRun(||| + echo "downloading binaries to $(pwd)/dist" + gsutil cp -r gs://loki-build-artifacts/${{ needs.shouldRelease.outputs.sha }}/dist . + |||), + + releaseStep('check if release exists') + + step.withId('check_release') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + set +e + isDraft="$(gh release view --json="isDraft" --jq=".isDraft" ${{ needs.shouldRelease.outputs.name }} 2>&1)" + set -e + if [[ "$isDraft" == "release not found" ]]; then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + + if [[ "$isDraft" == "true" ]]; then + echo "draft=true" >> $GITHUB_OUTPUT + fi + |||), + + releaseLibStep('create release') + + step.withId('release') + + step.withIf('${{ !fromJSON(steps.check_release.outputs.exists) }}') + + step.withRun(||| + npm install + npm exec -- release-please github-release \ + --draft \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --target-branch "${{ needs.shouldRelease.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" + |||), + + releaseStep('upload artifacts') + + step.withId('upload') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + gh release upload --clobber ${{ needs.shouldRelease.outputs.name }} dist/* + |||), + ]) + + job.withOutputs({ + sha: '${{ needs.shouldRelease.outputs.sha }}', + name: '${{ needs.shouldRelease.outputs.name }}', + draft: '${{ steps.check_release.outputs.draft }}', + exists: '${{ steps.check_release.outputs.exists }}', + }), + + publishImages: function(getDockerCredsFromVault=false, dockerUsername='grafanabot') + job.new() + + job.withNeeds(['createRelease']) + + job.withSteps( + [ + common.fetchReleaseLib, + common.googleAuth, + common.setupGoogleCloudSdk, + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + ] + (if getDockerCredsFromVault then [ + step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), + ] else [ + step.new('Login to DockerHub (from secrets)', 'docker/login-action@v3') + + step.with({ + username: dockerUsername, + password: '${{ secrets.DOCKER_PASSWORD }}', + }), + ]) + + [ + step.new('download images') + + step.withRun(||| + echo "downloading images to $(pwd)/images" + gsutil cp -r gs://loki-build-artifacts/${{ needs.createRelease.outputs.sha }}/images . + |||), + step.new('publish docker images', './lib/actions/push-images') + + step.with({ + imageDir: 'images', + imagePrefix: '${{ env.IMAGE_PREFIX }}', + }), + ] + ), + + publishRelease: job.new() + + job.withNeeds(['createRelease', 'publishImages']) + + job.withSteps([ + common.fetchReleaseRepo, + common.githubAppToken, + common.setToken, + releaseStep('publish release') + + step.withIf('${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + gh release edit ${{ needs.createRelease.outputs.name }} --draft=false + |||), + ]), +} diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet new file mode 100644 index 000000000000..17b22860d046 --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet @@ -0,0 +1,118 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; + +local setupValidationDeps = function(job) job { + steps: [ + common.checkout, + common.fetchReleaseLib, + common.fixDubiousOwnership, + step.new('install tar') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + apt update + apt install -qy tar xz-utils + |||), + step.new('install shellcheck', './lib/actions/install-binary') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + binary: 'shellcheck', + version: '0.9.0', + download_url: 'https://github.com/koalaman/shellcheck/releases/download/v${version}/shellcheck-v${version}.linux.x86_64.tar.xz', + tarball_binary_path: '*/${binary}', + smoke_test: '${binary} --version', + tar_args: 'xvf', + }), + step.new('install jsonnetfmt', './lib/actions/install-binary') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + binary: 'jsonnetfmt', + version: '0.18.0', + download_url: 'https://github.com/google/go-jsonnet/releases/download/v${version}/go-jsonnet_${version}_Linux_x86_64.tar.gz', + tarball_binary_path: '${binary}', + smoke_test: '${binary} --version', + }), + ] + job.steps, +}; + +local validationJob = job.new() + + job.withContainer({ + image: '${{ inputs.build_image }}', + }) + + job.withEnv({ + BUILD_IN_CONTAINER: false, + SKIP_VALIDATION: '${{ inputs.skip_validation }}', + }); + + +{ + local validationMakeStep = function(name, target) + step.new(name) + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(common.makeTarget(target)), + + test: setupValidationDeps( + validationJob + + job.withSteps([ + validationMakeStep('test', 'test'), + ]) + ), + + lint: setupValidationDeps( + validationJob + + job.withSteps( + [ + validationMakeStep('lint', 'lint'), + validationMakeStep('lint jsonnet', 'lint-jsonnet'), + validationMakeStep('lint scripts', 'lint-scripts'), + // step.new('format') + // + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + // + step.withRun(||| + // git fetch origin + // make check-format + // |||), + ] + [ + step.new('golangci-lint', 'golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + version: '${{ inputs.golang_ci_lint_version }}', + 'only-new-issues': true, + }), + ], + ) + ), + + check: setupValidationDeps( + validationJob + + job.withSteps([ + validationMakeStep('check generated files', 'check-generated-files'), + validationMakeStep('check mod', 'check-mod'), + validationMakeStep('check docs', 'check-doc'), + validationMakeStep('validate example configs', 'validate-example-configs'), + validationMakeStep('validate dev cluster config', 'validate-dev-cluster-config'), + validationMakeStep('check example config docs', 'check-example-config-doc'), + validationMakeStep('check helm reference doc', 'documentation-helm-reference-check'), + validationMakeStep('check drone drift', 'check-drone-drift'), + ]) + { + steps+: [ + step.new('build docs website') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + cat <> $GITHUB_OUTPUT + working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - env: + SHA: "${{ github.sha }}" + id: "release" + name: "release please" + run: | + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --label "backport main,autorelease: pending,product-approved" \ + --manifest-file .release-please-manifest.json \ + --pull-request-footer "Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + working-directory: "lib" + dist: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - id: "get-secrets" + name: "get nfpm signing keys" + uses: "grafana/shared-workflows/actions/get-vault-secrets@main" + with: + common_secrets: | + NFPM_SIGNING_KEY=packages-gpg:private-key + NFPM_PASSPHRASE=packages-gpg:passphrase + - env: + BUILD_IN_CONTAINER: false + DRONE_TAG: "${{ needs.version.outputs.version }}" + IMAGE_TAG: "${{ needs.version.outputs.version }}" + NFPM_SIGNING_KEY_FILE: "nfpm-private-key.key" + SKIP_ARM: false + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "build artifacts" + run: | + cat < $NFPM_SIGNING_KEY_FILE + make dist packages + EOF + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}" + path: "release/dist" + process_gcloudignore: false + fluent-bit: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluent-bit/Dockerfile" + outputs: "type=docker,dest=release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-bit-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + fluentd: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluentd/Dockerfile" + outputs: "type=docker,dest=release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + logcli: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/logcli/Dockerfile" + outputs: "type=docker,dest=release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logcli:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + logstash: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/logstash/Dockerfile" + outputs: "type=docker,dest=release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logstash-output-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + loki: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki/Dockerfile" + outputs: "type=docker,dest=release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-canary: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki-canary/Dockerfile" + outputs: "type=docker,dest=release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-canary:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-operator: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release/operator" + file: "release/operator/Dockerfile" + outputs: "type=docker,dest=release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-operator:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + promtail: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/promtail/Dockerfile" + outputs: "type=docker,dest=release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/promtail:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + querytee: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/querytee/Dockerfile" + outputs: "type=docker,dest=release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-query-tee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + version: + needs: + - "check" + outputs: + pr_created: "${{ steps.version.outputs.pr_created }}" + version: "${{ steps.version.outputs.version }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - id: "version" + name: "get release version" + run: | + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --dry-run \ + --dry-run-output release.json \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --manifest-file .release-please-manifest.json \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + cat release.json + + if [[ `jq length release.json` -gt 1 ]]; then + echo 'release-please would create more than 1 PR, so cannot determine correct version' + echo "pr_created=false" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ `jq length release.json` -eq 0 ]]; then + echo "pr_created=false" >> $GITHUB_OUTPUT + else + version="$(npm run --silent get-version)" + echo "Parsed version: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + echo "pr_created=true" >> $GITHUB_OUTPUT + fi + working-directory: "lib" +name: "create release PR" +"on": + push: + branches: + - "k[0-9]+" +permissions: + contents: "write" + id-token: "write" + pull-requests: "write" diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index eb99734aef43..1ad253e61b1c 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -1,21 +1,771 @@ ---- -name: 'create release PR for patch releases' -on: - push: - branches: - - 'release-[0-9].[0-9].x' - workflow_dispatch: {} -permissions: - contents: 'write' - issues: 'write' - pull-requests: 'write' +concurrency: + group: "create-release-pr-${{ github.sha }}" +env: + DOCKER_USERNAME: "grafana" + IMAGE_PREFIX: "grafana" + RELEASE_LIB_REF: "v1.11.5" + RELEASE_REPO: "grafana/loki" + SKIP_VALIDATION: false + USE_GITHUB_APP_TOKEN: true + VERSIONING_STRATEGY: "always-bump-patch" jobs: - create-release-pr: - uses: grafana/loki-release/.github/workflows/release-pr.yml@main + check: + uses: "grafana/loki-release/.github/workflows/check.yml@v1.11.5" with: - release_repo: grafana/loki + build_image: "grafana/loki-build-image:0.30.1" + golang_ci_lint_version: "v1.51.2" + release_lib_ref: "v1.11.5" skip_validation: false - versioning_strategy: always-bump-patch - secrets: - GCS_SERVICE_ACCOUNT_KEY: '${{ secrets.BACKEND_ENTERPRISE_DRONE }}' - GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' + use_github_app_token: true + create-release-pr: + needs: + - "dist" + - "fluent-bit" + - "fluentd" + - "logcli" + - "logstash" + - "loki" + - "loki-canary" + - "loki-operator" + - "promtail" + - "querytee" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - env: + SHA: "${{ github.sha }}" + id: "release" + name: "release please" + run: | + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --label "backport main,autorelease: pending,product-approved" \ + --manifest-file .release-please-manifest.json \ + --pull-request-footer "Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + working-directory: "lib" + dist: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - id: "get-secrets" + name: "get nfpm signing keys" + uses: "grafana/shared-workflows/actions/get-vault-secrets@main" + with: + common_secrets: | + NFPM_SIGNING_KEY=packages-gpg:private-key + NFPM_PASSPHRASE=packages-gpg:passphrase + - env: + BUILD_IN_CONTAINER: false + DRONE_TAG: "${{ needs.version.outputs.version }}" + IMAGE_TAG: "${{ needs.version.outputs.version }}" + NFPM_SIGNING_KEY_FILE: "nfpm-private-key.key" + SKIP_ARM: false + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "build artifacts" + run: | + cat < $NFPM_SIGNING_KEY_FILE + make dist packages + EOF + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}" + path: "release/dist" + process_gcloudignore: false + fluent-bit: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluent-bit/Dockerfile" + outputs: "type=docker,dest=release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-bit-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + fluentd: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/fluentd/Dockerfile" + outputs: "type=docker,dest=release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + logcli: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/logcli/Dockerfile" + outputs: "type=docker,dest=release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logcli:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + logstash: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/logstash/Dockerfile" + outputs: "type=docker,dest=release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/logstash-output-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + loki: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki/Dockerfile" + outputs: "type=docker,dest=release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-canary: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/loki-canary/Dockerfile" + outputs: "type=docker,dest=release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-canary:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + loki-operator: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release/operator" + file: "release/operator/Dockerfile" + outputs: "type=docker,dest=release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-operator:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + promtail: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/clients/cmd/promtail/Dockerfile" + outputs: "type=docker,dest=release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/promtail:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + - "linux/arm64" + - "linux/arm" + querytee: + needs: + - "version" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - id: "platform" + name: "parse image platform" + run: | + mkdir -p images + + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" + echo "platform=${platform}" >> $GITHUB_OUTPUT + echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT + working-directory: "release" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "Build and export" + timeout-minutes: 25 + uses: "docker/build-push-action@v5" + with: + context: "release" + file: "release/cmd/querytee/Dockerfile" + outputs: "type=docker,dest=release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + platforms: "${{ matrix.platform }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-query-tee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "upload artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "loki-build-artifacts/${{ github.sha }}/images" + path: "release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + process_gcloudignore: false + strategy: + fail-fast: true + matrix: + platform: + - "linux/amd64" + version: + needs: + - "check" + outputs: + pr_created: "${{ steps.version.outputs.pr_created }}" + version: "${{ steps.version.outputs.version }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - id: "version" + name: "get release version" + run: | + npm install + npm exec -- release-please release-pr \ + --consider-all-branches \ + --dry-run \ + --dry-run-output release.json \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --manifest-file .release-please-manifest.json \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + cat release.json + + if [[ `jq length release.json` -gt 1 ]]; then + echo 'release-please would create more than 1 PR, so cannot determine correct version' + echo "pr_created=false" >> $GITHUB_OUTPUT + exit 1 + fi + + if [[ `jq length release.json` -eq 0 ]]; then + echo "pr_created=false" >> $GITHUB_OUTPUT + else + version="$(npm run --silent get-version)" + echo "Parsed version: ${version}" + echo "version=${version}" >> $GITHUB_OUTPUT + echo "pr_created=true" >> $GITHUB_OUTPUT + fi + working-directory: "lib" +name: "create release PR" +"on": + push: + branches: + - "release-[0-9]+.[0-9]+.x" +permissions: + contents: "write" + id-token: "write" + pull-requests: "write" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ad73b085db0..9260cc8936d1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,19 +1,207 @@ ---- -name: 'create release' -on: +concurrency: + group: "create-release-${{ github.sha }}" +env: + IMAGE_PREFIX: "grafana" + RELEASE_LIB_REF: "v1.11.5" + RELEASE_REPO: "grafana/loki" + USE_GITHUB_APP_TOKEN: false +jobs: + createRelease: + if: "${{ fromJSON(needs.shouldRelease.outputs.shouldRelease) }}" + needs: + - "shouldRelease" + outputs: + draft: "${{ steps.check_release.outputs.draft }}" + exists: "${{ steps.check_release.outputs.exists }}" + name: "${{ needs.shouldRelease.outputs.name }}" + sha: "${{ needs.shouldRelease.outputs.sha }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "setup node" + uses: "actions/setup-node@v4" + with: + node-version: 20 + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - name: "download binaries" + run: | + echo "downloading binaries to $(pwd)/dist" + gsutil cp -r gs://loki-build-artifacts/${{ needs.shouldRelease.outputs.sha }}/dist . + working-directory: "release" + - env: + GH_TOKEN: "${{ steps.github_app_token.outputs.token }}" + id: "check_release" + name: "check if release exists" + run: | + set +e + isDraft="$(gh release view --json="isDraft" --jq=".isDraft" ${{ needs.shouldRelease.outputs.name }} 2>&1)" + set -e + if [[ "$isDraft" == "release not found" ]]; then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + + if [[ "$isDraft" == "true" ]]; then + echo "draft=true" >> $GITHUB_OUTPUT + fi + working-directory: "release" + - id: "release" + if: "${{ !fromJSON(steps.check_release.outputs.exists) }}" + name: "create release" + run: | + npm install + npm exec -- release-please github-release \ + --draft \ + --release-type simple \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --target-branch "${{ needs.shouldRelease.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" + working-directory: "lib" + - env: + GH_TOKEN: "${{ steps.github_app_token.outputs.token }}" + id: "upload" + name: "upload artifacts" + run: | + gh release upload --clobber ${{ needs.shouldRelease.outputs.name }} dist/* + working-directory: "release" + publishImages: + needs: + - "createRelease" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Login to DockerHub (from vault)" + uses: "grafana/shared-workflows/actions/dockerhub-login@main" + - name: "download images" + run: | + echo "downloading images to $(pwd)/images" + gsutil cp -r gs://loki-build-artifacts/${{ needs.createRelease.outputs.sha }}/images . + - name: "publish docker images" + uses: "./lib/actions/push-images" + with: + imageDir: "images" + imagePrefix: "${{ env.IMAGE_PREFIX }}" + publishRelease: + needs: + - "createRelease" + - "publishImages" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - env: + GH_TOKEN: "${{ steps.github_app_token.outputs.token }}" + if: "${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}" + name: "publish release" + run: | + gh release edit ${{ needs.createRelease.outputs.name }} --draft=false + working-directory: "release" + shouldRelease: + outputs: + branch: "${{ steps.extract_branch.outputs.branch }}" + name: "${{ steps.should_release.outputs.name }}" + sha: "${{ steps.should_release.outputs.sha }}" + shouldRelease: "${{ steps.should_release.outputs.shouldRelease }}" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - id: "extract_branch" + name: "extract branch name" + run: | + echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + working-directory: "release" + - id: "should_release" + name: "should a release be created?" + uses: "./lib/actions/should-release" + with: + baseBranch: "${{ steps.extract_branch.outputs.branch }}" +name: "create release" +"on": push: branches: - - 'release-[0-9].[0-9].x' - - 'k[0-9]*' - workflow_dispatch: {} + - "release-[0-9]+.[0-9]+.x" + - "k[0-9]+" permissions: - contents: write - pull-requests: write -jobs: - release: - uses: grafana/loki-release/.github/workflows/release.yml@main - with: - release_repo: grafana/loki - secrets: - GCS_SERVICE_ACCOUNT_KEY: '${{ secrets.BACKEND_ENTERPRISE_DRONE }}' - GH_TOKEN: '${{ secrets.GH_TOKEN }}' + contents: "write" + id-token: "write" + pull-requests: "write" diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 0e134950eab8..928eee2e123e 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,6 +1,3 @@ { - "cmd/loki": "2.9.4", - "cmd/loki-canary": "2.9.4", - "cmd/logcli": "2.9.4", - "clients/cmd/promtail": "2.9.4" + ".": "2.9.4" } diff --git a/Makefile b/Makefile index 928dea57aeb4..f7019ea0baf1 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ BUILD_IMAGE_VERSION := 0.30.1 # Docker image info IMAGE_PREFIX ?= grafana -IMAGE_TAG := $(shell ./tools/image-tag) +IMAGE_TAG ?= $(shell ./tools/image-tag) # Version info for binaries GIT_REVISION := $(shell git rev-parse --short HEAD) @@ -847,12 +847,16 @@ dev-k3d-down: .PHONY: trivy trivy: loki-image trivy i $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) + trivy fs go.mod # Synk is also used to scan for vulnerabilities, and detects things that trivy might miss .PHONY: snyk -snyk: loki-image - snyk container test $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) - snyk code test +snyk: + snyk test .PHONY: scan-vulnerabilities scan-vulnerabilities: trivy snyk + +.PHONY: release-workflows +release-workflows: + jsonnet -SJ .github/vendor -m .github/workflows .github/release-workflows.jsonnet