Merge remote-tracking branch 'upstream/main' into deb-rpm-permissions-2

.github/ISSUE_TEMPLATE/component_release_template.md

@@ -33,7 +33,7 @@ If including changes in this release, increment the version on `{{RELEASE_VERSIO
 - [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/1).
 - [ ] All the tasks in this issue have been reviewed by the release owner.
 - [ ] Create, update, triage and label all features and issues targeted for this release with `v{{RELEASE_VERSION}}`.
-- [ ] Finalize the code and create the the release branch `{{RELEASE_VERSION}}` from the `{{RELEASE_VERSION_X}}` branch.
+- [ ] Finalize the code and create the the release branch `{{RELEASE_BRANCH}}` from the `{{RELEASE_VERSION_X}}` branch.
 
 ### CI/CD
 

.github/ISSUE_TEMPLATE/plugin_onboarding_template.yaml

@@ -1,6 +1,6 @@
 ---
-name: On-board plugins
-description: On-board plugins
+name: On-board plugins to the distribution
+description: On-board plugins to the distribution
 title: '[On-boarding]: '
 labels: [on-boarding, untriaged]
 body:
@@ -38,7 +38,7 @@ body:
       required: true
   - type: textarea
     attributes:
-      description: 'Did you go through the [on-boarding document](https://github.com/opensearch-project/opensearch-build/blob/main/ONBOARDING.md)'
+      description: 'Did you go through the [on-boarding document](https://github.com/opensearch-project/opensearch-build/blob/main/ONBOARDING.md#plugin-onboarding)'
       label: 'Did you read the on-boarding document'
     id: document
     validations:

.github/ISSUE_TEMPLATE/standalone_releases_template.yaml

@@ -10,7 +10,7 @@ body:
         Thanks for taking the time to submit release request!
   - type: textarea
     attributes:
-      description: 'Did you go through the [on-boarding document](https://github.com/opensearch-project/opensearch-build/blob/main/ONBOARDING.md#onboarding-to-universal--1-click-release-process)'
+      description: 'Did you go through the [on-boarding document](https://github.com/opensearch-project/opensearch-build/blob/main/ONBOARDING.md#standalone-component-onboarding)'
       label: 'Did you read the on-boarding document'
     id: document
     validations:

.github/workflows/automatic-merges.yml

@@ -0,0 +1,70 @@
+---
+name: automatic-merges
+
+on:
+  workflow_run:
+    # If new workflows triggered on pull request are added, they should be added to this list
+    workflows:
+      - Link Checker
+      - yaml-lint
+      - dockerfile-lint
+      - groovy-tests
+      - manifests
+      - python-tests
+    types: completed
+
+jobs:
+  automatic-merge-manifests:
+    if: github.repository == 'opensearch-project/opensearch-build'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: find-triggering-pr
+        uses: peternied/find-triggering-pr@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: peternied/discerning-merger@v3
+        if: steps.find-triggering-pr.outputs.pr-number != null
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          pull-request-number: ${{ steps.find-triggering-pr.outputs.pr-number }}
+          allowed-authors: |
+            dependabot
+            opensearch-trigger-bot
+            peterzhuamazon
+            bbarani
+            gaiksaya
+            rishabh6788
+            zelinh
+            jordarlu
+            prudhvigodithi
+            Divyaasm
+            dblock
+            tianleh
+            peternied
+          allowed-files: |
+            manifests/*/*.yml
+
+  automatic-merge-new-manifests:
+    if: github.repository == 'opensearch-project/opensearch-build'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: find-triggering-pr
+        uses: peternied/find-triggering-pr@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: peternied/discerning-merger@v3
+        if: steps.find-triggering-pr.outputs.pr-number != null
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          pull-request-number: ${{ steps.find-triggering-pr.outputs.pr-number }}
+          allowed-authors: |
+            opensearch-ci-bot
+          allowed-files: |
+            manifests/*/*.yml
+            jenkins/check-for-build.jenkinsfile

.github/workflows/get-ci-image-tag.yml

@@ -0,0 +1,62 @@
+---
+name: Get CI Image Tag
+on:
+  workflow_call:
+    inputs:
+      product:
+        required: false
+        type: string
+        default: 'opensearch'
+      platform:
+        required: false
+        type: string
+      build_ref:
+        required: false
+        type: string
+        default: 'main'
+      version:
+        required: false
+        type: string
+      type:
+        required: false
+        type: string
+        default: 'build'
+    outputs:
+      ci-image-version-linux:
+        description: The ci image version for linux build
+        value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }}
+
+jobs:
+  Get-CI-Image-Tag:
+    runs-on: ubuntu-latest
+    outputs:
+      output-ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }}
+    steps:
+      - name: Install crane
+        uses: iarekylew00t/crane-installer@v1
+        with:
+          crane-release: v0.15.2
+      - name: Checkout opensearch-build repository
+        uses: actions/checkout@v3
+        with:
+          repository: 'opensearch-project/opensearch-build'
+          ref: ${{ inputs.build_ref }}
+          path: 'opensearch-build'
+      - name: Get ci image version from opensearch-build repository scripts
+        id: step-ci-image-version-linux
+        run: |
+          PRODUCT=${{ inputs.product }}
+          PLATFORM=${{ inputs.platform }}
+          TYPE=${{ inputs.type }}
+          if [[ -z "$PLATFORM" ]]; then
+              if [[ "$PRODUCT" = "opensearch" ]]; then
+                  PLATFORM="centos7" # Temp measure before centos7 deprecation on opensearch for k-NN
+              else
+                  PLATFORM="rockylinux8"
+              fi
+          fi
+          crane version
+          echo $PRODUCT $PLATFORM
+          CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p $PLATFORM -u $PRODUCT -t $TYPE | head -1`
+          echo $CI_IMAGE_VERSION
+          echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT

.github/workflows/license-header-checker.yml

@@ -9,4 +9,4 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Add License Header
-        uses: kt3k/[email protected]
+        run: npx @kt3k/license-checker

.github/workflows/manifests.yml

@@ -2,75 +2,53 @@
 name: manifests
 
 on:
-  push:
   pull_request:
-    paths:
-      - 'manifests/**/*.yml'
-      - '!manifests/templates/**/'
-  schedule:
-    - cron: 0 0 * * *
+  paths:
+    - 'manifests/**/*.yml'
+    - '!manifests/templates/**/'
+    - 'legacy-manifests/**/*.yml'
 
 jobs:
-  list-manifests11:
+  list-changed-manifests:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v3
-      - id: set-matrix
-        run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | awk -F/ '{if($2<2)print$0}' | jq -R -s -c 'split("\n")[:-1]')"
-
-  list-manifests17:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@v3
-      - id: set-matrix
-        run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | awk -F/ '{if($2>2)print$0}' | jq -R -s -c 'split("\n")[:-1]')"
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Get changed manifest files
+        uses: tj-actions/changed-files@v41
+        id: list-changed-manifests
+        with:
+          files: manifests/**/opensearch*.yml
+          json: true
+          quotepath: false
+          dir_names: false
 
+      - name: Set unique changed manifests as matrix
+        id: set-matrix
+        run: echo "matrix={\"manifest\":${{ steps.list-changed-manifests.outputs.all_changed_files }}}" >> "$GITHUB_OUTPUT"
 
-  manifest-checks-jdk11:
-    needs: list-manifests11
+  manifest-checks:
+    needs: [list-changed-manifests]
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: 3.9
-      JDK_VERSION: 11
     strategy:
-      matrix:
-        manifest: ${{ fromJson(needs.list-manifests11.outputs.matrix) }}
+      fail-fast: false
+      matrix: ${{ fromJson(needs.list-changed-manifests.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
-      - name: Set Up JDK ${{ env.JDK_VERSION }}
-        uses: actions/setup-java@v1
-        with:
-          java-version: ${{ env.JDK_VERSION }}
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Install Pipenv and Dependencies
+      - name: Check JDK Version
         run: |
-          python -m pip install --upgrade pipenv wheel
-      - name: OpenSearch Manifests
-        run: |-
-          ./ci.sh ${{ matrix.manifest }} --snapshot
-
-  manifest-checks-jdk17:
-    needs: list-manifests17
-    runs-on: ubuntu-latest
-    env:
-      PYTHON_VERSION: 3.9
-      JDK_VERSION: 17
-    strategy:
-      matrix:
-        manifest: ${{ fromJson(needs.list-manifests17.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set Up JDK ${{ env.JDK_VERSION }}
+          java_version=`cat ${{ matrix.manifest }} | yq -r .ci.image.args | grep -Eo '[0-9]+' || echo ''`
+          echo $java_version
+          echo "JAVA_VERSION=$java_version" >> "$GITHUB_ENV"
+      - name: Set Up JDK ${{ env.JAVA_VERSION }}
+        if: ${{ env.JAVA_VERSION }}
         uses: actions/setup-java@v1
         with:
-          java-version: ${{ env.JDK_VERSION }}
+          java-version: ${{ env.JAVA_VERSION }}
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v3
         with:

.github/workflows/os-increment-plugin-versions.yml

@@ -45,19 +45,22 @@ jobs:
           - {repo: security}
           - {repo: security-analytics}
           - {repo: sql}
+          - {repo: custom-codecs}
+          - {repo: flow-framework}
+          - {repo: skills}
         branch:
+          - 1.x
           - '1.3'
           - 2.x
-          - '2.8'
           - main
-          - '2.9'
-          - '2.10'
-          - '2.11'
+          - '2.12'
+          - '2.13'
         exclude:
           - {entry: {repo: geospatial}, branch: '1.3'}
           - {entry: {repo: neural-search}, branch: '1.3'}
           - {entry: {repo: security-analytics}, branch: '1.3'}
           - {entry: {repo: notifications, path: notifications}, branch: '1.3'}
+          - {entry: {repo: custom-codecs}, branch: '1.3'}
     steps:
       - name: Check out OpenSearch repo
         uses: actions/checkout@v3

.github/workflows/os-release-issues.yml

@@ -57,6 +57,9 @@ jobs:
           - {repo: security}
           - {repo: security-analytics}
           - {repo: sql}
+          - {repo: custom-codecs}
+          - {repo: flow-framework}
+          - {repo: skills}
         release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }}
     steps:
       - name: GitHub App token
@@ -97,14 +100,15 @@ jobs:
           # Read the file contents and replace the placeholders
           file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md"
           RELEASE_VERSION="${{ matrix.release_version }}"
+          RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2)
           BUILD_REPO_ISSUE_OUTPUT=$(cat <<EOF
           ${{ steps.check_if_build_repo_issue_exists.outputs.issues }}
           EOF
           )
           RELEASE_ISSUE_NUMBER=$(echo $BUILD_REPO_ISSUE_OUTPUT | jq -r '.[0].number')
           RELEASE_ISSUE="https://github.com/opensearch-project/opensearch-build/issues/${RELEASE_ISSUE_NUMBER}"
           RELEASE_VERSION_X=$(echo "${{ matrix.release_version }}" | awk -F'.' '{print $1}').x
-          sed -e "s|{{RELEASE_VERSION}}|${RELEASE_VERSION}|g" -e "s|{{RELEASE_ISSUE}}|${RELEASE_ISSUE}|g" -e "s|{{RELEASE_VERSION_X}}|${RELEASE_VERSION_X}|g" "$file_path" > "$file_path.tmp" && mv "$file_path.tmp" "$file_path"
+          sed -e "s|{{RELEASE_VERSION}}|${RELEASE_VERSION}|g" -e "s|{{RELEASE_ISSUE}}|${RELEASE_ISSUE}|g" -e "s|{{RELEASE_BRANCH}}|${RELEASE_BRANCH}|g" -e "s|{{RELEASE_VERSION_X}}|${RELEASE_VERSION_X}|g" "$file_path" > "$file_path.tmp" && mv "$file_path.tmp" "$file_path"
       - name: Create component release issue from file
         if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]'
         uses: peter-evans/create-issue-from-file@v4