e2e.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

on:
  pull_request:
  push:
    branches:
      - dev
      - release-*
      - dev-*

name: E2E

concurrency:
  group: e2e-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true


jobs:
  paths-filter:
    name: E2E-Path-Filter
    runs-on: ubuntu-latest
    outputs:
      not-ignore: ${{ steps.filter.outputs.not-ignore }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            not-ignore:
              - '!**/*.md'
  license-header:
    name: License header
    runs-on: ubuntu-latest
    if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
    needs: paths-filter
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Check license header
        uses: apache/skywalking-eyes/header@main
  code-style:
    if: github.repository == 'apache/incubator-streampark'
    name: Code style
    runs-on: ubuntu-latest
    needs: license-header
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Check codestyle
        run: ./mvnw -B -q -nsu checkstyle:check spotless:check
  build:
    name: E2E-Build
    runs-on: ubuntu-latest
    needs: code-style
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Set up JDK 8
        uses: actions/setup-java@v4
        with:
          java-version: 8
          distribution: 'adopt'
      - name: Cache local Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
          restore-keys: ${{ runner.os }}-maven-
      - name: Build Image
        run: |
          ./build.sh \
          && docker build -f docker/Dockerfile . -t apache/streampark:ci
      - name: Export Docker Images
        run: |
          docker save apache/streampark:ci -o /tmp/standalone-image.tar \
          && du -sh /tmp/standalone-image.tar
      - uses: actions/upload-artifact@v4
        name: Upload Docker Images
        with:
          name: standalone-image
          path: /tmp/standalone-image.tar
          retention-days: 1
  e2e:
    name: ${{ matrix.case.name }}
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      matrix:
        case:
          - name: EnvironmentTest
            class: org.apache.streampark.e2e.cases.EnvironmentTest
          - name: AlarmTest
            class: org.apache.streampark.e2e.cases.AlarmTest
          - name: UserManagementTest
            class: org.apache.streampark.e2e.cases.UserManagementTest
          - name: RoleManagementTest
            class: org.apache.streampark.e2e.cases.RoleManagementTest
          - name: TeamManagementTest
            class: org.apache.streampark.e2e.cases.TeamManagementTest
          - name: MemberManagementTest
            class: org.apache.streampark.e2e.cases.MemberManagementTest
          - name: ExternalLinkTest
            class: org.apache.streampark.e2e.cases.ExternalLinkTest
          - name: YarnQueueTest
            class: org.apache.streampark.e2e.cases.YarnQueueTest
          - name: TokenManagementTest
            class: org.apache.streampark.e2e.cases.TokenManagementTest
          - name: UploadManagementTest
            class: org.apache.streampark.e2e.cases.UploadManagementTest
          - name: ProjectsManagementTest
            class: org.apache.streampark.e2e.cases.ProjectsManagementTest
          - name: VariableManagementTest
            class: org.apache.streampark.e2e.cases.VariableManagementTest
          - name: Flink118OnRemoteClusterDeployTest
            class: org.apache.streampark.e2e.cases.Flink118OnRemoteClusterDeployTest
          - name: Flink117OnRemoteClusterDeployTest
            class: org.apache.streampark.e2e.cases.Flink117OnRemoteClusterDeployTest
          - name: Flink116OnRemoteClusterDeployTest
            class: org.apache.streampark.e2e.cases.Flink116OnRemoteClusterDeployTest
          - name: Flink116OnYarnClusterDeployTest
            class: org.apache.streampark.e2e.cases.Flink116OnYarnClusterDeployTest
          - name: Flink117OnYarnClusterDeployTest
            class: org.apache.streampark.e2e.cases.Flink117OnYarnClusterDeployTest
          - name: Flink118OnYarnClusterDeployTest
            class: org.apache.streampark.e2e.cases.Flink118OnYarnClusterDeployTest
    env:
      RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
    steps:
      - name: Maximize runner space
        uses: easimon/maximize-build-space@fc881a613ad2a34aca9c9624518214ebc21dfc0c
        with:
          root-reserve-mb: 30720
          temp-reserve-mb: 10240
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          remove-codeql: 'true'
          remove-docker-images: 'true'
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          java-version: 11
          distribution: 'adopt'
      - name: Collect Workflow Telemetry
        uses: ./.github/actions/workflow-telemetry-action
        with:
          comment_on_pr: false
      - name: Cache local Maven repository
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
          restore-keys: ${{ runner.os }}-maven-
      - uses: actions/download-artifact@v4
        name: Download Docker Images
        with:
          name: standalone-image
          path: /tmp
      - name: Load Docker Images
        run: |
          docker load -i /tmp/standalone-image.tar \
          && rm -f /tmp/standalone-image.tar
      - name: Run Test
        run: |
          ./mvnw -B -f streampark-e2e/pom.xml -am \
            -DfailIfNoTests=false \
            -Dtest=${{ matrix.case.class }} test
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload Recording
        with:
          name: recording-${{ matrix.case.name }}
          path: ${{ env.RECORDING_PATH }}
          retention-days: 1
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload Projects Build Logs
        with:
          name: ${{ matrix.case.name }}-projects-build-logs
          path: ~/streampark_build_logs
          retention-days: 1
  result:
    name: E2E - Result
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: [ paths-filter, license-header, code-style, build, e2e ]
    if: always()
    steps:
      - name: Status
        run: |
          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
            echo "Skip E2E!"
            exit 0
          fi
          if [[ ${{ needs.e2e.result }} != 'success' ]]; then
            echo "E2E Failed!"
            exit -1
          fi