-
Notifications
You must be signed in to change notification settings - Fork 29
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Jenkins Webhook and RFS E2E job on PR (#938)
This change is hopefully the final chapter in the Jenkins setup saga and a huge leap in the integration testing story. It adds support for triggering a Jenkins job through a configured Github Action and monitoring its completion to pass/fail the action. This design is one of many that can be taken, but a benefit of this approach is that it keeps the Github action in control of triggering Jenkins jobs and status updates, as well as doesn't require Jenkins to update the PR (though this may be a useful addition for logs or other data in the future). As a part of this pattern, a general python webhook was created to trigger Jenkins jobs and monitor for a terminal state, this could also potentially become its own GHA plugin separate from this repo. Additionally this adds our first GHA which triggers a Jenkins job for our RFS E2E integration test. This GHA is planned to be monitored as any bugs/issues are worked out and should not currently be restrictive for PRs to be merged. Jenkins pipeline files have also been updated to attain a lock on deployment stages to prevent different pipeline executions from using the same stage. The pipeline files have also been largely moved to the shared library, to allow any changes to the context options or other settings for the pipeline to be tested in PR actions. --------- Signed-off-by: Tanner Lewis <[email protected]> Co-authored-by: Peter Nied <[email protected]>
- Loading branch information
Showing
7 changed files
with
167 additions
and
108 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| name: Jenkins | ||
|
|
||
| on: | ||
| push: | ||
| branches-ignore: | ||
| - 'backport/**' | ||
| - 'dependabot/**' | ||
| pull_request_target: | ||
| types: [opened, synchronize, reopened] | ||
|
|
||
| env: | ||
| python-version: '3.11' | ||
|
|
||
| permissions: | ||
| contents: read # to fetch code (actions/checkout) | ||
|
|
||
| jobs: | ||
| rfs-e2e-aws-test: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Determine Github repository and branch | ||
| id: determine-repo-vars | ||
| run: | | ||
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then | ||
| branch_name="${{ github.event.pull_request.head.ref }}" | ||
| pr_repo_url="https://github.com/${{ github.event.pull_request.head.repo.full_name }}.git" | ||
| else | ||
| branch_name="${{ github.ref_name }}" | ||
| pr_repo_url="https://github.com/${{ github.repository }}.git" | ||
| fi | ||
| echo "Running jenkins test on repo: $pr_repo_url and branch: $branch_name" | ||
| echo "branch_name=$branch_name" >> $GITHUB_OUTPUT | ||
| echo "pr_repo_url=$pr_repo_url" >> $GITHUB_OUTPUT | ||
| - name: Jenkins Job Trigger and Monitor | ||
| uses: lewijacn/[email protected] | ||
| with: | ||
| jenkins_url: "https://migrations.ci.opensearch.org" | ||
| job_name: "rfs-default-e2e-test" | ||
| api_token: "${{ secrets.JENKINS_MIGRATIONS_GENERIC_WEBHOOK_TOKEN }}" | ||
| job_params: "GIT_REPO_URL=${{ steps.determine-repo-vars.outputs.pr_repo_url }},GIT_BRANCH=${{ steps.determine-repo-vars.outputs.branch_name }}" |
78 changes: 0 additions & 78 deletions
78
jenkins/migrationIntegPipelines/rfsBackfillE2EPipeline.groovy
This file was deleted.
Oops, something went wrong.
9 changes: 9 additions & 0 deletions
9
jenkins/migrationIntegPipelines/rfsDefaultE2ETestCover.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| def gitBranch = params.GIT_BRANCH ?: 'main' | ||
| def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git' | ||
|
|
||
| library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( | ||
| [$class: 'GitSCMSource', | ||
| remote: "${gitUrl}"]) | ||
|
|
||
| // Shared library function (location from root: vars/rfsDefaultE2ETest.groovy) | ||
| rfsDefaultE2ETest() |
9 changes: 9 additions & 0 deletions
9
jenkins/migrationIntegPipelines/trafficReplayDefaultE2ETestCover.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| def gitBranch = params.GIT_BRANCH ?: 'main' | ||
| def gitUrl = params.GIT_REPO_URL ?: 'https://github.com/opensearch-project/opensearch-migrations.git' | ||
|
|
||
| library identifier: "migrations-lib@${gitBranch}", retriever: modernSCM( | ||
| [$class: 'GitSCMSource', | ||
| remote: "${gitUrl}"]) | ||
|
|
||
| // Shared library function (location from root: vars/trafficReplayDefaultE2ETest.groovy) | ||
| trafficReplayDefaultE2ETest() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // Note: | ||
| // 1. There is a still a manual step needed on the EC2 source load balancer to replace its security group rule which allows all traffic (0.0.0.0/0) to | ||
| // allow traffic for the relevant service security group. This needs a better story around accepting user security groups in our Migration CDK. | ||
|
|
||
| def call(Map config = [:]) { | ||
| def sourceContextId = 'source-single-node-ec2' | ||
| def migrationContextId = 'migration-rfs' | ||
| def source_cdk_context = """ | ||
| { | ||
| "source-single-node-ec2": { | ||
| "suffix": "ec2-source-<STAGE>", | ||
| "networkStackSuffix": "ec2-source-<STAGE>", | ||
| "distVersion": "7.10.2", | ||
| "distributionUrl": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz", | ||
| "captureProxyEnabled": false, | ||
| "securityDisabled": true, | ||
| "minDistribution": false, | ||
| "cpuArch": "x64", | ||
| "isInternal": true, | ||
| "singleNodeCluster": true, | ||
| "networkAvailabilityZones": 2, | ||
| "dataNodeCount": 1, | ||
| "managerNodeCount": 0, | ||
| "serverAccessType": "ipv4", | ||
| "restrictServerAccessTo": "0.0.0.0/0" | ||
| } | ||
| } | ||
| """ | ||
| def migration_cdk_context = """ | ||
| { | ||
| "migration-rfs": { | ||
| "stage": "<STAGE>", | ||
| "vpcId": "<VPC_ID>", | ||
| "engineVersion": "OS_2.11", | ||
| "domainName": "os-cluster-<STAGE>", | ||
| "dataNodeCount": 2, | ||
| "openAccessPolicyEnabled": true, | ||
| "domainRemovalPolicy": "DESTROY", | ||
| "artifactBucketRemovalPolicy": "DESTROY", | ||
| "trafficReplayerServiceEnabled": false, | ||
| "reindexFromSnapshotServiceEnabled": true, | ||
| "sourceClusterEndpoint": "<SOURCE_CLUSTER_ENDPOINT>" | ||
| } | ||
| } | ||
| """ | ||
|
|
||
| defaultIntegPipeline( | ||
| sourceContext: source_cdk_context, | ||
| migrationContext: migration_cdk_context, | ||
| sourceContextId: sourceContextId, | ||
| migrationContextId: migrationContextId, | ||
| defaultStageId: 'rfs-integ', | ||
| skipCaptureProxyOnNodeSetup: true, | ||
| jobName: 'rfs-default-e2e-test', | ||
| integTestCommand: '/root/lib/integ_test/integ_test/backfill_tests.py' | ||
| ) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters