Merge branch 'develop' into uberpinguin/NMS-15809

.circleci/README.md

@@ -0,0 +1,68 @@
+# CircleCI Configuration
+
+This build is using a number of advanced features of CircleCI to make the CI
+pipeline a bit more manageable, most notably dynamic config[^1].
+
+## Dynamic Configuration
+
+Dynamic configuration allows us to do some logic at the beginning of a build
+to determine what parts of the pipeline to trigger.
+
+There is a small `circleci.yml` file that uses CircleCI's "continuation"
+support along with a python scripts to check which parts of the codebase
+have been modified, and then sets parameters to be used in a sub-workflow.
+
+The real workflow config file is generated dynamically based on the parameters set and 
+user provided properties. For backward compatibility we have kept the workflows logics
+(such as filters) similar to before.
+
+Each of the workflows in the "main" config then uses a `when:` field
+referencing various `trigger-*` parameters to enable or disable them.
+
+## Build paths
+
+The user has ability to modify the build path by
+1. using git commit:
+    * You can use the bundle names as the keywords (add `!` as prefix). Here are few examples,
+
+      | Keyword       | Description |
+      | ------------- | ------------- |
+      | !build-deploy | Runs build-deploy path |
+      | !smoke        | Runs smoke tests |
+      | !smoke-flaky  | Runs flaky smoke tests|
+      | !oci          | Runs Docker Container jobs |
+      | !doc          | Runs doc job  |
+      | !ui           | Runs ui job |
+      * For the latest list take a look at `.circleci/main/workflows/workflows_v2.json`.
+
+    * `process_generate.py` script attempts to detect and enable corresponding jobs automatically if incoming changes contains changes to:
+      * "IT.java" or "Test.java" files
+      * "docs" or "ui" folder
+      * "opennms-container" folder
+
+2. using `build-triggers.override.json` file
+    * rename `example-build-triggers.override.json` to `build-triggers.override.json`
+    * Modify `build-triggers.override.json` and enable the jobs you want to run by setting them to True
+
+    **Note:** When you enable a job(or workflow), we will enable their dependencies automatically.
+
+## Understanding the workflows and expanding them
+Workflows information is stored in `workflows_v2.json` file located under `main/workflows` folder.
+This file is broken down into `bundles` and `individual` sections.
+* `bundle` section allows for creating workflow(s) that contain of a set of `individual` job.
+* `individual` section allows for defining a job and it's dependencies and filters
+
+The names of `bundle` or `individual` are used in the `generate_main.py` file located under `pyscripts` folder.
+
+**Note:** There is a close relationship between properties defined in `example-build-triggers.override.json` and the logic used in `generate_main.py`.
+
+## Expanding the list of keywords detected
+The keywords used in detecting which workflow to run from a git commit message is defined in `process_generate.py` file located under `pyscripts` folder.
+
+
+# Smoke Tests
+See Readme file under smoke-test folder.
+
+# Footnotes
+
+[^1]: [Dynamic Configuration](https://circleci.com/docs/2.0/dynamic-config/)

.circleci/config.yml

@@ -0,0 +1,217 @@
+version: 2.1
+
+orbs:
+  continuation: circleci/[email protected]
+  # path-filtering: circleci/[email protected]
+
+parameters:
+  trigger-prebuild:
+    description: whether to trigger a pre-build evaluation
+    type: boolean
+    default: true
+  trigger-coverage-api:
+    description: whether to trigger a code coverage build
+    type: boolean
+    default: false
+  trigger-automation:
+    description: whether to trigger garbage collection for older feature images
+    type: boolean
+    default: false
+  trigger-flaky:
+    description: whether to enable running flaky smoke tests
+    type: boolean
+    default: false
+  is_pr:
+    description: whether this build is a pull request (only valid in non-setup workflows)
+    type: boolean
+    default: false
+
+setup: true
+
+commands:
+  shallow-clone:
+      description: "Quick shallow checkout (if possible)"
+      steps:
+        - run:
+            name: git clone
+            command: |
+              install -d -m 700 ~/.ssh
+              ssh-keyscan github.com >> ~/.ssh/known_hosts
+              ssh-keyscan -p 443 ssh.github.com >> ~/.ssh/known_hosts
+              chmod 600 ~/.ssh/known_hosts
+              git clone --no-checkout --filter=tree:0 "${CIRCLE_REPOSITORY_URL}" .
+              if [ -n "${CIRCLE_TAG}" ]; then
+                git checkout --force "${CIRCLE_TAG}"
+                git reset --hard "${CIRCLE_SHA1}"
+              else
+                git checkout --force -B "${CIRCLE_BRANCH}" "${CIRCLE_SHA1}"
+              fi
+  verify-config:
+    description: "Verify generated main.yml file"
+    steps:
+      - run:
+          name: Verify Dynamic Config
+          command: |
+            CIRCLE_BIN_DIR="$HOME/.local/bin"
+            mkdir -p "$CIRCLE_BIN_DIR"
+            export PATH="$CIRCLE_BIN_DIR:$PATH"
+            curl -fLSs https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh | DESTDIR="$CIRCLE_BIN_DIR" bash
+            "$CIRCLE_BIN_DIR/circleci" version
+            "$CIRCLE_BIN_DIR/circleci" config validate --skip-update-check /tmp/.circleci/main.yml || exit 1
+
+workflows:
+  coverage:
+    when:
+      and:
+        - equal: [ true, << pipeline.parameters.trigger-coverage-api >> ]
+        - equal: [ false, << pipeline.parameters.trigger-prebuild >> ]
+        - equal: [ false, << pipeline.parameters.trigger-automation >> ]
+        - equal: [ false, << pipeline.parameters.trigger-flaky >> ]
+    jobs:
+      - trigger-path-filtering:
+          base-revision: << pipeline.git.branch >>
+          mapping: |
+            .*                trigger-coverage true
+
+  pre-build:
+    when:
+      and:
+        - equal: [ false, << pipeline.parameters.trigger-coverage-api >> ]
+        - equal: [ true, << pipeline.parameters.trigger-prebuild >> ]
+        - equal: [ false, << pipeline.parameters.trigger-automation >> ]
+        - equal: [ false, << pipeline.parameters.trigger-flaky >> ]
+    jobs:
+      - trigger-path-filtering:
+          base-revision: << pipeline.git.branch >>
+          mapping: |
+            .*                trigger-coverage    false
+            .*                trigger-flaky-smoke false
+            ((?!docs/).)*     trigger-build       true
+            docs/.*           trigger-docs        true
+            ui/.*             trigger-ui          true
+            .circleci/.*      trigger-build       true
+            .circleci/main/jobs/build/build-docs.yml trigger-docs        true
+            .circleci/main/jobs/build/build-ui.yml trigger-ui        true
+            antora-playbook-local.yml      trigger-docs        true
+
+  pre-build-with-flaky:
+    when:
+      and:
+        - equal: [ false, << pipeline.parameters.trigger-coverage-api >> ]
+        - equal: [ true, << pipeline.parameters.trigger-prebuild >> ]
+        - equal: [ false, << pipeline.parameters.trigger-automation >> ]
+        - equal: [ true, << pipeline.parameters.trigger-flaky >> ]
+    jobs:
+      - trigger-path-filtering:
+          base-revision: << pipeline.git.branch >>
+          mapping: |
+            .*                trigger-coverage    false
+            .*                trigger-flaky-smoke true
+            ((?!docs/).)*     trigger-build       true
+            docs/.*           trigger-docs        true
+            ui/.*             trigger-ui          true
+            .circleci/.*      trigger-build       true
+            .circleci/main/jobs/build/build-docs.yml trigger-docs        true
+            .circleci/main/jobs/build/build-ui.yml trigger-ui        true
+            antora-playbook-local.yml      trigger-docs        true
+
+  automation:
+    when:
+      and:
+        - equal: [ false, << pipeline.parameters.trigger-coverage-api >> ]
+        - equal: [ false, << pipeline.parameters.trigger-prebuild >> ]
+        - equal: [ true, << pipeline.parameters.trigger-automation >> ]
+    jobs:
+      - docker_gc:
+          context:
+            - "CircleCI"
+            - "docker-publish-account"
+jobs:
+  docker_gc:
+    docker:
+      - image: docker
+    steps:
+      - run:
+          name:  Docker GC - remove inactive images on feature branches
+          # TODO replace expect when https://github.com/docker/hub-tool/pull/198 is merged
+          command: |
+            wget https://github.com/docker/hub-tool/releases/download/v0.4.5/hub-tool-linux-amd64.tar.gz
+            tar -xzf hub-tool-linux-amd64.tar.gz
+            cd hub-tool
+            DOCKERHUB_AUTH_HASH="$(echo -n "${DOCKERHUB_LOGIN}:${DOCKERHUB_PASS}" | base64)"
+            echo "{ \"auths\": { \"hub-tool\": { \"auth\": \"${DOCKERHUB_AUTH_HASH}\" } } }" > config.json
+            export DOCKER_CONFIG="$(pwd)"
+            for HUB_REPO in horizon minion sentinel meridian-minion; do
+              GC_LIST=$( ./hub-tool tag ls "opennms/${HUB_REPO}" | ( grep -E '^[^:][^:]*:(feature|jira|dependabot)' || true ) | ( grep inactive || true ) | cut -d" " -f1 )
+              for image in $GC_LIST; do
+                echo "Delete image $image"
+                ( yes || true ) | ./hub-tool tag rm $image
+              done
+            done
+
+  trigger-coverage:
+    docker:
+      - image: cimg/python:3.10.1
+    steps:
+      - shallow-clone
+      - run:
+          name: Creating build-trigger file
+          command: |
+               cp .circleci/example-build-triggers.override.json /tmp/build-triggers.json
+               sed -i 's/"coverage": false/"coverage": true/g' /tmp/build-triggers.json
+      - store_artifacts:
+          path: /tmp/build-triggers.json
+          destination: build-triggers.json      
+      - continuation/continue:
+          circleci_domain: circleci.com
+          configuration_path: .circleci/main.yml
+          parameters: '{ "trigger-coverage": true }'
+
+  trigger-path-filtering:
+    docker:
+      - image: cimg/python:3.10.1
+    parameters:
+      base-revision:
+        default: main
+        description: The revision to compare the current one against for the purpose of determining changed files.
+        type: string
+      mapping:
+        default: ""
+        description: Mapping of path regular expressions to pipeline parameters and values. One mapping per line, whitespace-delimited.
+        type: string
+    steps:
+      - shallow-clone
+      - run:
+          name: Trigger Status
+          command: |
+            echo Coverage API: << pipeline.parameters.trigger-coverage-api >>
+            echo Prebuild: << pipeline.parameters.trigger-prebuild >>
+            echo Automation: << pipeline.parameters.trigger-automation >>
+            echo Trigger Flaky Tests: << pipeline.parameters.trigger-flaky >>
+      - run:
+          name: Process Generate
+          environment:
+            BASE_REVISION: << parameters.base-revision >>
+            MAPPING: << parameters.mapping >>
+            OUTPUT_PATH: /tmp/pipeline-parameters.json
+          command: python3 .circleci/pyscripts/process_generate.py 
+      - run:
+          name: Generate Main YAML file
+          command: python3 .circleci/pyscripts/generate_main.py 
+      - verify-config
+      - store_artifacts:
+          path: /tmp/pipeline-parameters.json
+          destination: pipeline-parameters.json
+      - store_artifacts:
+          path: /tmp/build-triggers.json
+          destination: build-triggers.json
+      - store_artifacts:
+          path: /tmp/.circleci/main.yml
+          destination: main.yml
+      - store_artifacts:
+          path: /tmp/performance.txt
+          destination: performance.txt
+      - continuation/continue:
+          circleci_domain: circleci.com
+          configuration_path: /tmp/.circleci/main.yml
+          parameters: /tmp/pipeline-parameters.json

.circleci/epoch

@@ -0,0 +1,2 @@
+# increment this to force a full build
+4

.circleci/example-build-triggers.override.json

@@ -0,0 +1,15 @@
+{
+    "build-deploy": false,
+    "coverage": false,
+    "docs": false,
+    "ui": false,
+    "integration": false,
+    "smoke": false,
+    "smoke-flaky": false,
+    "rpms": false,
+    "debs": false,
+    "oci": false,
+    "build-publish": false,
+    "trivy-scan": false,
+    "experimental": false
+}

.circleci/main/@main.yml

@@ -0,0 +1,34 @@
+version: 2.1
+
+orbs:
+  cloudsmith: cloudsmith/[email protected]
+  continuation: circleci/[email protected]
+  jira: circleci/[email protected]
+  # path-filtering: circleci/[email protected]
+  sign-packages: opennms/[email protected]
+
+commands:
+  #commands:maven/maven.index#
+  #commands:cache/cache#
+  #commands:login/login.index#
+  #commands:executions/executions.index#
+  #commands:oci/oci#
+  #commands:oci/scan-image-trivy#
+  #commands:generic/generic#
+
+#workflows#
+
+jobs:
+  #jobs:build/build.index#
+  #jobs:tarball-assembly-only#
+  #jobs:rpms/rpms.index#
+  #jobs:debian packages/deb.index#
+  #jobs:oci/oci.index#
+  #jobs:tests/integration/integration.index#
+  #jobs:code-coverage#
+  #jobs:tests/smoke/smoke-test.index#
+  #jobs:create-merge-foundation-branch#
+  #jobs:merge-foundation-branch#
+  #jobs:create-merge-meridian-branch#
+  #jobs:merge-poweredby-branch#
+  #jobs:publish#

.circleci/main/aliases/general.yml

@@ -0,0 +1,9 @@
+aliases:
+  - &setup_dct_env
+    name: Setup DCT environment
+    command: |
+      echo "export DOCKER_CONTENT_TRUST=1" >> $BASH_ENV
+      echo "export DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE=\"$DCT_DELEGATE_KEY_PASSPHRASE\"" >> $BASH_ENV
+      echo "export MINION_DK_REPO=docker.io/opennms/minion" >> $BASH_ENV
+      echo "export MINION_AZ_REPO=opennmspubacr.azurecr.io/opennms/minion" >> $BASH_ENV
+      echo "export KEY_FOLDER=~/.docker/trust/private" >> $BASH_ENV