Merge branch 'apache:branch-4.15' into branch-4.15

.github/ISSUE_TEMPLATE/bp.md

@@ -9,7 +9,7 @@ assignees: ''
 
 **BP**
 
-> Follow the instructions at http://bookkeeper.apache.org/community/bookkeeper_proposals/ to create a proposal.
+> Follow the instructions at https://bookkeeper.apache.org/community/bookkeeper-proposals/ to create a proposal.
 
 This is the master ticket for tracking BP-xyz :
 

.github/actions/clean-disk/action.yml

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: clean disk
+description: makes some more space available on the disk by removing files
+inputs:
+  mode:
+    description: "Use 'full' to clean as much as possible"
+    required: false
+runs:
+  using: composite
+  steps:
+    - run: |
+        if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+          directories=(/usr/local/lib/android /opt/ghc)
+          if [[ "${{ inputs.mode }}" == "full" ]]; then
+            # remove these directories only when mode is 'full'
+            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
+          fi
+          emptydir=/tmp/empty$$/
+          mkdir $emptydir
+          echo "::group::Available diskspace"
+          time df -BM / /mnt
+          echo "::endgroup::"
+          for directory in "${directories[@]}"; do
+            echo "::group::Removing $directory"
+            # fast way to delete a lot of files on linux
+            time sudo eatmydata rsync -a --delete $emptydir ${directory}/
+            time sudo eatmydata rm -rf ${directory}
+            time df -BM / /mnt
+            echo "::endgroup::"
+          done
+          echo "::group::Cleaning apt state"
+          time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
+          time df -BM / /mnt
+          echo "::endgroup::"
+        fi
+        echo "::group::Available diskspace"
+        time df -BM / /mnt
+        echo "::endgroup::"
+      shell: bash

gradle.properties → .github/actions/copy-test-reports/action.yml

@@ -1,5 +1,4 @@
 #
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -17,14 +16,12 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-#
 
-protobufPluginVersion=0.8.18
-apcheRatPluginVersion=0.7.0
-shadowPluginVersion=6.1.0
-licenseGradlePluginVersion=0.15.0
-checkStyleVersion=6.19
-spotbugsPlugin=4.7.0
-testLogger=3.1.0
-testRetry=1.0.0
-owaspPlugin=6.5.3
+name: Copy test reports
+description: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+runs:
+  using: composite
+  steps:
+    - run: |
+        $GITHUB_WORKSPACE/dev/ci-tool move_test_reports
+      shell: bash

.github/actions/tune-runner-vm/action.yml

@@ -24,8 +24,79 @@ runs:
   steps:
     - run: |
         if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+            echo "::group::Configure and tune OS"
             # Ensure that reverse lookups for current hostname are handled properly
             # Add the current IP address, long hostname and short hostname record to /etc/hosts file
             echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+
+            # The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
+            # consumption is high.
+            # Set vm.swappiness=1 to avoid swapping and allow high RAM usage
+            echo 1 | sudo tee /proc/sys/vm/swappiness
+            (
+              shopt -s nullglob
+              # Set swappiness to 1 for all cgroups and sub-groups
+              for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
+                echo 1 | sudo tee $swappiness_file > /dev/null
+              done
+            ) || true
+
+            # use "madvise" Linux Transparent HugePages (THP) setting
+            # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
+            # "madvise" is generally a better option than the default "always" setting
+            # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
+            echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
+            echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
+            echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+            echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
+    
+            # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
+            # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
+            # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
+            # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
+            # https://www.spinics.net/lists/linux-ide/msg52562.html
+            for i in /sys/block/sd*/queue/discard_max_bytes; do
+              echo 0 | sudo tee $i
+            done
+            # disable any background jobs that run SSD discard/trim
+            sudo systemctl disable fstrim.timer || true
+            sudo systemctl stop fstrim.timer || true
+            sudo systemctl disable fstrim.service || true
+            sudo systemctl stop fstrim.service || true
+
+            # stop php-fpm
+            sudo systemctl stop php8.0-fpm.service || true
+            sudo systemctl stop php7.4-fpm.service || true
+            # stop mono-xsp4
+            sudo systemctl disable mono-xsp4.service || true
+            sudo systemctl stop mono-xsp4.service || true
+            sudo killall mono || true
+
+            # stop Azure Linux agent to save RAM
+            sudo systemctl stop walinuxagent.service || true
+          
+            # enable docker experimental mode which is
+            # required for using "docker build --squash" / "-Ddocker.squash=true"
+            daemon_json="$(sudo cat /etc/docker/daemon.json  | jq '.experimental = true')"
+            echo "$daemon_json" | sudo tee /etc/docker/daemon.json
+            # restart docker daemon
+            sudo systemctl restart docker
+            echo '::endgroup::'
+
+            # show memory
+            echo "::group::Available Memory"
+            free -m
+            echo '::endgroup::'
+            # show disk
+            echo "::group::Available diskspace"
+            df -BM
+            echo "::endgroup::"
+            # show cggroup
+            echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
+            CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
+            sudo cgget -a $CURRENT_CGGROUP || true
+            echo '::endgroup::'
         fi
-      shell: bash
+      shell: bash

.github/changes-filter.yaml

@@ -0,0 +1,16 @@
+# contains pattern definitions used in workflows "changes" step
+# pattern syntax: https://github.com/micromatch/picomatch
+all:
+  - '**'
+docs:
+  - 'site3/**'
+  - '.asf.yaml'
+  - '*.md'
+  - '**/*.md'
+  - '.github/changes-filter.yaml'
+  - '.github/ISSUE_TEMPLATE/**'
+need_owasp:
+  - 'pom.xml'
+  - '**/pom.xml'
+  - 'src/owasp-dependency-check-false-positives.xml'
+  - 'src/owasp-dependency-check-suppressions.xml'

.github/workflows/README.md

@@ -0,0 +1,60 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+## GitHub Workflows
+
+This directory contains all BookKeeper CI checks.
+
+### Required Workflows
+
+When adding new CI workflows, please update the [.asf.yaml](../../.asf.yaml) if the workflow is required to pass before
+a PR can be merged. Instructions on how to update the file are below.
+
+This project uses the [.asf.yaml](../../.asf.yaml) to configure which workflows are required to pass before a PR can
+be merged. In the `.asf.yaml`, the required contexts are defined in the `github.protected_branches.*.required_status_checks.contexts.[]`
+where * is any key in the `protected_branches` map.
+
+You can view the currently required status checks by running the following command:
+
+```shell
+curl -s -H 'Accept: application/vnd.github.v3+json' https://api.github.com/repos/apache/bookkeeper/branches/master | \
+jq .protection
+```
+
+These contexts get their names in one of two ways depending on how the workflow file is written in this directory. The
+following command will print out the names of each file and the associated with the check. If the `name` field is `null`,
+the context will be named by the `id`.
+
+```shell
+for f in .github/workflows/*.yaml .github/workflows/*.yml; \
+do FILE=$f yq eval -o j '.jobs | to_entries | {"file": env(FILE),"id":.[].key, "name":.[].value.name}' $f; \
+done
+```
+
+Duplicate names are allowed, and all checks with the same name will be treated the same (required or not required).
+
+When working on workflow changes, one way to find out the names of the status checks is to retrieve the names
+from the PR build run. The "check-runs" can be found by commit id. Here's an example:
+
+```shell
+curl -s "https://api.github.com/repos/apache/bookkeeper/commits/$(git rev-parse HEAD)/check-runs" | \
+  jq -r '.check_runs | .[] | .name' |sort
+```