From 488d377c567ce50db2dda1463ec539f80f9784ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 4 Jan 2024 14:42:51 +0100 Subject: [PATCH] Add tools to assemble DEB packages (#96) * Add tools to assemble DEB packages * Move wazuh-indexer-performance-analyzer.service to common * Enable assembly of DEB packages * Enable full set of plugins * Actually skip tar assembly * Add installation of dependencies for DEB assembly * Install dependencies using sudo * Format files * Refactor assemble script --- .github/workflows/build.yml | 2 +- .github/workflows/r_assemble.yml | 6 +- ...wazuh-indexer-performance-analyzer.service | 0 distribution/packages/src/deb/Makefile | 19 + .../packages/src/deb/debmake_install.sh | 46 +++ docker/README.md | 2 +- scripts/README.md | 184 ++++++--- scripts/assemble.sh | 356 +++++++++++------- scripts/provision.sh | 10 + 9 files changed, 439 insertions(+), 186 deletions(-) rename distribution/packages/src/{rpm => common}/wazuh-indexer-performance-analyzer.service (100%) create mode 100644 distribution/packages/src/deb/Makefile create mode 100644 distribution/packages/src/deb/debmake_install.sh create mode 100644 scripts/provision.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1483793de3f68..4b286884541c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: exclude: # skip arm64 until we have arm runners - architecture: arm64 - - distribution: [tar, deb] # Exclude deb assembly until it's implemented + - distribution: tar uses: ./.github/workflows/r_assemble.yml with: diff --git a/.github/workflows/r_assemble.yml b/.github/workflows/r_assemble.yml index c807a58a4f781..6414729a1a635 100644 --- a/.github/workflows/r_assemble.yml +++ b/.github/workflows/r_assemble.yml @@ -39,6 +39,11 @@ jobs: name: ${{ inputs.min }} path: artifacts/dist + - name: Provision + if: ${{ inputs.distribution == 'deb' }} + run: | + sudo bash scripts/provision.sh + - name: Run `assemble.sh` run: | bash scripts/assemble.sh -v ${{ vars.OPENSEARCH_VERSION }} -p linux -a ${{ inputs.architecture }} -d ${{ inputs.distribution }} @@ -55,4 +60,3 @@ jobs: name: ${{ inputs.name }} path: artifacts/dist/${{ steps.get_name.outputs.name }} if-no-files-found: error - diff --git a/distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service b/distribution/packages/src/common/wazuh-indexer-performance-analyzer.service similarity index 100% rename from distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service rename to distribution/packages/src/common/wazuh-indexer-performance-analyzer.service diff --git a/distribution/packages/src/deb/Makefile b/distribution/packages/src/deb/Makefile new file mode 100644 index 0000000000000..14e4dbd7efc82 --- /dev/null +++ b/distribution/packages/src/deb/Makefile @@ -0,0 +1,19 @@ +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# deb opensearch Makefile + +all: install + +install: + ./debmake_install.sh $(CURDIR) + +clean: ; + +distclean: clean + +.PHONY: all clean distclean install \ No newline at end of file diff --git a/distribution/packages/src/deb/debmake_install.sh b/distribution/packages/src/deb/debmake_install.sh new file mode 100644 index 0000000000000..4b10a36e001c4 --- /dev/null +++ b/distribution/packages/src/deb/debmake_install.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# debmake opensearch install script + +set -ex + +if [ -z "$1" ]; then + echo "Missing curdir path" + exit 1 +fi + +curdir=$1 +product_dir=/usr/share/wazuh-indexer +# config_dir=/etc/wazuh-indexer +data_dir=/var/lib/wazuh-indexer +log_dir=/var/log/wazuh-indexer +pid_dir=/var/run/wazuh-indexer +buildroot=${curdir}/debian/wazuh-indexer + +# Create necessary directories +mkdir -p "${buildroot}" +mkdir -p "${buildroot}${pid_dir}" +mkdir -p "${buildroot}${product_dir}/plugins" + +# Install directories/files +cp -a "${curdir}"/etc "${curdir}"/usr "${curdir}"/var "${buildroot}"/ +chmod -c 0755 "${buildroot}${product_dir}"/bin/* +if [ -d "${buildroot}${product_dir}"/plugins/opensearch-security ]; then + chmod -c 0755 "${buildroot}${product_dir}"/plugins/opensearch-security/tools/* +fi + +# Symlinks (do not symlink config dir as security demo installer has dependency, if no presense it will switch to rpm/deb mode) +ln -s ${data_dir} "${buildroot}${product_dir}/data" +ln -s ${log_dir} "${buildroot}${product_dir}/logs" + +# Change Permissions +chmod -Rf a+rX,u+w,g-w,o-w "${buildroot}"/* + +exit 0 diff --git a/docker/README.md b/docker/README.md index 28a58329ab7d7..4b41a42e0e9c9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ -# Frontend development environments +# Indexer development environments Install [Docker Desktop][docker-desktop] as per its instructions, available for Windows, Mac and Linux (Ubuntu, Debian & Fedora). diff --git a/scripts/README.md b/scripts/README.md index e71941bed4085..8c98f8206cab5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,9 +2,9 @@ The packages' generation process consists on 2 steps: -* **Build**: compiles the Java application and bundles it into a package. -* **Assembly**: uses the package from the previous step and inflates it with plugins and -configuration files, ready for production deployment. +- **Build**: compiles the Java application and bundles it into a package. +- **Assembly**: uses the package from the previous step and inflates it with plugins and + configuration files, ready for production deployment. We usually generate the packages using GitHub Actions, however, the process is designed to be independent enough for maximum portability. GitHub Actions provides infrastructure, while @@ -27,7 +27,6 @@ act -j build -W .github/workflows/build.yml --artifact-server-path ./artifacts [Build slim packages/build] 🏁 Job succeeded ``` - #### Running in Docker Within the [Docker environment](../docker): @@ -38,60 +37,162 @@ bash scripts/build.sh -v 2.11.0 -s false -p linux -a {x64|arm64} -d {rpm|deb|tar The generated package is sent to `artifacts/` - ## Assemble - +### DEB + +The script will: + +- Extract the deb package using `ar` and `tar` tools. + + > By default, `ar` and `tar` tools expect the package to be in `wazuh-indexer/artifacts/tmp/deb`. The script takes care of creating the required folder structure, copying also the min package and the Makefile. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.0_amd64.deb + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.deb`. + `Makefile` and the `debmake_install` are copied over from `wazuh-indexer/distribution/packages/src/deb`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + +- Install the plugins using the `opensearch-plugin` CLI tool. +- Set up configuration files. + + > Included in `min-package`. Default files are overwritten. + +- Bundle a DEB file with `debmake` and the `Makefile`. + + > `debmake` and other dependencies can be installed using the provision.sh script. The + > script is invoked by the GitHub Workflow. + + Current folder loadout at this stage: + + ``` + artifacts/ + |-- artifact_name.txt + |-- dist + | |-- wazuh-indexer-min_4.9.0_amd64.deb + | `-- wazuh-indexer_4.9.0_amd64.deb + `-- tmp + `-- deb + |-- Makefile + |-- data.tar.gz + |-- debmake_install.sh + |-- etc + |-- usr + |-- var + `-- wazuh-indexer-min_4.9.0_amd64.deb + ``` + +### Running in Act + +```console +act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifacts --matrix distribution:deb --matrix architecture:x64 --var OPENSEARCH_VERSION=2.11.0 + +[Build slim packages/build] 🏁 Job succeeded +``` + +#### Running in Docker + +Pre-requisites: + +- Current directory: `wazuh-indexer/` +- Existing deb package in `wazuh-indexer/artifacts/dist/deb`, as a result of the _Build_ stage. + +```console +MIN_PKG_PATH="./artifacts" +docker run --rm \ + -v ./scripts/:/home/wazuh-indexer/scripts \ + -v $MIN_PKG_PATH:/home/wazuh-indexer/artifacts \ + -v ./distribution/packages/src:/home/wazuh-indexer/distribution/packages/src \ + -w /home/wazuh-indexer \ + -it ubuntu:jammy /bin/bash + +# https://github.com/opensearch-project/opensearch-build/blob/2.11.1/docker/ci/dockerfiles/current/build.ubuntu2004.opensearch.x64.arm64.dockerfile + +# Install necessary packages +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && + apt-get install -y debmake debhelper-compat && + apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && + apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && + apt-get clean -y + +# Install aptly and required changes to debmake +# Remove lintian for now due to it takes nearly 20 minutes for OpenSearch as well as nearly an hour for OpenSearch-Dashboards during debmake +curl -o- https://www.aptly.info/pubkey.txt | apt-key add - && + echo "deb http://repo.aptly.info/ squeeze main" | tee -a /etc/apt/sources.list.d/aptly.list && + apt-get update -y && apt-get install -y aptly && apt-get clean -y && + dpkg -r lintian + +bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d deb +``` + ### RPM -The `assemble.sh` script will use the output from the `build.sh` script and use it as a -base to bundle together a final package containing the plugins, the production configuration +The `assemble.sh` script will use the output from the `build.sh` script and use it as a +base to bundle together a final package containing the plugins, the production configuration and the service files. The script will: - Extract the rpm package using `rpm2cpio` and `cpio` tools. - - > By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. - - Current folder loadout at this stage: - ``` - /rpm/$ARCH - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` - - `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. - `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. - The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. + + > By default, `rpm2cpio` and `cpio` tools expect the package to be in `wazuh-indexer/artifacts/tmp/rpm`. The script takes care of creating the required folder structure, copying also the min package and the SPEC file. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` + + `usr`, `etc` and `var` folders contain `wazuh-indexer` files, extracted from `wazuh-indexer-min-*.rpm`. + `wazuh-indexer.rpm.spec` is copied over from `wazuh-indexer/distribution/packages/src/rpm/wazuh-indexer.rpm.spec`. + The `wazuh-indexer-performance-analyzer.service` file is also copied from the same folder. It is a dependency of the SPEC file. - Install the plugins using the `opensearch-plugin` CLI tool. - Set up configuration files. - > Included in `min-package`. Default files are overwritten. - + > Included in `min-package`. Default files are overwritten. + - Bundle an RPM file with `rpmbuild` and the SPEC file `wazuh-indexer.rpm.spec`. - - `rpmbuild` is part of the `rpm` OS package. - > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + - `rpmbuild` is part of the `rpm` OS package. - Current folder loadout at this stage: - ``` - /rpm/$ARCH - /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} - /etc - /usr - /var - wazuh-indexer-min-*.rpm - wazuh-indexer.rpm.spec - ``` + > `rpmbuild` is invoked from `wazuh-indexer/artifacts/tmp/rpm`. It creates the {BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} folders and applies the rules in the SPEC file. If successful, `rpmbuild` will generate the package in the `RPMS/` folder. The script will copy it to `wazuh-indexer/artifacts/dist` and clean: remove the `tmp\` folder and its contents. + + Current folder loadout at this stage: + + ``` + /rpm/$ARCH + /{BUILD,RPMS,SOURCES,SRPMS,SPECS,TMP} + /etc + /usr + /var + wazuh-indexer-min-*.rpm + wazuh-indexer.rpm.spec + ``` ### Running in Act @@ -105,8 +206,8 @@ act -j assemble -W .github/workflows/build.yml --artifact-server-path ./artifact Pre-requisites: -* Current directory: `wazuh-indexer/` -* Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. +- Current directory: `wazuh-indexer/` +- Existing rpm package in `wazuh-indexer/artifacts/dist/rpm`, as a result of the _Build_ stage. ```console MIN_PKG_PATH="./artifacts" @@ -121,4 +222,3 @@ apt-get update apt-get install -y rpm2cpio rpm cpio bash scripts/assemble.sh -v 2.11.0 -p linux -a x64 -d rpm ``` - diff --git a/scripts/assemble.sh b/scripts/assemble.sh index dba52e4181d8c..fa9205b00f931 100755 --- a/scripts/assemble.sh +++ b/scripts/assemble.sh @@ -9,19 +9,25 @@ set -ex +# Minimum required plugins +# plugins=( +# "performance-analyzer" +# "opensearch-security" +# ) + plugins=( "alerting" # "opensearch-alerting" "opensearch-job-scheduler" - "opensearch-anomaly-detection" # requires "opensearch-job-scheduler" - "asynchronous-search" # "opensearch-asynchronous-search" + "opensearch-anomaly-detection" # Requires "opensearch-job-scheduler" + "asynchronous-search" # "opensearch-asynchronous-search" "opensearch-cross-cluster-replication" "geospatial" # "opensearch-geospatial" "opensearch-index-management" "opensearch-knn" "opensearch-ml-plugin" # "opensearch-ml" - "neural-search" # "opensearch-neural-search" + "neural-search" # "opensearch-neural-search" "opensearch-notifications-core" - "notifications" # "opensearch-notifications" requires "opensearch-notifications-core" + "notifications" # "opensearch-notifications". Requires "opensearch-notifications-core" "opensearch-observability" "performance-analyzer" # "opensearch-performance-analyzer" "opensearch-reports-scheduler" @@ -30,12 +36,14 @@ plugins=( "opensearch-sql-plugin" # "opensearch-sql" ) +# ==== +# Usage +# ==== function usage() { echo "Usage: $0 [args]" echo "" echo "Arguments:" echo -e "-v VERSION\t[Required] OpenSearch version." - echo -e "-q QUALIFIER\t[Optional] Version qualifier." echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'." echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'." echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'." @@ -43,200 +51,211 @@ function usage() { echo -e "-h help" } -while getopts ":h:v:q:o:p:a:d:" arg; do - case $arg in - h) - usage - exit 1 - ;; - v) - VERSION=$OPTARG - ;; - q) - QUALIFIER=$OPTARG - ;; - o) - OUTPUT=$OPTARG - ;; - p) - PLATFORM=$OPTARG - ;; - a) - ARCHITECTURE=$OPTARG - ;; - d) - DISTRIBUTION=$OPTARG - ;; - :) - echo "Error: -${OPTARG} requires an argument" +# ==== +# Parse arguments +# ==== +function parse_args() { + + while getopts ":h:v:o:p:a:d:" arg; do + case $arg in + h) + usage + exit 1 + ;; + v) + VERSION=$OPTARG + ;; + o) + OUTPUT=$OPTARG + ;; + p) + PLATFORM=$OPTARG + ;; + a) + ARCHITECTURE=$OPTARG + ;; + d) + DISTRIBUTION=$OPTARG + ;; + :) + echo "Error: -${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac + done + + if [ -z "$VERSION" ]; then + echo "Error: You must specify the OpenSearch version" usage exit 1 - ;; - ?) - echo "Invalid option: -${arg}" - exit 1 - ;; - esac -done + fi -if [ -z "$VERSION" ]; then - echo "Error: You must specify the OpenSearch version" - usage - exit 1 -fi + [ -z "$OUTPUT" ] && OUTPUT=artifacts -[ -z "$OUTPUT" ] && OUTPUT=artifacts + # Assemble distribution artifact + # see https://github.com/opensearch-project/OpenSearch/blob/main/settings.gradle#L34 for other distribution targets -# Assemble distribution artifact -# see https://github.com/opensearch-project/OpenSearch/blob/main/settings.gradle#L34 for other distribution targets + [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') + [ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) + [ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" -[ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}') -[ -z "$ARCHITECTURE" ] && ARCHITECTURE=$(uname -m) -[ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar" - -case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in + case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in linux-tar-x64 | darwin-tar-x64) PACKAGE="tar" EXT="tar.gz" - # TYPE="archives" TARGET="$PLATFORM-$PACKAGE" SUFFIX="$PLATFORM-x64" ;; linux-tar-arm64 | darwin-tar-arm64) PACKAGE="tar" EXT="tar.gz" - # TYPE="archives" TARGET="$PLATFORM-arm64-$PACKAGE" SUFFIX="$PLATFORM-arm64" ;; linux-deb-x64) PACKAGE="deb" EXT="deb" - # TYPE="packages" TARGET="deb" SUFFIX="amd64" ;; linux-deb-arm64) PACKAGE="deb" EXT="deb" - # TYPE="packages" TARGET="arm64-deb" SUFFIX="arm64" ;; linux-rpm-x64) PACKAGE="rpm" EXT="rpm" - # TYPE="packages" TARGET="rpm" SUFFIX="x86_64" ;; linux-rpm-arm64) PACKAGE="rpm" EXT="rpm" - # TYPE="packages" TARGET="arm64-rpm" SUFFIX="aarch64" ;; - # windows-zip-x64) - # PACKAGE="zip" - # EXT="zip" - # # TYPE="archives" - # TARGET="$PLATFORM-$PACKAGE" - # SUFFIX="$PLATFORM-x64" - # ;; - # windows-zip-arm64) - # PACKAGE="zip" - # EXT="zip" - # # TYPE="archives" - # TARGET="$PLATFORM-arm64-$PACKAGE" - # SUFFIX="$PLATFORM-arm64" - # ;; *) echo "Unsupported platform-distribution-architecture combination: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" exit 1 ;; -esac + esac +} -echo "Assembling OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" -# wazuh-indexer-min_4.9.0-1-x64_78fcc3db6a5b470294319e48b58c3d715bee39d1.rpm -ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*.$EXT") +# ==== +# Set up configuration files +# ==== +function add_configuration_files() { + # swap configuration files + cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ + cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options + cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml -# Create temporal directory and copy the min package there for extraction -TMP_DIR="${OUTPUT}/tmp/${TARGET}" -mkdir -p "$TMP_DIR" -cp "${OUTPUT}/dist/$ARTIFACT_BUILD_NAME" "${TMP_DIR}" + rm -r $PATH_CONF/security + rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml + # Remove symbolic links and bat files + find . -type l -exec rm -rf {} \; + find . -name "*.bat" -exec rm -rf {} \; +} + +# ==== +# Copy performance analyzer service file +# ==== +function enable_performance_analyzer() { + mkdir -p "${TMP_DIR}"/usr/lib/systemd/system + cp "distribution/packages/src/common/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system +} + +# ==== +# Move performance-analyzer-rca to its final location +# ==== +function enable_performance_analyzer_rca() { + local rca_src="${1}/plugins/opensearch-performance-analyzer/performance-analyzer-rca" + local rca_dest="${1}" + mv "${rca_src}" "${rca_dest}" +} + +# ==== +# Install plugins +# ==== +function install_plugins() { + # Install plugins from Maven repository + echo "Install plugins" + for plugin in "${plugins[@]}"; do + plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" + OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" + done +} + +# ==== +# Clean +# ==== +function clean() { + echo "Cleaning temporary ${TMP_DIR} folder" + rm -r "${OUTPUT}/tmp" + echo "After execution, shell path is $(pwd)" + # Store package's name to file. Used by GH Action. + echo "${package_name}" >"${OUTPUT}/artifact_name.txt" +} + +# ==== +# Tar assemble +# ==== function assemble_tar() { cd "${TMP_DIR}" PATH_CONF="./config" PATH_BIN="./bin" - # Step 1: extract + # Extract echo "Extract ${ARTIFACT_BUILD_NAME} archive" tar -zvxf "${ARTIFACT_BUILD_NAME}" cd "$(ls -d wazuh-indexer-*/)" - # Step 2: install plugins - echo "Install plugins" - for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" - "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" - done - - # Step 3: swap configuration files - cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ - cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options - cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml + # Install plugins + install_plugins + # Swap configuration files + add_configuration_files - rm -r $PATH_CONF/security - rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml - - # Step 4: pack + # Pack archive_name="wazuh-indexer-$(cat VERSION)" cd .. tar -cvf "${archive_name}-${SUFFIX}.${EXT}" "${archive_name}" cd ../../.. cp "${TMP_DIR}/${archive_name}-${SUFFIX}.${EXT}" "${OUTPUT}/dist/" - echo "Cleaning temporary ${TMP_DIR} folder" - rm -r "${TMP_DIR}" - echo "After execution, shell path is $(pwd)" + clean } - +# ==== +# RPM assemble +# ==== function assemble_rpm() { # Copy spec cp "distribution/packages/src/rpm/wazuh-indexer.rpm.spec" "${TMP_DIR}" # Copy performance analyzer service file - mkdir -p "${TMP_DIR}"/usr/lib/systemd/system - cp "distribution/packages/src/rpm/wazuh-indexer-performance-analyzer.service" "${TMP_DIR}"/usr/lib/systemd/system + enable_performance_analyzer cd "${TMP_DIR}" + local src_path="./usr/share/wazuh-indexer" PATH_CONF="./etc/wazuh-indexer" - PATH_BIN="./usr/share/wazuh-indexer/bin" + PATH_BIN="${src_path}/bin" # Extract min-package. Creates usr/, etc/ and var/ in the current directory echo "Extract ${ARTIFACT_BUILD_NAME} archive" - rpm2cpio "${ARTIFACT_BUILD_NAME}" | cpio -imdv - - # Install plugins from Maven repository - echo "Install plugins" - for plugin in "${plugins[@]}"; do - plugin_from_maven="org.opensearch.plugin:${plugin}:$VERSION.0" - OPENSEARCH_PATH_CONF=$PATH_CONF "${PATH_BIN}/opensearch-plugin" install --batch --verbose "${plugin_from_maven}" - done - - # Set up configuration files - cp $PATH_CONF/security/* $PATH_CONF/opensearch-security/ - cp $PATH_CONF/jvm.prod.options $PATH_CONF/jvm.options - cp $PATH_CONF/opensearch.prod.yml $PATH_CONF/opensearch.yml + rpm2cpio "${ARTIFACT_BUILD_NAME}" | cpio -imdv - rm -r $PATH_CONF/security - rm $PATH_CONF/jvm.prod.options $PATH_CONF/opensearch.prod.yml - - # Remove symbolic links and bat files - find . -type l -exec rm -rf {} \; - find . -name "*.bat" -exec rm -rf {} \; + # Install plugins + install_plugins + enable_performance_analyzer_rca ${src_path} + # Swap configuration files + add_configuration_files # Generate final package local topdir @@ -251,33 +270,88 @@ function assemble_rpm() { --define "_architecture ${SUFFIX}" \ ${spec_file} - # Move to the root folder, copy the package and clean. + # Move to the root folder, copy the package and clean. cd ../../.. package_name="wazuh-indexer-${version}-1.${SUFFIX}.${EXT}" cp "${TMP_DIR}/RPMS/${SUFFIX}/${package_name}" "${OUTPUT}/dist/" - - echo "Cleaning temporary ${TMP_DIR} folder" - rm -r "${TMP_DIR}" - echo "After execution, shell path is $(pwd)" - # Store package's name to file. Used by GH Action. - echo "${package_name}" > "${OUTPUT}/artifact_name.txt" + + clean } -case $SUFFIX.$EXT in - linux-arm64.tar.gz) - assemble_tar - ;; - linux-x64.tar.gz) +# ==== +# DEB assemble +# ==== +function assemble_deb() { + # Copy spec + cp "distribution/packages/src/deb/Makefile" "${TMP_DIR}" + cp "distribution/packages/src/deb/debmake_install.sh" "${TMP_DIR}" + chmod a+x "${TMP_DIR}/debmake_install.sh" + # Copy performance analyzer service file + enable_performance_analyzer + + cd "${TMP_DIR}" + local src_path="./usr/share/wazuh-indexer" + PATH_CONF="./etc/wazuh-indexer" + PATH_BIN="${src_path}/bin" + + # Extract min-package. Creates usr/, etc/ and var/ in the current directory + echo "Extract ${ARTIFACT_BUILD_NAME} archive" + ar xf "${ARTIFACT_BUILD_NAME}" data.tar.gz + tar zvxf data.tar.gz + + # Install plugins + install_plugins + enable_performance_analyzer_rca ${src_path} + # Swap configuration files + add_configuration_files + + # Generate final package + local version + version=$(cat ./usr/share/wazuh-indexer/VERSION) + debmake \ + --fullname "Wazuh Team" \ + --email "hello@wazuh.com" \ + --invoke debuild \ + --package wazuh-indexer \ + --native \ + --revision 1 \ + --upstreamversion "${version}" + + # Move to the root folder, copy the package and clean. + cd ../../.. + package_name="wazuh-indexer_${version}_${SUFFIX}.${EXT}" + # debmake creates the package one level above + cp "${TMP_DIR}/../${package_name}" "${OUTPUT}/dist/" + + clean +} + +# ==== +# Main function +# ==== +function main() { + parse_args "${@}" + + echo "Assembling wazuh-indexer for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE" + # wazuh-indexer-min_4.9.0-1-x64_78fcc3db6a5b470294319e48b58c3d715bee39d1.rpm + ARTIFACT_BUILD_NAME=$(ls "${OUTPUT}/dist/" | grep "wazuh-indexer-min.*.$EXT") + + # Create temporal directory and copy the min package there for extraction + TMP_DIR="${OUTPUT}/tmp/${TARGET}" + mkdir -p "$TMP_DIR" + cp "${OUTPUT}/dist/$ARTIFACT_BUILD_NAME" "${TMP_DIR}" + + case $PACKAGE in + tar) assemble_tar ;; - aarch64.rpm) - assemble_rpm - ;; - x86_64.rpm) + rpm) assemble_rpm ;; - amd64.deb) - ;; - arm64.deb) + deb) + assemble_deb ;; -esac + esac +} + +main "${@}" diff --git a/scripts/provision.sh b/scripts/provision.sh new file mode 100644 index 0000000000000..3cd155dc44fb5 --- /dev/null +++ b/scripts/provision.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Provision script for assembly of DEB packages + +# Install necessary packages +apt-get update -y && apt-get upgrade -y && apt-get install -y curl build-essential curl && + apt-get install -y debmake debhelper-compat && + apt-get install -y libxrender1 libxtst6 libasound2 libxi6 libgconf-2-4 && + apt-get install -y libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libatspi2.0-dev libxcomposite-dev libxdamage1 libxfixes3 libxfixes-dev libxrandr2 libgbm-dev libxkbcommon-x11-0 libpangocairo-1.0-0 libcairo2 libcairo2-dev libnss3 libnspr4 libnspr4-dev freeglut3 && + apt-get clean -y