Merge branch '4.9.0' into 62-add-docker-compose-env

Signed-off-by: Álex Ruiz <[email protected]>

.github/workflows/build.yml

@@ -0,0 +1,62 @@
+name: Build slim packages
+
+# This workflow runs when any of the following occur:
+  # - Run manually
+on:
+  workflow_dispatch:
+
+
+# Used to run locally using https://github.com/nektos/act
+env:
+  ACT: 
+  VERSION: 2.11.0
+  SNAPSHOT: false
+  PLATFORM: linux
+  BUILD: bash scripts/build.sh
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    # Permissions to upload the package
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+        matrix:
+          # act is resource-heavy. Avoid running parallel builds with it:
+          # DISTRIBUTION: [ rpm ]
+          # ARCHITECTURE: [ x64 ]
+          DISTRIBUTION: [ tar, rpm, deb ]
+          ARCHITECTURE: [ x64, arm64 ]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v3
+      with:
+        distribution: temurin
+        java-version: 11
+
+    - name: Setup Gradle
+      uses: gradle/[email protected]
+
+    - name: Execute build script
+      run: |
+        $BUILD -v $VERSION -s $SNAPSHOT -p $PLATFORM -a ${{ matrix.ARCHITECTURE }} -d ${{ matrix.DISTRIBUTION }}
+
+    # The package name is stored in the artifacts/artifact_name.txt file
+    - name: Read package name
+      id: package_name
+      run: |
+        echo $(ls -la)
+        echo "package_name=$(cat artifacts/artifact_name.txt)" >> $GITHUB_OUTPUT
+        echo "$(cat artifacts/artifact_name.txt)"
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ steps.package_name.outputs.package_name }}
+        path: artifacts/dist/${{ steps.package_name.outputs.package_name }}
+        if-no-files-found: error
+
+  # assemble:
+  # release:

.gitignore

@@ -1,3 +1,4 @@
+# build files
 artifacts/
 
 # intellij files

ecs/README.md

@@ -5,36 +5,72 @@ This script generates the ECS mappings for the Wazuh indices.
 ### Requirements
 
 - ECS repository clone. The script is meant to be launched from the root level of that repository.
-- Python 3.6 or higher
-- jq
+- `Python` 3.6 or higher + `venv` module
+- `jq`
 
-### Folder structrue
+### Folder structure
 
 There is a folder for each module. Inside each folder, there is a `fields` folder with the required
 files to generate the mappings. These are the inputs for the ECS generator.
 
 ### Usage
 
-**Copy the `generate.sh` script to the root level of the ECS repository.**
+1. Get a copy of the ECS repository at the same level as the `wazuh-indexer` repo:
 
-Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments,
-plus 2 optional arguments to upload the mappings to the Wazuh indexer (using **composable** indexes).
+    ```console
+    git clone [email protected]:elastic/ecs.git
+    ```
 
-```plaintext
-Usage: ./generate.sh <ECS_VERSION> <INDEXER_SRC> <MODULE> [--upload <URL>]
-  * ECS_VERSION: ECS version to generate mappings for
-  * INDEXER_SRC: Path to the wazuh-indexer repository
-  * MODULE: Module to generate mappings for
-  * --upload <URL>: Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200
-Example: ./generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200
-```
+2. Install the dependencies:
 
-For example, to generate the mappings for the `vulnerability-detector` module using the
-ECS version `v8.10.0` and the Wazuh indexer in path `~/wazuh/wazuh-indexer`:
+    ```console
+    cd ecs
+    python3 -m venv env
+    source env/bin/activate
+    pip install -r scripts/requirements.txt
+    ```
 
-```bash
-./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector
-```
+2. Copy the `generate.sh` script to the root level of the ECS repository.
+
+    ```console
+    cp generate.sh ../../ecs
+    cd ../../ecs
+    bash generate.sh
+    ```
+
+    Expected output:
+    ```
+    Usage: generate.sh <ECS_VERSION> <INDEXER_SRC> <MODULE> [--upload <URL>]
+      * ECS_VERSION: ECS version to generate mappings for
+      * INDEXER_SRC: Path to the wazuh-indexer repository
+      * MODULE: Module to generate mappings for
+      * --upload <URL>: Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200
+    Example: generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200
+    ```
+
+3. Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments,
+plus 2 optional arguments to upload the mappings to the `wazuh-indexer`. Both, composable and legacy mappings
+are generated. For example, to generate the mappings for the `vulnerability-detector` module using the
+    ECS version `v8.10.0` and assuming that path of this repository is `~/wazuh/wazuh-indexer`:
+
+    ```bash
+    ./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector
+    ```
+
+    The tool will output the folder where they have been generated.
+
+    ```console
+    Loading schemas from git ref v8.10.0
+    Running generator. ECS version 8.10.0
+    Replacing "match_only_text" type with "text"
+    Mappings saved to ~/wazuh/wazuh-indexer/ecs/vulnerability-detector/mappings/v8.10.0
+    ```
+
+4. When you are done. Exit the virtual environment.
+
+    ```console
+    deactivate
+    ```
 
 ### Output
 
@@ -48,8 +84,8 @@ For our use case, the most important files are under `mappings/<ECS_VERSION>/gen
 The original output is `template.json`, which is not compatible with OpenSearch by default. In order
 to make this template compatible with OpenSearch, the following changes are made:
 
-- the `order` property is renamed to `priority`.
-- the `mappings` and `settings` properties are nested under the `template` property.
+- The `order` property is renamed to `priority`.
+- The `mappings` and `settings` properties are nested under the `template` property.
 
 The script takes care of these changes automatically, generating the `opensearch-template.json` file as a result.
 
@@ -62,7 +98,7 @@ curl -u admin:admin -k -X PUT "https://indexer:9200/_index_template/wazuh-vulner
 ```
 
 Notes:
-- PUT and POST are interchangable.
+- PUT and POST are interchangeable.
 - The name of the index template does not matter. Any name can be used.
 - Adjust credentials and URL accordingly.
 
@@ -95,7 +131,7 @@ The script will generate a JSON file with the events, and will also ask whether
 indexer. If the upload option is selected, the script will ask for the indexer URL and port, credentials,
 and index name.
 
-The script uses log file. Check it out for debugging or additonal information.
+The script uses log file. Check it out for debugging or additional information.
 
 #### References
 

scripts/build.sh

@@ -70,15 +70,19 @@ fi
 
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 
+echo "Creating output directory $OUTPUT/maven/org/opensearch if it doesn't already exist"
 mkdir -p $OUTPUT/maven/org/opensearch
 
 # Build project and publish to maven local.
+echo "Building and publishing OpenSearch project to Maven Local"
 ./gradlew publishToMavenLocal -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Publish to existing test repo, using this to stage release versions of the artifacts that can be released from the same build.
+echo "Publishing OpenSearch to Test Repository"
 ./gradlew publishNebulaPublicationToTestRepository -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Copy maven publications to be promoted
+echo "Copying Maven publications to $OUTPUT/maven/org"
 cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org
 
 # Assemble distribution artifact
@@ -103,6 +107,20 @@ case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in
         TARGET="$PLATFORM-arm64-$PACKAGE"
         SUFFIX="$PLATFORM-arm64"
         ;;
+    linux-deb-x64)
+        PACKAGE="deb"
+        EXT="deb"
+        TYPE="packages"
+        TARGET="deb"
+        SUFFIX="amd64"
+        ;;
+    linux-deb-arm64)
+        PACKAGE="deb"
+        EXT="deb"
+        TYPE="packages"
+        TARGET="arm64-deb"
+        SUFFIX="arm64"
+        ;;
     linux-rpm-x64)
         PACKAGE="rpm"
         EXT="rpm"
@@ -142,20 +160,10 @@ echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
 ./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Copy artifact to dist folder in bundle build output
+echo "Copying artifact to ${OUTPUT}/dist"
 [[ "$SNAPSHOT" == "true" ]] && IDENTIFIER="-SNAPSHOT"
 ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$SUFFIX.$EXT"`
+# [WAZUH] Used by the GH workflow to upload the artifact
+echo "$ARTIFACT_BUILD_NAME" > "$OUTPUT/artifact_name.txt"
 mkdir -p "${OUTPUT}/dist"
 cp distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME
-
-echo "Building core plugins..."
-mkdir -p "${OUTPUT}/core-plugins"
-cd plugins
-../gradlew assemble -Dbuild.snapshot="$SNAPSHOT" -Dbuild.version_qualifier=$QUALIFIER
-cd ..
-for plugin in plugins/*; do
-  PLUGIN_NAME=$(basename "$plugin")
-  if [ -d "$plugin" ] && [ "examples" != "$PLUGIN_NAME" ]; then
-    PLUGIN_ARTIFACT_BUILD_NAME=`ls "$plugin"/build/distributions/ | grep "$PLUGIN_NAME.*$IDENTIFIER.zip"`
-    cp "$plugin"/build/distributions/"$PLUGIN_ARTIFACT_BUILD_NAME" "${OUTPUT}"/core-plugins/"$PLUGIN_ARTIFACT_BUILD_NAME"
-  fi
-done