Merge branch '4.9.0' into 135-amazon-security-lake-logstash

.gitignore

@@ -1,6 +1,9 @@
 # build files
 artifacts/
 
+.java
+.m2
+
 # intellij files
 .idea/
 *.iml

distribution/packages/src/deb/debian/rules

@@ -13,17 +13,20 @@
 #export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
 #export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
 
+SHELL != sh -c "command -v /bin/bash"
+.ONESHELL:
+
 %:
 	dh $@
 
+override_dh_strip_nondeterminism:
+	echo "Skipping dh_strip_nondeterminism"
+
+override_dh_fixperms:
+	echo "Skipping dh_fixperms"
+
 override_dh_builddeb:
 	dh_builddeb -- -Zgzip
 
 override_dh_gencontrol:
 	dh_gencontrol -- -DLicense=Apache-2.0
-
-#override_dh_auto_install:
-#	dh_auto_install -- prefix=/usr
-
-#override_dh_install:
-#	dh_install --list-missing -X.pyc -X.pyo

distribution/packages/src/deb/debmake_install.sh

@@ -12,17 +12,22 @@
 set -ex
 
 if [ -z "$1" ]; then
-    echo "Missing curdir path"
-    exit 1
+	echo "Missing curdir path"
+	exit 1
 fi
 
 curdir=$1
-product_dir=/usr/share/wazuh-indexer
-config_dir=/etc/wazuh-indexer
-data_dir=/var/lib/wazuh-indexer
-log_dir=/var/log/wazuh-indexer
-pid_dir=/run/wazuh-indexer
-buildroot=${curdir}/debian/wazuh-indexer
+
+name="wazuh-indexer"
+
+product_dir="/usr/share/${name}"
+config_dir="/etc/${name}"
+# data_dir="/var/lib/${name}"
+# log_dir="/var/log/${name}"
+pid_dir="/run/${name}"
+service_dir="/usr/lib/systemd/system"
+
+buildroot="${curdir}/debian/${name}"
 
 # Create necessary directories
 mkdir -p "${buildroot}"
@@ -31,13 +36,60 @@ mkdir -p "${buildroot}${product_dir}/plugins"
 
 # Install directories/files
 cp -a "${curdir}"/etc "${curdir}"/usr "${curdir}"/var "${buildroot}"/
-chmod -c 0755 "${buildroot}${product_dir}"/bin/*
-if [ -d "${buildroot}${product_dir}"/plugins/opensearch-security ]; then
-    chmod -c 0755 "${buildroot}${product_dir}"/plugins/opensearch-security/tools/*
+
+# General permissions for most of the package's files:
+find "${buildroot}" -type d -exec chmod 750 {} \;
+find "${buildroot}" -type f -exec chmod 640 {} \;
+
+# Permissions for the Systemd files
+systemd_files=()
+systemd_files+=("${buildroot}/${service_dir}/${name}.service")
+systemd_files+=("${buildroot}/${service_dir}/${name}-performance-analyzer.service")
+systemd_files+=("${buildroot}/${service_dir}/${name}-performance-analyzer.service")
+systemd_files+=("${buildroot}/etc/init.d/${name}")
+systemd_files+=("${buildroot}/usr/lib/sysctl.d/${name}.conf")
+systemd_files+=("${buildroot}/usr/lib/tmpfiles.d/${name}.conf")
+
+for i in "${systemd_files[@]}"; do
+	chmod -c 0644 "$i"
+done
+
+# Permissions for config files
+config_files=()
+config_files+=("${buildroot}/${config_dir}/log4j2.properties")
+config_files+=("${buildroot}/${config_dir}/jvm.options")
+config_files+=("${buildroot}/${config_dir}/opensearch.yml")
+
+for i in "${config_files[@]}"; do
+	chmod -c 0660 "$i"
+done
+
+# Plugin-related files
+if [ -e "${buildroot}/${config_dir}/opensearch-observability/observability.yml" ]; then
+	chmod -c 660 "${buildroot}/${config_dir}/opensearch-observability/observability.yml"
+fi
+
+if [ -e "${buildroot}/${config_dir}/opensearch-reports-scheduler/reports-scheduler.yml" ]; then
+	chmod -c 660 "${buildroot}/${config_dir}/opensearch-reports-scheduler/reports-scheduler.yml"
 fi
 
-# Change Permissions
-chmod -Rf a+rX,u+w,g-w,o-w "${buildroot}"/*
-chmod -c 660 "${buildroot}${config_dir}"/wazuh-template.json
+# Files that need other permissions
+chmod -c 440 "${buildroot}${product_dir}/VERSION"
+if [ -d "${buildroot}${product_dir}/plugins/opensearch-security" ]; then
+	chmod -c 0740 "${buildroot}${product_dir}"/plugins/opensearch-security/tools/*.sh
+fi
+
+binary_files=()
+binary_files+=("${buildroot}${product_dir}"/bin/*)
+binary_files+=("${buildroot}${product_dir}"/jdk/bin/*)
+binary_files+=("${buildroot}${product_dir}"/jdk/lib/jspawnhelper)
+binary_files+=("${buildroot}${product_dir}"/jdk/lib/modules)
+binary_files+=("${buildroot}${product_dir}"/performance-analyzer-rca/bin/*)
+
+for i in "${binary_files[@]}"; do
+	chmod -c 750 "$i"
+done
+
+chmod -c 660 "${buildroot}${config_dir}/wazuh-template.json"
 
 exit 0

distribution/packages/src/rpm/wazuh-indexer.rpm.spec

@@ -17,7 +17,7 @@
 %define _source_filedigest_algorithm 8
 %define _binary_filedigest_algorithm 8
 
-# Fixed in Fedora: 
+# Fixed in Fedora:
 # https://www.endpointdev.com/blog/2011/10/rpm-building-fedoras-sharedstatedir/
 %define _sharedstatedir /var/lib
 
@@ -43,32 +43,36 @@ ExclusiveArch: %{_architecture}
 AutoReqProv: no
 
 %description
-Wazuh indexer is a near real-time full-text search and analytics engine that 
-gathers security-related data into one platform. This Wazuh central component 
-indexes and stores alerts generated by the Wazuh server. Wazuh indexer can be 
-configured as a single-node or multi-node cluster, providing scalability and 
+Wazuh indexer is a near real-time full-text search and analytics engine that
+gathers security-related data into one platform. This Wazuh central component
+indexes and stores alerts generated by the Wazuh server. Wazuh indexer can be
+configured as a single-node or multi-node cluster, providing scalability and
 high availability.
 For more information, see: https://www.wazuh.com/
 
 %prep
 # No-op. We are using dir so no need to setup.
 
 %build
-# No-op. This is all pre-built Java. Nothing to do here.
+
+%define observability_plugin %( if [ -f %{_topdir}/etc/wazuh-indexer/opensearch-observability/observability.yml ]; then echo "1" ; else echo "0"; fi )
+%define reportsscheduler_plugin %( if [ -f %{_topdir}/etc/wazuh-indexer/opensearch-reports-scheduler/reports-scheduler.yml ]; then echo "1" ; else echo "0"; fi )
 
 %install
 set -e
 cd %{_topdir} && pwd
+
 # Create necessary directories
 mkdir -p %{buildroot}%{pid_dir}
 mkdir -p %{buildroot}%{product_dir}/plugins
+
 # Install directories/files
 cp -a etc usr var %{buildroot}
-chmod 0750 %{buildroot}%{product_dir}/bin/*
+chmod 0755 %{buildroot}%{product_dir}/bin/*
 if [ -d %{buildroot}%{product_dir}/plugins/opensearch-security ]; then
-    chmod 0640 %{buildroot}%{product_dir}/plugins/opensearch-security/tools/*
-    chmod 0740 %{buildroot}%{product_dir}/plugins/opensearch-security/tools/*.sh
+    chmod 0755 %{buildroot}%{product_dir}/plugins/opensearch-security/tools/*
 fi
+
 # Pre-populate the folders to ensure rpm build success even without all plugins
 mkdir -p %{buildroot}%{config_dir}/opensearch-observability
 mkdir -p %{buildroot}%{config_dir}/opensearch-reports-scheduler
@@ -81,6 +85,70 @@ fi
 if [ ! -f %{buildroot}%{data_dir}/performance_analyzer_enabled.conf ]; then
     echo 'true' > %{buildroot}%{data_dir}/performance_analyzer_enabled.conf
 fi
+
+# Build a filelist to be included in the %files section
+echo '%defattr(640, %{name}, %{name}, 750)' > filelist.txt
+find %{buildroot} -type d >> filelist.txt
+sed -i 's|%{buildroot}|%%dir |' filelist.txt
+find %{buildroot} -type f >> filelist.txt
+sed -i 's|%{buildroot}||' filelist.txt
+
+# The %install section gets executed under a dash shell,
+# which doesn't have array structures.
+# Below, we are building a list of directories
+# which will later be excluded from filelist.txt
+set -- "%%dir %{_sysconfdir}"
+set -- "$@" "%%dir %{_sysconfdir}/sysconfig"
+set -- "$@" "%%dir %{_sysconfdir}/init.d"
+set -- "$@" "%%dir /usr"
+set -- "$@" "%%dir /usr/lib"
+set -- "$@" "%%dir /usr/lib/systemd/system"
+set -- "$@" "%%dir /usr/lib/tmpfiles.d"
+set -- "$@" "%%dir /usr/share"
+set -- "$@" "%%dir /var"
+set -- "$@" "%%dir /var/lib"
+set -- "$@" "%%dir /var/log"
+set -- "$@" "%%dir /usr/lib/sysctl.d"
+set -- "$@" "%%dir /usr/lib/systemd"
+set -- "$@" "%%dir /usr/lib/systemd"
+set -- "$@" "%{_sysconfdir}/sysconfig/%{name}"
+set -- "$@" "%{config_dir}/log4j2.properties"
+set -- "$@" "%{config_dir}/jvm.options"
+set -- "$@" "%{config_dir}/opensearch.yml"
+set -- "$@" "%{config_dir}/wazuh-template.json"
+set -- "$@" "%{product_dir}/VERSION"
+set -- "$@" "%{product_dir}/plugins/opensearch-security/tools/.*\.sh"
+set -- "$@" "%{product_dir}/bin/.*"
+set -- "$@" "%{product_dir}/jdk/bin/.*"
+set -- "$@" "%{product_dir}/jdk/lib/jspawnhelper"
+set -- "$@" "%{product_dir}/jdk/lib/modules"
+set -- "$@" "%{product_dir}/performance-analyzer-rca/bin/.*"
+set -- "$@" "%{product_dir}/NOTICE.txt"
+set -- "$@" "%{product_dir}/README.md"
+set -- "$@" "%{product_dir}/LICENSE.txt"
+set -- "$@" "%{_prefix}/lib/systemd/system/%{name}.service"
+set -- "$@" "%{_prefix}/lib/systemd/system/%{name}-performance-analyzer.service"
+set -- "$@" "%{_sysconfdir}/init.d/%{name}"
+set -- "$@" "%{_sysconfdir}/sysconfig/%{name}"
+set -- "$@" "%{_prefix}/lib/sysctl.d/%{name}.conf"
+set -- "$@" "%{_prefix}/lib/tmpfiles.d/%{name}.conf"
+set -- "$@" "%%dir %{product_dir}/bin/opensearch-performance-analyzer"
+
+# Check if we are including the observability and reports scheduler
+# plugins
+if [ %observability_plugin -eq 1 ]; then
+    set -- "$@" "%{config_dir}/opensearch-observability/observability.yml"
+fi
+
+if [ %reportsscheduler_plugin -eq 1 ]; then
+    set -- "$@" "%{config_dir}/opensearch-reports-scheduler/reports-scheduler.yml"
+fi
+
+for i in "$@"
+do
+	sed -ri "\|^$i$|d" filelist.txt
+done
+
 # Change Permissions
 chmod -Rf a+rX,u+w,g-w,o-w %{buildroot}/*
 exit 0
@@ -107,6 +175,7 @@ exit 0
 set -e
 chown -R %{name}.%{name} %{config_dir}
 chown -R %{name}.%{name} %{log_dir}
+
 # Apply PerformanceAnalyzer Settings
 chmod a+rw /tmp
 if ! grep -q '## OpenSearch Performance Analyzer' %{config_dir}/jvm.options; then
@@ -152,47 +221,45 @@ if command -v systemctl >/dev/null && systemctl is-active %{name}-performance-an
 fi
 exit 0
 
-%files
-# Permissions
-%defattr(-, %{name}, %{name})
+%files -f %{_topdir}/filelist.txt
+%defattr(640, %{name}, %{name}, 750)
 
-# Root dirs/docs/licenses
-%dir %{product_dir}
 %doc %{product_dir}/NOTICE.txt
 %doc %{product_dir}/README.md
 %license %{product_dir}/LICENSE.txt
 
-# Config dirs/files
-%dir %{config_dir}
-%{config_dir}/jvm.options.d
-%{config_dir}/opensearch-*
-%config(noreplace) %{config_dir}/opensearch.yml
-%config(noreplace) %{config_dir}/jvm.options
-%config(noreplace) %{config_dir}/log4j2.properties
-%config(noreplace) %{data_dir}/rca_enabled.conf
-%config(noreplace) %{data_dir}/performance_analyzer_enabled.conf
-
 # Service files
 %attr(0644, root, root) %{_prefix}/lib/systemd/system/%{name}.service
 %attr(0644, root, root) %{_prefix}/lib/systemd/system/%{name}-performance-analyzer.service
 %attr(0644, root, root) %{_sysconfdir}/init.d/%{name}
-%attr(0644, root, root) %config(noreplace) %{_sysconfdir}/sysconfig/%{name}
 %attr(0644, root, root) %config(noreplace) %{_prefix}/lib/sysctl.d/%{name}.conf
 %attr(0644, root, root) %config(noreplace) %{_prefix}/lib/tmpfiles.d/%{name}.conf
 
-# Main dirs
-%{product_dir}/bin
-%{product_dir}/jdk
-%{product_dir}/lib
-%{product_dir}/modules
-%{product_dir}/performance-analyzer-rca
-%{product_dir}/plugins
-%{log_dir}
-%{pid_dir}
-%dir %{data_dir}
-
-# Wazuh additional files
+
+# Configuration files
+%config(noreplace) %attr(0660, root, %{name}) "%{_sysconfdir}/sysconfig/%{name}"
+%config(noreplace) %attr(660, %{name}, %{name}) %{config_dir}/log4j2.properties
+%config(noreplace) %attr(660, %{name}, %{name}) %{config_dir}/jvm.options
+%config(noreplace) %attr(660, %{name}, %{name}) %{config_dir}/opensearch.yml
+
+
+%if %observability_plugin
+%config(noreplace) %attr(660, %{name}, %{name}) %{config_dir}/opensearch-observability/observability.yml
+%endif
+
+%if %reportsscheduler_plugin
+%config(noreplace) %attr(660, %{name}, %{name}) %{config_dir}/opensearch-reports-scheduler/reports-scheduler.yml
+%endif
+
+
+# Files that need other permissions
 %attr(440, %{name}, %{name}) %{product_dir}/VERSION
+%attr(740, %{name}, %{name}) %{product_dir}/plugins/opensearch-security/tools/*.sh
+%attr(750, %{name}, %{name}) %{product_dir}/bin/*
+%attr(750, %{name}, %{name}) %{product_dir}/jdk/bin/*
+%attr(750, %{name}, %{name}) %{product_dir}/jdk/lib/jspawnhelper
+%attr(750, %{name}, %{name}) %{product_dir}/jdk/lib/modules
+%attr(750, %{name}, %{name}) %{product_dir}/performance-analyzer-rca/bin/*
 %attr(660, %{name}, %{name}) %{config_dir}/wazuh-template.json
 
 %changelog

ecs/generate.sh

@@ -27,6 +27,7 @@ generate_mappings() {
     --subset "$IN_FILES_DIR/subset.yml" \
     --template-settings "$IN_FILES_DIR/template-settings.json" \
     --template-settings-legacy "$IN_FILES_DIR/template-settings-legacy.json" \
+    --mapping-settings "$IN_FILES_DIR/mapping-settings.json" \
     --out "$OUT_DIR" || exit 1
 
   # Replace "match_only_text" type (not supported by OpenSearch) with "text"

ecs/vulnerability-detector/event-generator/event_generator.py

@@ -163,15 +163,16 @@ def generate_random_vulnerability():
             'temporal': round(random.uniform(0, 10), 1),
             'version': round(random.uniform(0, 10), 1)
         },
-        'severity': random.choice(['low', 'medium', 'high', 'critical'])
+        'severity': random.choice(['Low', 'Medium', 'High', 'Critical'])
     }
     return vulnerability
 
 
 def generate_random_wazuh():
     wazuh = {
         'cluster': {
-            'name': f'wazuh-cluster-{random.randint(0,10)}'
+            'name': f'wazuh-cluster-{random.randint(0,10)}',
+            'node': f'wazuh-cluster-node-{random.randint(0,10)}'
         }
     }
     return wazuh
@@ -186,7 +187,7 @@ def generate_random_data(number):
             'ecs': {'version': '1.7.0'},
             # 'event': generate_random_event(),
             'host': generate_random_host(),
-            'labels': generate_random_labels(),
+            # 'labels': generate_random_labels(),
             'message': f'message{random.randint(0, 99999)}',
             'package': generate_random_package(),
             'tags': generate_random_tags(),

ecs/vulnerability-detector/fields/custom/wazuh.yml

@@ -8,4 +8,9 @@
       type: keyword
       level: custom
       description: >
-        Wazuh cluster name.
+        Wazuh cluster name.
+    - name: cluster.node
+      type: keyword
+      level: custom
+      description: >
+        Wazuh cluster node name.

ecs/vulnerability-detector/fields/mapping-settings.json

@@ -0,0 +1,4 @@
+{
+    "dynamic": "strict",
+    "date_detection": false
+}