debug

.github/workflows/_build-package.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with: 
-          python-version: 3.11
+          python-version: 3.12
       - name: Install build dependencies
         run: pip install --no-cache-dir -U pip .['build']
       - name: Build package

.github/workflows/_integration-tests.yml

@@ -1,22 +1,46 @@
 name: integration-tests
 on: 
   workflow_call:
-    secrets:
-      DOCKER_TOKEN:
-        required: true
 jobs:
-  integration-tests:
-    name: Run integration tests
+  test-alert-endpoints:
+    name: Test alert endpoints
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Install dependencies
         run: pip install --no-cache-dir -U pip .['test']
-      - name: Docker login
-        run: docker login -u kamforka -p ${{ secrets.DOCKER_TOKEN }}
       - name: Run integration tests
-        run: scripts/ci.py --test
+        run: pytest -v tests/test_alert_endpoint.py
+
+  test-case-endpoints:
+    name: Test case endpoints
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - name: Install dependencies
+        run: pip install --no-cache-dir -U pip .['test']
+      - name: Run integration tests
+        run: pytest -v tests/test_case_endpoint.py
+
+  test-other-endpoints:
+    name: Test other endpoints
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - name: Install dependencies
+        run: pip install --no-cache-dir -U pip .['test']
+      - name: Run integration tests
+        run: |
+          pytest -v --ignore=tests/test_alert_endpoint.py --ignore=tests/test_case_endpoint.py

.github/workflows/_static-checks.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}

.github/workflows/_upload-package.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with: 
-          python-version: 3.11
+          python-version: 3.12
       - name: Install build dependencies
         run: pip install --no-cache-dir -U pip .['build']
       - name: Upload to PyPI

.github/workflows/integrator.yml

@@ -0,0 +1,62 @@
+name: integrator-image
+on: 
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  changes:
+    name: Change detection
+    runs-on: ubuntu-latest
+    outputs:
+      integrator: ${{ steps.filter.outputs.integrator }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            integrator:
+              - 'docker/thehive4py-integrator/**'
+  build:
+    name: Build and push
+    needs: changes
+    if: ${{ needs.changes.outputs.integrator == 'true' }}
+    runs-on: ubuntu-latest
+    env:
+      INTEGRATOR_BUILD_CTX: docker/thehive4py-integrator
+      INTEGRATOR_IMAGE_NAME: kamforka/thehive4py-integrator
+      THEHIVE_VERSION: 5.3.0
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set variables
+        id: variables
+        run: |
+          echo "integrator_image_fullname=$INTEGRATOR_IMAGE_NAME:thehive-$THEHIVE_VERSION" >> "$GITHUB_OUTPUT"
+          echo "integrator_image_fullname_with_hash=$INTEGRATOR_IMAGE_NAME:thehive-$THEHIVE_VERSION-$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: kamforka
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ env.INTEGRATOR_BUILD_CTX }}
+          platforms: linux/amd64,linux/arm64
+          # push: ${{ github.ref == 'refs/heads/main' }}
+          push: true
+          tags: ${{ steps.variables.outputs.integrator_image_fullname }},${{ steps.variables.outputs.integrator_image_fullname_with_hash}}
+          build-args: |
+            THEHIVE_VERSION=${{ env.THEHIVE_VERSION }}

.github/workflows/main-cicd.yml

@@ -10,8 +10,6 @@ jobs:
     uses: ./.github/workflows/_static-checks.yml
   integration-tests:
     uses: ./.github/workflows/_integration-tests.yml
-    secrets:
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
   build-package:
     uses: ./.github/workflows/_build-package.yml
   upload-package:

README.md

@@ -258,7 +258,7 @@ If you are a first time contributor to github projects please make yourself comf
 Navigate to the cloned repository's directory and install the package with development extras using pip:
 
 ```
-pip install -e '.[dev]'
+pip install -e .[dev]
 ```
 
 This command installs the package in editable mode (`-e`) and includes additional development dependencies.
@@ -304,14 +304,37 @@ With pre-commit hooks in place, your changes will be automatically validated for
 
 ## Testing
 
+> IMPORTANT NOTE: Since TheHive 5.3 the licensing constraints has been partially lifted therefore a public integrator image is available for running tests both locally and in github.
+
 `thehive4py` primarily relies on integration tests, which are designed to execute against a live TheHive 5.x instance. These tests ensure that the library functions correctly in an environment closely resembling real-world usage.
 
-However, due to licensing constraints with TheHive 5.x, the integration tests are currently not available for public or local use.
+### Test requirements
+
+Since the test suite relies on the existence of a live TheHive docker container a local docker engine installation is a must.
+If you are unfamiliar with docker please check out the [official documentation][get-docker].
+
+### Test setup
+
+The test suite relies on a self-contained TheHive image which we call: [thehive4py-integrator]. 
+This image contains everything that is needed to run a barebone TheHive instance. 
+
+The test suite uses this image to create a container locally with the predefined name `thehive4py-integration-tester` which will act as a unique id. 
+The container will expose TheHive on a random port to make sure it causes no conflicts for any other containers which expose ports.
+The suite can identify this random port by querying the container info based on the predefined name. 
+During the very first test run the container will take a longer time to initialize due to the ElasticSearch and TheHive startups.
+Once TheHive is responsive the suite will initialize the instance with a setup required by the tests (e.g.: test users, organisations, etc.).
+After a successful initialization the 
+
+### Testing locally
+To execute the whole test suite locally one can use the `scripts/ci.py` utility script like:
 
-To ensure code quality and prevent broken code from being merged, a private image is available for the integration-test workflow. This means that any issues should be detected and addressed during the PR phase.
+    ./scripts/ci.py --test
 
-The project is actively working on a solution to enable developers to run integration tests locally, providing a more accessible and comprehensive testing experience.
+Note however that the above will execute the entire test suite which can take several minutes to complete.
+In case one wants to execute only a portion of the test suite then the easiest workaround is to use `pytest` and pass the path to the specific test module. For example to only execute tests for the alert endpoints one can do:
 
-While local testing is in development, relying on the automated PR checks ensures the reliability and quality of the `thehive4py` library.
+    pytest -v tests/test_alert_endpoint.py
 
-[query-api-docs]: https://docs.strangebee.com/thehive/api-docs/#operation/Query%20API
+[get-docker]: https://docs.docker.com/get-docker/
+[query-api-docs]: https://docs.strangebee.com/thehive/api-docs/#operation/Query%20API
+[thehive4py-integrator]: https://hub.docker.com/repository/docker/kamforka/thehive4py-integrator/general

docker/thehive4py-integrator/Dockerfile

@@ -0,0 +1,55 @@
+FROM alpine:3.17 as base
+
+# BUILDER STAGE
+FROM base as builder
+
+ARG ES_VERSION=7.17.19
+ARG THEHIVE_VERSION=5.3.0
+
+RUN apk update && apk upgrade && apk add curl
+
+## ES DOWNLOAD
+ARG ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz
+
+RUN curl -Lo /tmp/elasticsearch.tgz ${ES_DOWNLOAD_URL} \
+    && tar -xzf /tmp/elasticsearch.tgz -C /tmp \
+    && mv /tmp/elasticsearch-${ES_VERSION} /tmp/elasticsearch
+
+## THEHIVE DOWNLOAD
+ARG THEHIVE_DOWNLOAD_URL=https://archives.strangebee.com/zip/thehive-${THEHIVE_VERSION}-1.zip 
+
+RUN curl -Lo /tmp/thehive.zip ${THEHIVE_DOWNLOAD_URL}
+RUN unzip -qo /tmp/thehive.zip -d /tmp \
+    && mv /tmp/thehive-${THEHIVE_VERSION}-1 /tmp/thehive
+
+# FINAL STAGE
+FROM base
+RUN apk update && apk upgrade && apk add --no-cache openjdk11-jre-headless bash su-exec curl jq
+
+## ES SETUP
+COPY --from=builder /tmp/elasticsearch /usr/share/elasticsearch
+COPY configs/elasticsearch.yml /usr/share/elasticsearch/config/elasticsearch.yml
+
+RUN adduser -u 1000 -g 1000 -Dh /usr/share/elasticsearch elasticsearch \
+    && mkdir -p /usr/share/elasticsearch/data \
+    && chown -R elasticsearch:elasticsearch /usr/share/elasticsearch \
+    && rm -rf /usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64
+
+## THEHIVE SETUP
+COPY --from=builder /tmp/thehive /opt/thehive/
+COPY configs/thehive.conf /opt/thehive/conf/application.conf
+
+RUN adduser -u 1001 -g 1001 -Dh /opt/thehive thehive \
+    && mkdir /var/log/thehive \
+    && chown -R thehive:thehive /opt/thehive /var/log/thehive
+
+
+## ENTRYPOINT
+COPY entrypoint.sh /
+RUN chmod +x entrypoint.sh
+
+RUN /entrypoint.sh warmup
+
+EXPOSE 9000
+ENTRYPOINT /entrypoint.sh startup
+HEALTHCHECK --start-period=60s CMD curl -f http://localhost:9000/api/status

docker/thehive4py-integrator/configs/elasticsearch.yml

@@ -0,0 +1,7 @@
+http.host: 0.0.0.0
+transport.host: 0.0.0.0
+discovery.type: single-node
+cluster.name: thehive4py
+xpack.security.enabled: false
+xpack.ml.enabled: false
+script.allowed_types: "inline,stored"

docker/thehive4py-integrator/configs/thehive.conf

@@ -0,0 +1,22 @@
+play.http.secret.key="supersecret"
+play.http.parser.maxDiskBuffer: 20MB
+
+db {
+  provider: janusgraph
+  janusgraph {
+    storage {
+      backend: berkeleyje
+      directory: /opt/thehive/db
+    }
+
+    index.search {
+      backend: elasticsearch
+      hostname: ["127.0.0.1"]
+    }
+  }
+}
+
+storage {
+  provider: localfs
+  localfs.location: /opt/thehive/data
+}