feat: add hook for schemathesis tests (#513)

* feat: add hook for schemathesis tests * feat: add put patch delete method, refactor * fix: change patch to delete * feat: move test-fuzzer-api to new folder * fix: uncomment code * feat: update max-response-time * fix: syntax --------- Co-authored-by: Dee <[email protected]>
input-output-hk · May 29, 2024 · c8427e5 · c8427e5
1 parent 09737e3
commit c8427e5
Show file tree

Hide file tree

Showing 8 changed files with 107 additions and 61 deletions.
diff --git a/.github/workflows/generate-allure-report.yml b/.github/workflows/generate-allure-report.yml
@@ -53,7 +53,7 @@ jobs:
         if: always()
         continue-on-error: true
         with:
-          earthfile: ./catalyst-gateway/tests/
+          earthfile: ./catalyst-gateway/tests/schemathesis_tests
           flags: --allow-privileged
           targets: test-fuzzer-api
           target_flags:

diff --git a/.gitignore b/.gitignore
@@ -107,3 +107,6 @@ dev-catalyst-voice-9f78f27c6bc5.json
 # Specifically exclude it in the directory it appears, if its required.
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock
+
+#Hypothesis
+.hypothesis
diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile
@@ -1,64 +1,5 @@
 VERSION 0.8
 
-package-schemathesis:
-    FROM python:3.12-alpine3.19
-
-    ARG tag="latest"
-    ARG max_examples=1000
-    # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465
-    ARG max_response_time=1000
-    ARG wait_for_schema=15
-    ARG workers=2
-    ARG schema_version=30
-    ARG openapi_spec
-
-    RUN apk add --no-cache gcc musl-dev
-    RUN python -m pip install schemathesis==3.27.1
-    RUN mkdir /results
-
-    VOLUME /results
-    ENTRYPOINT st run --checks all $openapi_spec \
-                    --workers=$workers \
-                    --wait-for-schema=$wait_for_schema \
-                    --max-response-time=$max_response_time \
-                    --hypothesis-max-examples=$max_examples \
-                    --data-generation-method=all \
-                    --skip-deprecated-operations \
-                    --force-schema-version=$schema_version \
-                    --show-trace \
-                    --force-color  \
-                    --junit-xml=/results/junit-report.xml \
-                    --cassette-path=/results/cassette.yaml
-
-    SAVE IMAGE schemathesis:$tag
-
-# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs
-test-fuzzer-api:
-    FROM earthly/dind:alpine-3.19
-    RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784
-    RUN apk add yq zstd
-    ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json"
-
-    COPY schemathesis-docker-compose.yml .
-    WITH DOCKER \
-        --compose schemathesis-docker-compose.yml \
-        --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \
-        --load event-db:latest=(../event-db+build) \
-        --load cat-gateway:latest=(../+package-cat-gateway) \
-        --service event-db \
-        --service cat-gateway \
-        --allow-privileged
-        RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \
-            docker-compose logs cat-gateway > ./cat-gateway.log && zstd -9 cat-gateway.log && \
-            docker cp st:/results/junit-report.xml junit-report.xml && \
-            docker cp st:/results/cassette.yaml cassette.yaml
-    END
-    WAIT
-        SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml
-        SAVE ARTIFACT cat-gateway.log.zst AS LOCAL cat-gateway.log.zst
-        SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml
-    END
-
 # test-lint-openapi - OpenAPI linting from an artifact
 # testing whether the OpenAPI generated during build stage follows good practice.
 test-lint-openapi:

diff --git a/catalyst-gateway/tests/api_tests/.gitignore b/catalyst-gateway/tests/api_tests/.gitignore
@@ -1,2 +1,3 @@
 __pycache__
-junit-report.xml
+junit-report.xml
+.hypothesis/
diff --git a/catalyst-gateway/tests/schemathesis_tests/.gitignore b/catalyst-gateway/tests/schemathesis_tests/.gitignore
@@ -0,0 +1,3 @@
+__pycache__
+junit-report.xml
+.hypothesis/
diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile
@@ -0,0 +1,62 @@
+VERSION 0.8
+
+package-schemathesis:
+    FROM python:3.12-alpine3.19
+
+    ARG tag="latest"
+    ARG max_examples=1000
+    # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465
+    ARG max_response_time=3000
+    ARG wait_for_schema=15
+    ARG workers=2
+    ARG schema_version=30
+    ARG openapi_spec
+
+    RUN apk add --no-cache gcc musl-dev
+    RUN python -m pip install schemathesis==3.27.1
+    RUN mkdir /results
+    COPY ./hooks/hooks.py .    
+
+    VOLUME /results
+    ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \
+                    && st run --checks all $openapi_spec \
+                    --workers=$workers \
+                    --wait-for-schema=$wait_for_schema \
+                    --max-response-time=$max_response_time \
+                    --hypothesis-max-examples=$max_examples \
+                    --data-generation-method=all \
+                    --skip-deprecated-operations \
+                    --force-schema-version=$schema_version \
+                    --show-trace \
+                    --force-color  \
+                    --junit-xml=/results/junit-report.xml \
+                    --cassette-path=/results/cassette.yaml
+
+    SAVE IMAGE schemathesis:$tag
+
+# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs
+test-fuzzer-api:
+    FROM earthly/dind:alpine-3.19
+    RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784
+    RUN apk add yq zstd
+    ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json"
+
+    COPY schemathesis-docker-compose.yml .
+    WITH DOCKER \
+        --compose schemathesis-docker-compose.yml \
+        --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \
+        --load event-db:latest=(../../event-db+build) \
+        --load cat-gateway:latest=(../../+package-cat-gateway) \
+        --service event-db \
+        --service cat-gateway \
+        --allow-privileged
+        RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \
+            docker-compose logs cat-gateway > ./cat-gateway.log && zstd -9 cat-gateway.log && \
+            docker cp st:/results/junit-report.xml junit-report.xml && \
+            docker cp st:/results/cassette.yaml cassette.yaml
+    END
+    WAIT
+        SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml
+        SAVE ARTIFACT cat-gateway.log.zst AS LOCAL cat-gateway.log.zst
+        SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml
+    END
diff --git a/catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py b/catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py
@@ -0,0 +1,36 @@
+import copy
+import schemathesis
+import schemathesis.schemas
+from typing import Any, Dict
+
+@schemathesis.hook
+def before_load_schema(
+    context: schemathesis.hooks.HookContext,
+    raw_schema: Dict[str, Any],
+    ) -> None:
+    paths = dict(raw_schema["paths"])
+    for endpoint in paths.keys(): 
+        endpoint_data = dict(raw_schema["paths"][endpoint])
+
+        # Get first method data as reference
+        ref_info = next(iter(endpoint_data.values()))
+
+        # Create new method data using reference data, replacing 'responses' field
+        new_info = copy.deepcopy(ref_info)
+        new_info["responses"] = {"405": {"description": "method not allowed"}}
+
+        # Update endpoint if a method is not declared
+        if "get" not in endpoint_data:
+            endpoint_data.update([("get", new_info),])
+        if "post" not in endpoint_data:
+            endpoint_data.update([("post", new_info),])
+        if "put" not in endpoint_data:
+            endpoint_data.update([("put", new_info),])
+        if "patch" not in endpoint_data:
+            endpoint_data.update([("patch", new_info),])
+        if "delete" not in endpoint_data:
+            endpoint_data.update([("delete", new_info),])
+
+        # Update endpoint
+        raw_schema["paths"][endpoint] = endpoint_data
+
diff --git a/...way/tests/schemathesis-docker-compose.yml → ...sis_tests/schemathesis-docker-compose.yml b/...way/tests/schemathesis-docker-compose.yml → ...sis_tests/schemathesis-docker-compose.yml