diff --git a/.github/workflows/generate-allure-report.yml b/.github/workflows/generate-allure-report.yml index fec4a0ca2d..89ec6926f5 100644 --- a/.github/workflows/generate-allure-report.yml +++ b/.github/workflows/generate-allure-report.yml @@ -53,7 +53,7 @@ jobs: if: always() continue-on-error: true with: - earthfile: ./catalyst-gateway/tests/ + earthfile: ./catalyst-gateway/tests/schemathesis_tests flags: --allow-privileged targets: test-fuzzer-api target_flags: diff --git a/.gitignore b/.gitignore index d6aba0034f..ee537b0c12 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ dev-catalyst-voice-9f78f27c6bc5.json # Specifically exclude it in the directory it appears, if its required. # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock + +#Hypothesis +.hypothesis diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index 2713223062..8e3c08596d 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -1,64 +1,5 @@ VERSION 0.8 -package-schemathesis: - FROM python:3.12-alpine3.19 - - ARG tag="latest" - ARG max_examples=1000 - # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 - ARG max_response_time=1000 - ARG wait_for_schema=15 - ARG workers=2 - ARG schema_version=30 - ARG openapi_spec - - RUN apk add --no-cache gcc musl-dev - RUN python -m pip install schemathesis==3.27.1 - RUN mkdir /results - - VOLUME /results - ENTRYPOINT st run --checks all $openapi_spec \ - --workers=$workers \ - --wait-for-schema=$wait_for_schema \ - --max-response-time=$max_response_time \ - --hypothesis-max-examples=$max_examples \ - --data-generation-method=all \ - --skip-deprecated-operations \ - --force-schema-version=$schema_version \ - --show-trace \ - --force-color \ - --junit-xml=/results/junit-report.xml \ - --cassette-path=/results/cassette.yaml - - SAVE IMAGE schemathesis:$tag - -# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs -test-fuzzer-api: - FROM earthly/dind:alpine-3.19 - RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 - RUN apk add yq zstd - ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" - - COPY schemathesis-docker-compose.yml . - WITH DOCKER \ - --compose schemathesis-docker-compose.yml \ - --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ - --load event-db:latest=(../event-db+build) \ - --load cat-gateway:latest=(../+package-cat-gateway) \ - --service event-db \ - --service cat-gateway \ - --allow-privileged - RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \ - docker-compose logs cat-gateway > ./cat-gateway.log && zstd -9 cat-gateway.log && \ - docker cp st:/results/junit-report.xml junit-report.xml && \ - docker cp st:/results/cassette.yaml cassette.yaml - END - WAIT - SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml - SAVE ARTIFACT cat-gateway.log.zst AS LOCAL cat-gateway.log.zst - SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml - END - # test-lint-openapi - OpenAPI linting from an artifact # testing whether the OpenAPI generated during build stage follows good practice. test-lint-openapi: diff --git a/catalyst-gateway/tests/api_tests/.gitignore b/catalyst-gateway/tests/api_tests/.gitignore index 40f65ddf37..3b9825d2f9 100644 --- a/catalyst-gateway/tests/api_tests/.gitignore +++ b/catalyst-gateway/tests/api_tests/.gitignore @@ -1,2 +1,3 @@ __pycache__ -junit-report.xml \ No newline at end of file +junit-report.xml +.hypothesis/ \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/.gitignore b/catalyst-gateway/tests/schemathesis_tests/.gitignore new file mode 100644 index 0000000000..3b9825d2f9 --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/.gitignore @@ -0,0 +1,3 @@ +__pycache__ +junit-report.xml +.hypothesis/ \ No newline at end of file diff --git a/catalyst-gateway/tests/schemathesis_tests/Earthfile b/catalyst-gateway/tests/schemathesis_tests/Earthfile new file mode 100644 index 0000000000..b98d1d4f76 --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/Earthfile @@ -0,0 +1,62 @@ +VERSION 0.8 + +package-schemathesis: + FROM python:3.12-alpine3.19 + + ARG tag="latest" + ARG max_examples=1000 + # TODO: https://github.com/input-output-hk/catalyst-voices/issues/465 + ARG max_response_time=3000 + ARG wait_for_schema=15 + ARG workers=2 + ARG schema_version=30 + ARG openapi_spec + + RUN apk add --no-cache gcc musl-dev + RUN python -m pip install schemathesis==3.27.1 + RUN mkdir /results + COPY ./hooks/hooks.py . + + VOLUME /results + ENTRYPOINT export SCHEMATHESIS_HOOKS=hooks \ + && st run --checks all $openapi_spec \ + --workers=$workers \ + --wait-for-schema=$wait_for_schema \ + --max-response-time=$max_response_time \ + --hypothesis-max-examples=$max_examples \ + --data-generation-method=all \ + --skip-deprecated-operations \ + --force-schema-version=$schema_version \ + --show-trace \ + --force-color \ + --junit-xml=/results/junit-report.xml \ + --cassette-path=/results/cassette.yaml + + SAVE IMAGE schemathesis:$tag + +# test-fuzzer-api - Fuzzy test cat-gateway using openapi specs +test-fuzzer-api: + FROM earthly/dind:alpine-3.19 + RUN apk update && apk add iptables-legacy # workaround for https://github.com/earthly/earthly/issues/3784 + RUN apk add yq zstd + ARG OPENAPI_SPEC="http://127.0.0.1:3030/docs/cat-gateway.json" + + COPY schemathesis-docker-compose.yml . + WITH DOCKER \ + --compose schemathesis-docker-compose.yml \ + --load schemathesis:latest=(+package-schemathesis --openapi_spec=$OPENAPI_SPEC) \ + --load event-db:latest=(../../event-db+build) \ + --load cat-gateway:latest=(../../+package-cat-gateway) \ + --service event-db \ + --service cat-gateway \ + --allow-privileged + RUN docker run --net=host --name=st schemathesis:latest || echo fail > fail && \ + docker-compose logs cat-gateway > ./cat-gateway.log && zstd -9 cat-gateway.log && \ + docker cp st:/results/junit-report.xml junit-report.xml && \ + docker cp st:/results/cassette.yaml cassette.yaml + END + WAIT + SAVE ARTIFACT junit-report.xml AS LOCAL schemathesis.junit-report.xml + SAVE ARTIFACT cat-gateway.log.zst AS LOCAL cat-gateway.log.zst + SAVE ARTIFACT cassette.yaml AS LOCAL cassette.yaml + END diff --git a/catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py b/catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py new file mode 100644 index 0000000000..1948e9d6db --- /dev/null +++ b/catalyst-gateway/tests/schemathesis_tests/hooks/hooks.py @@ -0,0 +1,36 @@ +import copy +import schemathesis +import schemathesis.schemas +from typing import Any, Dict + +@schemathesis.hook +def before_load_schema( + context: schemathesis.hooks.HookContext, + raw_schema: Dict[str, Any], + ) -> None: + paths = dict(raw_schema["paths"]) + for endpoint in paths.keys(): + endpoint_data = dict(raw_schema["paths"][endpoint]) + + # Get first method data as reference + ref_info = next(iter(endpoint_data.values())) + + # Create new method data using reference data, replacing 'responses' field + new_info = copy.deepcopy(ref_info) + new_info["responses"] = {"405": {"description": "method not allowed"}} + + # Update endpoint if a method is not declared + if "get" not in endpoint_data: + endpoint_data.update([("get", new_info),]) + if "post" not in endpoint_data: + endpoint_data.update([("post", new_info),]) + if "put" not in endpoint_data: + endpoint_data.update([("put", new_info),]) + if "patch" not in endpoint_data: + endpoint_data.update([("patch", new_info),]) + if "delete" not in endpoint_data: + endpoint_data.update([("delete", new_info),]) + + # Update endpoint + raw_schema["paths"][endpoint] = endpoint_data + diff --git a/catalyst-gateway/tests/schemathesis-docker-compose.yml b/catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml similarity index 100% rename from catalyst-gateway/tests/schemathesis-docker-compose.yml rename to catalyst-gateway/tests/schemathesis_tests/schemathesis-docker-compose.yml