diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8ce697805f..d364125714d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - quickwit/**.rs - quickwit/**.toml - quickwit/**.proto + - ./quickwit/rest-api-tests/** # The following step is just meant to install rustup actually. # The next one installs the correct toolchain. - name: Install rustup @@ -79,8 +80,19 @@ jobs: tool: cargo-nextest - name: cargo nextest if: steps.modified.outputs.rust_src == 'true' - run: cargo nextest run --features=postgres --profile ci --retries 1 + run: cargo nextest run --features=postgres --retries 1 working-directory: ./quickwit + - name: cargo build + if: steps.modified.outputs.rust_src == 'true' + run: cargo build --features=postgres --bin quickwit + working-directory: ./quickwit + - name: Install python packages + run: sudo pip3 install pyaml requests + if: steps.modified.outputs.rust_src == 'true' + - name: run REST API tests + if: steps.modified.outputs.rust_src == 'true' + run: python3 ./run_tests.py --binary ../target/debug/quickwit + working-directory: ./quickwit/rest-api-tests lints: name: Lints runs-on: "ubuntu-latest" diff --git a/quickwit/rest-api-tests/rest_api_test.py b/quickwit/rest-api-tests/run_tests.py similarity index 71% rename from quickwit/rest-api-tests/rest_api_test.py rename to quickwit/rest-api-tests/run_tests.py index 647df620625..7874f6fa395 100755 --- a/quickwit/rest-api-tests/rest_api_test.py +++ b/quickwit/rest-api-tests/run_tests.py @@ -6,9 +6,13 @@ import yaml import sys from os import path as osp +from os import mkdir import gzip import http import json +import tempfile +import shutil +import time def debug_http(): old_send = http.client.HTTPConnection.send @@ -50,6 +54,18 @@ def load_data(path): else: return open(path, 'rb').read() +def run_request_with_retry(run_req, expected_status_code=None, num_retries=10, wait_time=0.5): + for try_number in range(num_retries + 1): + r = run_req() + if expected_status_code is None or r.status_code == expected_status_code: + return r + print("Failed with", r.text, r.status_code) + if try_number < num_retries: + print("Retrying...") + time.sleep(wait_time) + raise Exception("Wrong status code. Got %s, expected %s" % (r.status_code, expected_status_code)) + + def run_request_step(method, step): assert method in {"GET", "POST", "PUT", "DELETE"} if "headers" not in step: @@ -70,12 +86,10 @@ def run_request_step(method, step): if ndjson is not None: kvargs["data"] = "\n".join([json.dumps(doc) for doc in ndjson]) kvargs.setdefault("headers")["Content-Type"] = "application/json" - r = method_req(url, **kvargs) - expected_status_code = step.get("status_code", 200) - if expected_status_code is not None: - if r.status_code != expected_status_code: - print(r.text) - raise Exception("Wrong status code. Got %s, expected %s" % (r.status_code, expected_status_code)) + expected_status_code = step.get("status_code", None) + num_retries = step.get("num_retries", 0) + run_req = lambda : method_req(url, **kvargs) + r = run_request_with_retry(run_req, expected_status_code, num_retries) expected_resp = step.get("expected", None) if expected_resp is not None: try: @@ -140,13 +154,15 @@ def add_path(self, path): path_tree.add_script(path_segs[-1]) def visit_nodes(self, visitor, path=[]): - visitor.enter_directory(path) + success = True + success &= visitor.enter_directory(path) for script in self.scripts: - visitor.run_scenario(path, script) + success &= visitor.run_scenario(path, script) for k in sorted(self.children.keys()): child_path = path + [k] - self.children[k].visit_nodes(visitor, child_path) - visitor.exit_directory(path) + success &= self.children[k].visit_nodes(visitor, child_path) + success &= visitor.exit_directory(path) + return success # Returns a new dictionary without modifying the arguments. # The new dictionary is the result of merging the two dictionaries @@ -164,10 +180,12 @@ def __init__(self, engine): self.context = {} def run_setup_teardown_scripts(self, script_name, path): cwd = "/".join(path) + success = True for file_name in [script_name + ".yaml", script_name + "." + self.engine + ".yaml"]: script_fullpath = cwd + "/" + file_name if osp.exists(script_fullpath): - self.run_scenario(path, file_name) + success &= self.run_scenario(path, file_name) + return success def load_context(self, path): context = {"cwd": "/".join(path)} for file_name in ["_ctx.yaml", "_ctx." + self.engine + ".yaml"]: @@ -180,13 +198,14 @@ def load_context(self, path): def enter_directory(self, path): print("============") self.load_context(path) - self.run_setup_teardown_scripts("_setup", path) + return self.run_setup_teardown_scripts("_setup", path) def exit_directory(self, path): - self.run_setup_teardown_scripts("_teardown", path) + success = self.run_setup_teardown_scripts("_teardown", path) self.context_stack.pop() self.context = {} for ctx in self.context_stack: self.context.update(ctx) + return success def run_scenario(self, path, script): scenario_path = "/".join(path + [script]) steps = list(open_scenario(scenario_path)) @@ -208,9 +227,10 @@ def run_scenario(self, path, script): print(step) print(e) print("--------------") - break + return False else: print("🟢 %s: %d steps (%d skipped)" % (scenario_path, num_steps_executed, num_steps_skipped)) + return True def build_path_tree(paths): paths.sort() @@ -222,7 +242,7 @@ def build_path_tree(paths): def run(scenario_paths, engine): path_tree = build_path_tree(scenario_paths) visitor = Visitor(engine=engine) - path_tree.visit_nodes(visitor) + return path_tree.visit_nodes(visitor) def filter_test(prefixes, test_name): for prefix in prefixes: @@ -239,6 +259,36 @@ def filter_tests(prefixes, test_names): if filter_test(prefixes, test_name) ] +class QuickwitRunner: + def __init__(self, quickwit_bin_path): + self.quickwit_dir = tempfile.TemporaryDirectory() + print('created temporary directory', self.quickwit_dir, self.quickwit_dir.name) + qwdata = osp.join(self.quickwit_dir.name, "qwdata") + config = osp.join(self.quickwit_dir.name, "config") + mkdir(qwdata) + mkdir(config) + shutil.copy("../../config/quickwit.yaml", config) + shutil.copy(quickwit_bin_path, self.quickwit_dir.name) + self.proc = subprocess.Popen(["./quickwit", "run"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=self.quickwit_dir.name) + for i in range(100): + try: + print("Checking on quickwit") + res = requests.get("http://localhost:7280/health/readyz") + if res.status_code == 200 and res.text.strip() == "true": + print("Quickwit started") + time.sleep(6) + break + except: + pass + print("Server not ready yet. Sleep and retry...") + time.sleep(1) + else: + print("Quickwit never started. Exiting.") + sys.exit(2) + def __del__(self): + print("Killing Quickwit") + subprocess.Popen.kill(self.proc) + def main(): import argparse arg_parser = argparse.ArgumentParser( @@ -247,11 +297,28 @@ def main(): ) arg_parser.add_argument("--engine", help="Targetted engine (elastic/quickwit).", default="quickwit") arg_parser.add_argument("--test", help="Specific prefix to select the tests to run. If not specified, all tests are run.", nargs="*") + arg_parser.add_argument("--binary", help="Specific the quickwit binary to run.", nargs="?") parsed_args = arg_parser.parse_args() + + print(parsed_args) + + quickwit_process = None + if parsed_args.binary is not None: + if parsed_args.engine != "quickwit": + print("The --binary option is only supported for quickwit engine.") + sys.exit(3) + binary = parsed_args.binary + quickwit_process = QuickwitRunner(binary) + quickwit_process + scenario_filepaths = glob.glob("scenarii/**/*.yaml", recursive=True) scenario_filepaths = list(filter_tests(parsed_args.test, scenario_filepaths)) - run(scenario_filepaths, engine=parsed_args.engine) + return run(scenario_filepaths, engine=parsed_args.engine) if __name__ == "__main__": - main() + import sys + if main(): + sys.exit(0) + else: + sys.exit(1) diff --git a/quickwit/rest-api-tests/scenarii/default_search_fields/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/default_search_fields/_setup.quickwit.yaml index f9e65264173..6e25010f9a6 100644 --- a/quickwit/rest-api-tests/scenarii/default_search_fields/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/default_search_fields/_setup.quickwit.yaml @@ -26,7 +26,6 @@ json: - regular_field - some_dynamic_field - inner_json.somefieldinjson - --- method: POST endpoint: defaultsearchfields/ingest