From a95e3795bbdb4cf76300a6d3f2fc412e5526411b Mon Sep 17 00:00:00 2001 From: Alan Christie Date: Mon, 24 Jul 2023 12:09:34 +0100 Subject: [PATCH] feat: Now supports comma-separated input file lists --- .pre-commit-config.yaml | 12 +++++------ DEVELOPER-README.md | 2 +- build-requirements.txt | 2 +- src/jote/jote.py | 45 ++++++++++++++++++++++++----------------- 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 566c357..eef8ba6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ --- -minimum_pre_commit_version: 2.18.1 +minimum_pre_commit_version: 3.3.3 exclude: ^src/jote/VERSION repos: @@ -9,7 +9,7 @@ repos: # Conventional Commit message checker (commitizen) - repo: https://github.com/commitizen-tools/commitizen - rev: v2.23.0 + rev: v3.5.3 hooks: - id: commitizen stages: @@ -20,7 +20,7 @@ repos: # Standard pre-commit rules - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.4.0 hooks: - id: check-case-conflict - id: check-docstring-first @@ -33,7 +33,7 @@ repos: - --markdown-linebreak-ext=md # Black (uncompromising) Python code formatter - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.7.0 hooks: - id: black args: @@ -41,7 +41,7 @@ repos: - py39 # MyPy - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.942 + rev: v1.4.1 hooks: - id: mypy files: ^src @@ -50,7 +50,7 @@ repos: - --non-interactive # Pylint - repo: https://github.com/pycqa/pylint - rev: v2.13.5 + rev: v2.17.4 hooks: - id: pylint files: ^src diff --git a/DEVELOPER-README.md b/DEVELOPER-README.md index 35dd097..33d3855 100644 --- a/DEVELOPER-README.md +++ b/DEVELOPER-README.md @@ -41,7 +41,7 @@ To build the package distribution manually run: - To install the local build, without needing to publish the package run: - - pip install dist/im_jote-1.0.0-py3-none-any.whl + pip install dist/im_jote-*-py3-none-any.whl --- diff --git a/build-requirements.txt b/build-requirements.txt index 7a46ed1..0fbcd16 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -1,5 +1,5 @@ pyroma == 4.2 pytest == 7.4.0 pytest-cov == 4.1.0 -mypy ==1.4.1 +mypy == 1.4.1 pre-commit == 3.3.3 diff --git a/src/jote/jote.py b/src/jote/jote.py index 9944c98..ca0e17c 100755 --- a/src/jote/jote.py +++ b/src/jote/jote.py @@ -67,7 +67,6 @@ class TestResult(Enum): def _print_test_banner(collection: str, job_name: str, job_test_name: str) -> None: - print(" ---") print(f"+ collection={collection} job={job_name} test={job_test_name}") @@ -180,7 +179,6 @@ def _add_grouped_test( """ for run_group_name in run_group_names: - # Find the test-group for this test test_group_definition: Optional[DefaultMunch] = None for test_group in test_groups: @@ -272,7 +270,6 @@ def _load( num_tests: int = 0 for jd_filename in manifest_munch["job-definition-files"]: - # Does the definition comply with the schema? # No options here - it must. jd_path: str = os.path.join(_DEFINITION_DIRECTORY, jd_filename) @@ -349,7 +346,6 @@ def _copy_inputs(test_inputs: List[str], project_path: str) -> bool: # The files are assumed to reside in the repo's 'data' directory. print(f'# Copying inputs (from "${{PWD}}/{_DATA_DIRECTORY_PATH}")...') for test_input in test_inputs: - print(f"# + {test_input}") if not test_input.startswith(_DATA_DIRECTORY_PATH): @@ -370,7 +366,6 @@ def _copy_inputs(test_inputs: List[str], project_path: str) -> bool: def _check_exists(name: str, path: str, expected: bool, fix_permissions: bool) -> bool: - exists: bool = os.path.exists(path) if expected and not exists: print(f"# exists ({expected}) [FAILED]") @@ -427,7 +422,6 @@ def _check_exists(name: str, path: str, expected: bool, fix_permissions: bool) - def _check_line_count(name: str, path: str, expected: int) -> bool: - line_count: int = 0 with open(path, "rt", encoding="UTF-8") as check_file: for _ in check_file: @@ -682,17 +676,32 @@ def _run_a_test( else: # It is an input (not an option). # The input is a list if it's declared as 'multiple'. + # + # We also have to deal with each file being a potential pair + # i.e. "data/nsp13-x0176_0B.mol,data/nsp13-x0176_0B_apo-desolv.pdb" + # This will appear in job_variables as: - + # "nsp13-x0176_0B.mol,nsp13-x0176_0B_apo-desolv.pdb" + # and in the input files as two files: - + # "data/nsp13-x0176_0B.mol" and "data/nsp13-x0176_0B_apo-desolv.pdb" if job_definition.variables.inputs.properties[variable].multiple: job_variables[variable] = [] for value in job_definition.tests[job_test_name].inputs[ variable ]: - job_variables[variable].append(os.path.basename(value)) - input_files.append(value) + basename_values = [] + for value_item in value.split(","): + value_basename = os.path.basename(value_item) + basename_values.append(value_basename) + input_files.append(value_item) + job_variables[variable].append(",".join(basename_values)) else: value = job_definition.tests[job_test_name].inputs[variable] - job_variables[variable] = os.path.basename(value) - input_files.append(value) + basename_values = [] + for value_item in value.split(","): + value_basename = os.path.basename(value_item) + basename_values.append(value_basename) + input_files.append(value_item) + job_variables[variable].append(",".join(basename_values)) decoded_command: str = "" test_environment: Dict[str, str] = {} @@ -728,6 +737,9 @@ def _run_a_test( # Get the raw (encoded) command from the job definition... raw_command: str = job_definition.command # Decode it using our variables... + if args.verbose: + print(f"> raw_command={raw_command}") + print(f"> job_variables={job_variables}") decoded_command, test_status = decoder.decode( raw_command, job_variables, @@ -800,7 +812,6 @@ def _run_a_test( # Run the container if not args.dry_run: - timeout_minutes: int = DEFAULT_TEST_TIMEOUT_M if "timeout-minutes" in job_definition.tests[job_test_name]: timeout_minutes = job_definition.tests[job_test_name]["timeout-minutes"] @@ -888,7 +899,6 @@ def _run_ungrouped_tests( tests_failed: int = 0 for job_test_name in job_definition.tests: - # If a job test has been named, # skip this test if it doesn't match. # We do not include this test in the count. @@ -957,11 +967,9 @@ def _run_grouped_tests( test_result: Optional[TestResult] = None for jd_filename, grouped_tests in grouped_job_definitions.items(): - # The grouped definitions are indexed by JobDefinition filename # and for each there is a list of dictionaries (indexed by group name). for file_run_group in grouped_tests: - run_group_name: str = file_run_group["test-group-name"] if args.run_group and run_group_name != args.run_group: # A specific group has been named @@ -1004,7 +1012,6 @@ def _run_grouped_tests( and existing_group_test[3] == job[1] and existing_group_test[4] == job_test_name ): - new_test = False break @@ -1012,7 +1019,6 @@ def _run_grouped_tests( existing_group_test[2] != job[0] or existing_group_test[3] != job[1] ): - # Oops - ordinal used elsewhere in this group. # Return a failure! print("! FAILURE") @@ -1052,7 +1058,6 @@ def _run_grouped_tests( # 3. stop the compose file group_compose_file: Optional[str] = None for index, grouped_test in enumerate(grouped_tests): - # For each grouped test we have a test-group definition [at index 0], # an 'ordinal' [1], 'collection' [2], 'job name' [3], 'job test' [4] # and the 'job' definition [5] @@ -1254,7 +1259,10 @@ def main() -> int: ) arg_parser.add_argument( - "-v", "--verbose", action="store_true", help="Displays test stdout" + "-v", + "--verbose", + action="store_true", + help="Displays test stdout amongst other things", ) arg_parser.add_argument( @@ -1457,7 +1465,6 @@ def main() -> int: # MAIN # ----------------------------------------------------------------------------- if __name__ == "__main__": - _RET_VAL: int = main() if _RET_VAL != 0: sys.exit(_RET_VAL)