From 0d3d5d7589e33d2058cce58907e7c7eb57ab7bb8 Mon Sep 17 00:00:00 2001 From: hkir-dev Date: Fri, 6 Sep 2024 10:15:09 +0100 Subject: [PATCH] jinja documentation fixes --- setup.py | 2 +- src/tdta/documentation.py | 72 +++++++++++++++++++++++++++------- src/tdta/git_utils.py | 43 ++++++++++++++++++++ src/test/generate_docs_test.py | 2 +- 4 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 src/tdta/git_utils.py diff --git a/setup.py b/setup.py index 6f66e31..fd3e2fe 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="tdta", - version="0.1.0.dev20", + version="0.1.0.dev21", description="The aim of this project is to provide taxonomy development tools custom actions.", long_description=README, long_description_content_type="text/markdown", diff --git a/src/tdta/documentation.py b/src/tdta/documentation.py index 25b994d..e71620c 100644 --- a/src/tdta/documentation.py +++ b/src/tdta/documentation.py @@ -1,17 +1,20 @@ import os -from pathlib import Path +import shutil +from pathlib import Path from jinja2 import Template from urllib.parse import urlparse from tdta.tdt_export import db_to_cas from tdta.utils import read_project_config +from tdta.command_line_utils import runcmd +from tdta.version_control import git_update_local ANNOTATIONS_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../resources/annotation_template.md") TAXONOMY_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../resources/taxonomy_template.md") -def generate_documentation(sqlite_db: str, output_folder: str, project_config=None): +def generate_documentation(sqlite_db: str, output_folder: str, project_config=None, git_push=True): """ Generate markdown documentation for a CAS database. Parameters: @@ -19,32 +22,71 @@ def generate_documentation(sqlite_db: str, output_folder: str, project_config=No output_folder: Path to the output documentation folder. project_config: Project configuration. """ - if not os.path.exists(output_folder): - os.makedirs(output_folder) - + project_folder = Path(output_folder).parent.absolute() + index_file = os.path.join(output_folder, "index.md") cell_sets_folder = os.path.join(output_folder, "cell_sets") - if not os.path.exists(cell_sets_folder): - os.makedirs(cell_sets_folder) + + clear_docs_folder(cell_sets_folder, index_file, output_folder) cas_obj = db_to_cas(sqlite_db) cas = cas_obj.to_dict() if project_config is None: - project_config = read_project_config(Path(output_folder).parent.absolute()) + project_config = read_project_config(project_folder) cas = transform_cas(cas, project_config) + generate_annotation_docs(cas, cell_sets_folder) + generate_taxonomy_doc(cas, index_file, output_folder) + + if git_push: + runcmd("cd {dir} && git add --all {docs_folder}".format(dir=project_folder, + docs_folder=os.path.relpath(output_folder, project_folder))) + git_update_local(project_folder.absolute().as_posix(), "Update project documentation") + runcmd("cd {dir} && git push".format(dir=project_folder)) + print("Taxonomy documentation sent to GitHub.") + print("Github action is triggered to publish the documentation on the website. Please check the status of the action.") + + +def generate_taxonomy_doc(cas, index_file, output_folder): + """ + Generate the taxonomy documentation (index.md). + Parameters: + cas: CAS object + index_file: Path to the index file + output_folder: Path to the output folder + """ + taxonomy_template = read_jinja_template(TAXONOMY_TEMPLATE) + rendered_file = taxonomy_template.render(cas=cas) + with open(index_file, "w") as fh: + fh.write(rendered_file) + print("Taxonomy documentation generated at {out_dir}".format(out_dir=output_folder)) + + +def generate_annotation_docs(cas, cell_sets_folder): + """ + Generate markdown documentation for each cell set in the CAS. + Parameters: + cas: CAS object + cell_sets_folder: Path to the cell sets folder + """ annotation_template = read_jinja_template(ANNOTATIONS_TEMPLATE) - cell_sets_folder = os.path.join(output_folder, "cell_sets") for annotation in cas["annotations"]: rendered_file = annotation_template.render(annotation=annotation, metadata=cas) annotation_file_name = annotation["cell_set_accession"].replace(":", "_") - - with open(os.path.join(cell_sets_folder, annotation_file_name + ".md"), "w") as fh: + with open(os.path.join(cell_sets_folder, annotation_file_name + ".md"), "w") as fh: fh.write(rendered_file) - taxonomy_template = read_jinja_template(TAXONOMY_TEMPLATE) - rendered_file = taxonomy_template.render(cas=cas) - with open(os.path.join(output_folder, "index.md"), "w") as fh: - fh.write(rendered_file) + +def clear_docs_folder(cell_sets_folder, index_file, output_folder): + """ + Deletes the existing docs folder content. + """ + if not os.path.exists(output_folder): + os.makedirs(output_folder) + else: + shutil.rmtree(cell_sets_folder, ignore_errors=True) + if os.path.isfile(index_file): + os.remove(index_file) + os.makedirs(cell_sets_folder) def transform_cas(cas, project_config): diff --git a/src/tdta/git_utils.py b/src/tdta/git_utils.py new file mode 100644 index 0000000..c9dfa64 --- /dev/null +++ b/src/tdta/git_utils.py @@ -0,0 +1,43 @@ +import os +import zipfile + +from tdta.command_line_utils import runcmd + +GITHUB_SIZE_LIMIT = 100 * 1000 * 1000 # 100 MB + + +def add_new_files_to_git(project_folder, new_files): + """ + Runs git add command to add imported files to the version control. + Parameters: + project_folder: project folder path + new_files: imported/created file paths to add to the version control + """ + for file_path in new_files: + if os.path.getsize(file_path) > GITHUB_SIZE_LIMIT: + zip_path = zip_file(file_path) + new_files.remove(file_path) + runcmd("cd {dir} && git add {zip_path}".format(dir=project_folder, zip_path=zip_path)) + runcmd("cd {dir} && git reset {file_path}".format(dir=project_folder, file_path=file_path)) + + runcmd("cd {dir} && git add {files}". + format(dir=project_folder, + files=" ".join([t.replace(project_folder, ".", 1) for t in new_files]))) + + +def zip_file(file_path): + """ + Zips the file if it exceeds the GitHub size limit. + Parameters: + file_path: file path to zip + Returns: zipped file path + """ + folder = os.path.dirname(file_path) + base_name = os.path.basename(file_path) + zip_base = os.path.splitext(base_name)[0] + + single_zip_path = os.path.join(folder, f"{zip_base}.zip") + with zipfile.ZipFile(single_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.write(file_path, base_name) + + return single_zip_path diff --git a/src/test/generate_docs_test.py b/src/test/generate_docs_test.py index 5114172..2365b22 100644 --- a/src/test/generate_docs_test.py +++ b/src/test/generate_docs_test.py @@ -18,7 +18,7 @@ def setUp(self): def test_documentation_generation(self): generate_documentation(TEST_DB, TEST_OUTPUT, project_config={"id": "CS202210140", - "custom_purl": "https://purl.brain-bican.org/taxonomy/CS202210140/CS202210140_non-neuronal/"}) + "custom_purl": "https://purl.brain-bican.org/taxonomy/CS202210140/CS202210140_non-neuronal/"}, git_push=False) self.assertTrue(os.path.exists(TEST_OUTPUT)) def test_hierarchy_breadcrumb(self):