jinja documentation fixes

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name="tdta",
-    version="0.1.0.dev20",
+    version="0.1.0.dev21",
     description="The aim of this project is to provide taxonomy development tools custom actions.",
     long_description=README,
     long_description_content_type="text/markdown",

src/tdta/documentation.py

@@ -1,50 +1,92 @@
 import os
-from pathlib import Path
+import shutil
 
+from pathlib import Path
 from jinja2 import Template
 from urllib.parse import urlparse
 
 from tdta.tdt_export import db_to_cas
 from tdta.utils import read_project_config
+from tdta.command_line_utils import runcmd
+from tdta.version_control import git_update_local
 
 ANNOTATIONS_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../resources/annotation_template.md")
 TAXONOMY_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../resources/taxonomy_template.md")
 
 
-def generate_documentation(sqlite_db: str, output_folder: str, project_config=None):
+def generate_documentation(sqlite_db: str, output_folder: str, project_config=None, git_push=True):
     """
     Generate markdown documentation for a CAS database.
     Parameters:
         sqlite_db: Path to the CAS database.
         output_folder: Path to the output documentation folder.
         project_config: Project configuration.
     """
-    if not os.path.exists(output_folder):
-        os.makedirs(output_folder)
-
+    project_folder = Path(output_folder).parent.absolute()
+    index_file = os.path.join(output_folder, "index.md")
     cell_sets_folder = os.path.join(output_folder, "cell_sets")
-    if not os.path.exists(cell_sets_folder):
-        os.makedirs(cell_sets_folder)
+
+    clear_docs_folder(cell_sets_folder, index_file, output_folder)
 
     cas_obj = db_to_cas(sqlite_db)
     cas = cas_obj.to_dict()
     if project_config is None:
-        project_config = read_project_config(Path(output_folder).parent.absolute())
+        project_config = read_project_config(project_folder)
     cas = transform_cas(cas, project_config)
 
+    generate_annotation_docs(cas, cell_sets_folder)
+    generate_taxonomy_doc(cas, index_file, output_folder)
+
+    if git_push:
+        runcmd("cd {dir} && git add --all {docs_folder}".format(dir=project_folder,
+                                                                docs_folder=os.path.relpath(output_folder, project_folder)))
+        git_update_local(project_folder.absolute().as_posix(), "Update project documentation")
+        runcmd("cd {dir} && git push".format(dir=project_folder))
+        print("Taxonomy documentation sent to GitHub.")
+        print("Github action is triggered to publish the documentation on the website. Please check the status of the action.")
+
+
+def generate_taxonomy_doc(cas, index_file, output_folder):
+    """
+    Generate the taxonomy documentation (index.md).
+    Parameters:
+        cas: CAS object
+        index_file: Path to the index file
+        output_folder: Path to the output folder
+    """
+    taxonomy_template = read_jinja_template(TAXONOMY_TEMPLATE)
+    rendered_file = taxonomy_template.render(cas=cas)
+    with open(index_file, "w") as fh:
+        fh.write(rendered_file)
+    print("Taxonomy documentation generated at {out_dir}".format(out_dir=output_folder))
+
+
+def generate_annotation_docs(cas, cell_sets_folder):
+    """
+    Generate markdown documentation for each cell set in the CAS.
+    Parameters:
+        cas: CAS object
+        cell_sets_folder: Path to the cell sets folder
+    """
     annotation_template = read_jinja_template(ANNOTATIONS_TEMPLATE)
-    cell_sets_folder = os.path.join(output_folder, "cell_sets")
     for annotation in cas["annotations"]:
         rendered_file = annotation_template.render(annotation=annotation, metadata=cas)
         annotation_file_name = annotation["cell_set_accession"].replace(":", "_")
-
-        with open(os.path.join(cell_sets_folder,  annotation_file_name + ".md"), "w") as fh:
+        with open(os.path.join(cell_sets_folder, annotation_file_name + ".md"), "w") as fh:
             fh.write(rendered_file)
 
-    taxonomy_template = read_jinja_template(TAXONOMY_TEMPLATE)
-    rendered_file = taxonomy_template.render(cas=cas)
-    with open(os.path.join(output_folder, "index.md"), "w") as fh:
-        fh.write(rendered_file)
+
+def clear_docs_folder(cell_sets_folder, index_file, output_folder):
+    """
+    Deletes the existing docs folder content.
+    """
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+    else:
+        shutil.rmtree(cell_sets_folder, ignore_errors=True)
+        if os.path.isfile(index_file):
+            os.remove(index_file)
+    os.makedirs(cell_sets_folder)
 
 
 def transform_cas(cas, project_config):

src/tdta/git_utils.py

@@ -0,0 +1,43 @@
+import os
+import zipfile
+
+from tdta.command_line_utils import runcmd
+
+GITHUB_SIZE_LIMIT = 100 * 1000 * 1000  # 100 MB
+
+
+def add_new_files_to_git(project_folder, new_files):
+    """
+    Runs git add command to add imported files to the version control.
+    Parameters:
+        project_folder: project folder path
+        new_files: imported/created file paths to add to the version control
+    """
+    for file_path in new_files:
+        if os.path.getsize(file_path) > GITHUB_SIZE_LIMIT:
+            zip_path = zip_file(file_path)
+            new_files.remove(file_path)
+            runcmd("cd {dir} && git add {zip_path}".format(dir=project_folder, zip_path=zip_path))
+            runcmd("cd {dir} && git reset {file_path}".format(dir=project_folder, file_path=file_path))
+
+    runcmd("cd {dir} && git add {files}".
+           format(dir=project_folder,
+                  files=" ".join([t.replace(project_folder, ".", 1) for t in new_files])))
+
+
+def zip_file(file_path):
+    """
+    Zips the file if it exceeds the GitHub size limit.
+    Parameters:
+        file_path: file path to zip
+    Returns: zipped file path
+    """
+    folder = os.path.dirname(file_path)
+    base_name = os.path.basename(file_path)
+    zip_base = os.path.splitext(base_name)[0]
+
+    single_zip_path = os.path.join(folder, f"{zip_base}.zip")
+    with zipfile.ZipFile(single_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        zipf.write(file_path, base_name)
+
+    return single_zip_path

src/test/generate_docs_test.py

@@ -18,7 +18,7 @@ def setUp(self):
 
     def test_documentation_generation(self):
         generate_documentation(TEST_DB, TEST_OUTPUT, project_config={"id": "CS202210140",
-                                                                     "custom_purl": "https://purl.brain-bican.org/taxonomy/CS202210140/CS202210140_non-neuronal/"})
+                                                                     "custom_purl": "https://purl.brain-bican.org/taxonomy/CS202210140/CS202210140_non-neuronal/"}, git_push=False)
         self.assertTrue(os.path.exists(TEST_OUTPUT))
 
     def test_hierarchy_breadcrumb(self):