Merge pull request #191 from brain-bican/preserve_annot

gh pages documentation support added

Dockerfile

@@ -5,6 +5,8 @@ RUN mkdir $WORKSPACE
 RUN mkdir $WORKSPACE/dendR
 RUN mkdir $WORKSPACE/scripts
 RUN mkdir $WORKSPACE/resources
+RUN mkdir $WORKSPACE/resources/assets
+RUN mkdir $WORKSPACE/resources/github_actions
 
 ENV DEBIAN_FRONTEND=noninteractive
 # for cellxgene-census
@@ -56,6 +58,7 @@ ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 ADD Makefile $WORKSPACE
 ADD resources/repo_README.md $WORKSPACE/resources
 ADD resources/repo_PURL_config.yml $WORKSPACE/resources
+ADD resources/repo_mkdocs.yml $WORKSPACE/resources
 ADD requirements.txt $WORKSPACE
 ADD tdt/tdt.py $WORKSPACE
 ADD tdt/api/tdt_api.py $WORKSPACE
@@ -74,6 +77,8 @@ ADD scripts/configurations.py $WORKSPACE/scripts
 ADD scripts/export.py $WORKSPACE/scripts
 ADD scripts/upgrade.py $WORKSPACE/scripts
 ADD scripts/.gitignore $WORKSPACE/scripts
+ADD resources/assets/logo.webp $WORKSPACE/resources/assets
+ADD resources/github_actions/publish-docs.yml $WORKSPACE/resources/github_actions
 
 RUN python3 -m pip install  -r $WORKSPACE/requirements.txt
 
@@ -87,8 +92,8 @@ RUN python3 -m pip install numpy==1.26.4
 RUN python3 -m pip install marshmallow==3.21.1
 RUN python3 -m pip install python-dateutil==2.9.0
 RUN python3 -m pip install cap-anndata==0.2.1
-RUN python3 -m pip install --no-deps cas-tools==1.0.7
-RUN python3 -m pip install --no-deps tdta==0.1.0.dev17
+RUN python3 -m pip install --no-deps cas-tools==1.0.8
+RUN python3 -m pip install --no-deps tdta==0.1.0.dev24
 
 #RUN Rscript $WORKSPACE/dendR/install_packages.R
 
@@ -129,8 +134,9 @@ RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/g
 RUN apt-get update &&  \
     apt-get install -y aha \
     sqlite3 \
-    python3-psycopg2 \
-    gh
+    python3-psycopg2
+
+RUN apt-get install gh --allow-unauthenticated
 
 # restore WORKDIR
 WORKDIR /tools

docs/Gh_Pages.md

@@ -0,0 +1,33 @@
+# Generating Taxonomy Github Pages
+
+_This feature is supported on TDT versions >= 1.0.7. Please consider upgrading your project through following the [upgrade guide](Upgrade.md)_
+
+The Taxonomy Development Tools (TDT) provides a feature to generate a GitHub Pages site for your taxonomy. This feature allows you to create a user-friendly interface for your taxonomy, making it easier for users to explore and understand the data. 
+
+These pages also works in synergy with the [PURL](https://purl.brain-bican.org/) system, allowing you to create a permanent URL for your taxonomy and cell sets.
+
+To generate the GitHub Pages for your taxonomy, follow these steps:
+
+1- In the TDT UI run Actions > Generate > Github Pages
+
+<p align="center">
+    <img src="https://raw.githubusercontent.com/brain-bican/taxonomy-development-tools/main/docs/images/screenshots/docs_menu.png" alt="Documentation generation action." width="300"/>
+</p>
+
+This operation will generate documentation on your project root directory under the `docs` folder and push them to the GitHub.
+
+2- (This step only needed once, at the first publishing of the GH pages) Navigate to your GitHub repository and go to the `Settings` tab.
+
+On the left panel select `Pages` and under the `Code and Automation` section. Then select the `gh-pages` branch in the `Branch` section and `Save`.
+
+<p align="center">
+    <img src="https://raw.githubusercontent.com/brain-bican/taxonomy-development-tools/main/docs/images/screenshots/docs_config.png" alt="GH Pages configuration" width="500"/>
+</p>
+
+Then please wait for a few minutes for the GitHub Pages to be generated. In the background a GitHub action is preparing that branch for you. Then your taxonomy will be available at `https://brain-bican.github.io/<your_repository_name>/`.
+
+## Troubleshooting
+
+If you cannot see the `gh_pages` branch to select at step 2, you may need to wait a few minutes for the GitHub Pages to be generated. In the background a GitHub action is preparing that branch for you.
+
+If the branch is still not visible, please navigate to the `Actions` tab in your repository and check the status of the `Publish mkdocs documentation` action. If it is still running, please wait for it to complete (depending on the size of your taxonomy it may take up to 30 minutes). If the action is failed please [report an issue](https://github.com/brain-bican/taxonomy-development-tools/issues/new?assignees=&labels=bug&projects=&template=bug_report.md&title=) on the TDT repository. Please add the failed steps logs to the issue for the swift resolution.

docs/config.md

@@ -31,6 +31,7 @@ creators:
 * `citation`: DOI of associated paper (where the taxonomy is for a published dataset)
 * `creators`: ORCIDs or all other authors should be included here
 * `datasets_folder`: (Optional) _This property is only support in linux/macOS systems._ The folder where the AnnData files are stored. If not provided, the datasets will be stored in the default location (`$USER_HOME/tdt_datasets`). Please note that using a network drive will slow down the AnnData related actions such as `Export > To CAS` and `Export > To AnnData`.
+* `custom_purl`: (Optional) A custom purl for the taxonomy to be used on GitHub Pages documentation. If not provided, a purl will be generated based on the `github_org` and `repo` fields.
 
 ## Configure seeding a new taxonomy from an existing informal taxonomy.
 

nanobot/nanobot.toml

@@ -63,6 +63,16 @@ commands = [
   ["tdta", "anndata", "-db", "./build/nanobot.db", "-j", "./$$TAXONOMY_ID$$.json", "-o", "/tdt_datasets/$$TAXONOMY_ID$$.h5ad", "-c", "/tdt_datasets"]
 ]
 
+[actions.generate]
+label = "Generate"
+commands = []
+
+[actions.sub_generate_docs]
+label = "Github Pages"
+commands = [
+  ["tdta", "docs", "-db", "./build/nanobot.db", "-o", "./docs"]
+]
+
 [actions.github]
 label = "Version Control"
 commands = []

nanobot/src/resources/table.html

@@ -539,11 +539,13 @@ <h1 class="modal-title fs-5" id="{{ name|id }}ModalLabel">{{ value.label or name
     // generate at string per accession and store in all_ats
     for (var row_index in at_data) {
       var row = at_data[row_index];
-      source_taxonomy = row["source_taxonomy"].split("/").at(-1).split(".")[0];
-      transferred_cell_label = row["transferred_cell_label"];
-      ats = all_ats[row["target_node_accession"]] || [];
-      ats.push(source_taxonomy + ": " + transferred_cell_label);
-      all_ats[row["target_node_accession"]] = ats;
+      if (row["source_taxonomy"]) {
+        source_taxonomy = row["source_taxonomy"].split("/").at(-1).split(".")[0];
+        transferred_cell_label = row["transferred_cell_label"];
+        ats = all_ats[row["target_node_accession"]] || [];
+        ats.push(source_taxonomy + ": " + transferred_cell_label);
+        all_ats[row["target_node_accession"]] = ats;
+      }
     }
     $.each(all_ats, (key, val) => {
       var at_cell = $("a[name='at_" + key + "']");

requirements.txt

@@ -14,7 +14,7 @@ mkdocs
 # installed from Dockerfile
 # cas-tools==1.0.7
 # tdta==0.1.0.dev5
-cell-annotation-schema==0.2b0
+cell-annotation-schema==1.0.0
 pandas
 pyarrow
 flask

resources/github_actions/publish-docs.yml

@@ -0,0 +1,34 @@
+name: Publish mkdocs documentation
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+
+permissions:
+  contents: write
+
+jobs:
+  build-and-publish:
+    name: Publish mkdocs documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v3
+
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: 3.x
+
+      - run: pip install mkdocs-material mkdocs-include-dir-to-nav mkdocs-nav-weight==0.2.0
+
+      - run: mkdocs gh-deploy --force

resources/repo_PURL_config.yml

@@ -1,12 +1,11 @@
 # PURL configuration for https://purl.brain-bican.org/taxonomy/$$TAXONOMY_ID$$
-
 idspace: $$TAXONOMY_ID$$
 base_url: /taxonomy/$$TAXONOMY_ID$$
 
 products:
 - $$TAXONOMY_ID$$.json: https://raw.githubusercontent.com/$$PROJECT_GITHUB_ORG$$/$$PROJECT_REPO$$/main/$$TAXONOMY_ID$$.json
 
-base_redirect: https://github.com/$$PROJECT_GITHUB_ORG$$/$$PROJECT_REPO$$
+base_redirect: https://brain-bican.github.io/$$PROJECT_REPO$$
 
 entries:
 
@@ -20,3 +19,11 @@ entries:
   tests:
     - from: /releases/2023-09-24/
       to: https://raw.githubusercontent.com/$$PROJECT_GITHUB_ORG$$/$$PROJECT_REPO$$/v2023-09-24/
+
+# This config should be the last item since it will catch all remaining paths
+# https://purl.brain-bican.org/taxonomy/$$TAXONOMY_ID$$/accession_id
+- regex: ^/taxonomy/$$TAXONOMY_ID$$/(.*)$
+  replacement: https://brain-bican.github.io/$$PROJECT_REPO$$/cell_sets/$1
+  tests:
+  - from: /$$TAXONOMY_ID$$/$$TAXONOMY_ID$$_1
+    to: https://brain-bican.github.io/$$PROJECT_REPO$$/cell_sets/$$TAXONOMY_ID$$_1

resources/repo_README.md

@@ -2,39 +2,30 @@
 
 $$TAXONOMY_DESCRIPTION$$
 
-Curate your taxonomy in 3 simple steps:
+## Stable release versions
 
-1. [Get Taxonomy Development Tools](#get-taxonomy-development-tools)  
-1. [Load your data](#load-your-data)  
-1. [Browse](#browse)  
+The latest version of the taxonomy can always be found at:
 
-## Get Taxonomy Development Tools 
+https://purl.brain-bican.org/taxonomy/$$TAXONOMY_ID$$/$$TAXONOMY_ID$$.json
 
-Pull the latest TDT docker image via following the steps defined in the project [GitHub Container Registry](https://github.com/brain-bican/taxonomy-development-tools/pkgs/container/taxonomy-development-tools). 
+You can view the taxonomy details in the online documentation by visiting [https://purl.brain-bican.org/taxonomy/$$TAXONOMY_ID$$](https://purl.brain-bican.org/taxonomy/$$TAXONOMY_ID$$)
 
-```
-docker pull ghcr.io/brain-bican/taxonomy-development-tools:latest
-```
+## Curate taxonomy
 
-## Load your data
+Set up and manage your taxonomy in just two easy steps:
 
-Place your data (ex. [AIT115_annotation_sheet.tsv](https://github.com/brain-bican/taxonomy-development-tools/tree/main/examples/nhp_basal_ganglia/AIT115_annotation_sheet.tsv)) and configuration file (ex. [ingestion_config.yaml](https://github.com/brain-bican/taxonomy-development-tools/tree/main/examples/nhp_basal_ganglia/ingestion_config.yaml)) into your project's `input_data` folder.  
+1. [Explore Your Taxonomy](#explore-taxonomy)
+1. [Load Your Data](#load-your-data)
 
-Run following command in your project root folder to ingest your data files:
 
-```
-bash ./run.sh make load_data
-```
+###  Explore Taxonomy
 
-## Browse
+First, clone this repository. Then, execute `bash ./run.sh make serve`  in the root directory of the project to launch the online data editor. Start browsing your data at [http://localhost:3000/annotation](http://localhost:3000/annotation).
 
-Run following command in your project root folder to run the online data editor:
-```
-bash ./run.sh make serve
-```
+For complete instructions, check out the [quick start quide](https://brain-bican.github.io/taxonomy-development-tools/QuickStart/)
 
-This command will print a set of logs including a log like `nanobot::serve: listening on 0.0.0.0:3000`. This means your web editor is ready, and you can start editing your data.
+###  Load your data
 
-You can start browsing web taxonomy editor from: [http://localhost:3000/table](http://localhost:3000/table)
+To upload your data into the editor, please refer to the [curation guide](https://brain-bican.github.io/taxonomy-development-tools/Curation/)
 
-_For further details see [Taxonomy Development Tools Documentation](https://brain-bican.github.io/taxonomy-development-tools/)_
+_This repository is powered by the Taxonomy Development Tools. For more information, see the [Taxonomy Development Tools Documentation](https://brain-bican.github.io/taxonomy-development-tools/)_

resources/repo_mkdocs.yml

@@ -0,0 +1,44 @@
+site_name: $$TAXONOMY_NAME$$
+repo_url: https://github.com/$$PROJECT_GITHUB_ORG$$/$$PROJECT_REPO$$
+repo_name: $$PROJECT_GITHUB_ORG$$/$$PROJECT_REPO$$
+docs_dir: docs
+
+nav:
+  - Taxonomy: index.md
+  - Cell Sets: cell_sets
+
+theme:
+  name: material
+  features:
+    - navigation.instant
+    - navigation.tabs
+    - navigation.tabs.sticky
+    - navigation.top
+    - navigation.tracking
+    - search.highlight
+    - search.share
+    - search.suggest
+    - toc.follow
+  palette:
+    # Palette toggle for light mode
+    - scheme: default
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+    # Palette toggle for dark mode
+    - scheme: slate
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+  favicon: assets/logo.webp
+  logo: assets/logo.webp
+
+plugins:
+  - search
+  - include_dir_to_nav
+  - mkdocs-nav-weight:
+      section_renamed: false
+      index_weight: -10
+      warning: true
+      reverse: false
+      headless_included: true

scripts/import.py

@@ -13,14 +13,15 @@
 from pathlib import Path
 from ruamel.yaml import YAML
 
+from tdta.git_utils import add_new_files_to_git
+
 from cas.ingest.ingest_user_table import ingest_user_data
 from cas.flatten_data_to_tables import serialize_to_tables
 from cas.file_utils import read_cas_json_file
 from cas_schema import schemas
 
 # see Dockerfile
 WORKSPACE = "/tools"
-GITHUB_SIZE_LIMIT = 100 * 1000 * 1000  # 100 MB
 
 
 @click.group()
@@ -141,25 +142,6 @@ def preserve_existing_annotations(cas_obj, curation_tables_folder):
         print("Existing annotations couldn't be found in the folder to copy: " + curation_tables_folder)
 
 
-def add_new_files_to_git(project_folder, new_files):
-    """
-    Runs git add command to add imported files to the version control.
-    Parameters:
-        project_folder: project folder path
-        new_files: imported/created file paths to add to the version control
-    """
-    for file_path in new_files:
-        if os.path.getsize(file_path) > GITHUB_SIZE_LIMIT:
-            zip_path = zip_file(file_path)
-            new_files.remove(file_path)
-            runcmd("cd {dir} && git add {zip_path}".format(dir=project_folder, zip_path=zip_path))
-            runcmd("cd {dir} && git reset {file_path}".format(dir=project_folder, file_path=file_path))
-
-    runcmd("cd {dir} && git add {files}".
-           format(dir=project_folder,
-                  files=" ".join([t.replace(project_folder, ".", 1) for t in new_files])))
-
-
 def unzip_files_in_folder(folder_path):
     files_in_folder = os.listdir(folder_path)
     for file_name in files_in_folder:
@@ -170,24 +152,6 @@ def unzip_files_in_folder(folder_path):
             print(f"Extracted '{zip_path}'.")
 
 
-def zip_file(file_path):
-    """
-    Zips the file if it exceeds the GitHub size limit.
-    Parameters:
-        file_path: file path to zip
-    Returns: zipped file path
-    """
-    folder = os.path.dirname(file_path)
-    base_name = os.path.basename(file_path)
-    zip_base = os.path.splitext(base_name)[0]
-
-    single_zip_path = os.path.join(folder, f"{zip_base}.zip")
-    with zipfile.ZipFile(single_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-        zipf.write(file_path, base_name)
-
-    return single_zip_path
-
-
 def add_user_table_to_nanobot(user_data_path, schema_folder, curation_tables_folder, cas_schema, delete_source=False, force=False):
     """
     Adds user data to the nanobot. Adds user table to the curation tables folder and updates the nanobot table schema.