Add UI for collections, fix local graphs and improve tables

.gitignore

@@ -3,6 +3,11 @@ files/
 uploads/
 server/
 ssh_config
+logs/
+
+# App testing files
+webapp/cypress/screenshots
+webapp/cypress/videos
 
 #starting material files
 greyGroup_chemInventory*

pydatalab/pydatalab/blocks/blocks.py

@@ -51,7 +51,7 @@ class DataBlock:
     cache: Optional[Dict[str, Any]] = None
     plot_functions: Optional[Sequence[Callable[[], None]]] = None
     # whether this datablock can operate on collection data, or just individual items
-    __supports_collections: bool = False
+    _supports_collections: bool = False
 
     def __init__(
         self,
@@ -63,10 +63,11 @@ def __init__(
         if dictionary is None:
             dictionary = {}
 
-        if item_id is None and not self.__supports_collections:
+        if item_id is None and not self._supports_collections:
+            breakpoint()
             raise RuntimeError(f"Must supply `item_id` to make {self.__class__.__name__}.")
 
-        if collection_id is not None and not self.__supports_collections:
+        if collection_id is not None and not self._supports_collections:
             raise RuntimeError(
                 f"This block ({self.__class__.__name__}) does not support collections."
             )
@@ -174,20 +175,20 @@ def update_from_web(self, data):
 class NotSupportedBlock(DataBlock):
     blocktype = "notsupported"
     description = "Block not supported"
-    __supports_collections = True
+    _supports_collections = True
 
 
 class CommentBlock(DataBlock):
     blocktype = "comment"
     description = "Comment"
-    __supports_collections = True
+    _supports_collections = True
 
 
 class MediaBlock(DataBlock):
     blocktype = "media"
     description = "Media"
     accepted_file_extensions = (".png", ".jpeg", ".jpg", ".tif", ".tiff", ".mp4", ".mov", ".webm")
-    __supports_collections = False
+    _supports_collections = False
 
     @property
     def plot_functions(self):
@@ -216,7 +217,7 @@ class NMRBlock(DataBlock):
     description = "Simple NMR Block"
     accepted_file_extensions = ".zip"
     defaults = {"process number": 1}
-    __supports_collections = False
+    _supports_collections = False
 
     @property
     def plot_functions(self):

pydatalab/pydatalab/models/traits.py

@@ -79,7 +79,6 @@ def add_missing_collection_relationships(cls, values):
         if len([d for d in values.get("relationships", []) if d.type == "collections"]) != len(
             values.get("collections", [])
         ):
-            breakpoint()
             raise RuntimeError("Relationships and collections mismatch")
 
         return values

pydatalab/pydatalab/routes/v0_1/blocks.py

@@ -115,7 +115,7 @@ def add_collection_data_block():
         )
 
     # get the new display_order:
-    display_order_result = flask_mongo.db.items.find_one(
+    display_order_result = flask_mongo.db.collections.find_one(
         {"collection_id": collection_id, **get_default_permissions(user_only=True)},
         {"display_order": 1},
     )
@@ -130,6 +130,9 @@ def add_collection_data_block():
     )
 
 
+add_collection_data_block.methods = ("POST",)  # type: ignore
+
+
 def _save_block_to_db(block: DataBlock) -> bool:
     """Save data for a single block within an item to the database,
     overwriting previous data saved there.

pydatalab/pydatalab/routes/v0_1/collections.py

@@ -16,7 +16,7 @@
 collection = Blueprint("collections", __name__)
 
 
-@collection.route("/collections/")
+@collection.route("/collections")
 def get_collections():
 
     collections = flask_mongo.db.collections.aggregate(
@@ -87,7 +87,7 @@ def get_collection(collection_id):
     )
 
 
-@collection.route("/collections/", methods=["PUT"])
+@collection.route("/collections", methods=["PUT"])
 def create_collection():
     request_json = request.get_json()  # noqa: F821 pylint: disable=undefined-variable
     data = request_json.get("data", {})
@@ -301,7 +301,7 @@ def delete_collection(collection_id: str):
     )
 
 
-@collection.route("/search-collections/", methods=["GET"])
+@collection.route("/search-collections", methods=["GET"])
 def search_collections():
     query = request.args.get("query", type=str)
     nresults = request.args.get("nresults", default=100, type=int)

pydatalab/pydatalab/routes/v0_1/graphs.py

@@ -1,19 +1,36 @@
-from typing import Callable, Dict, Optional
+from typing import Callable, Dict, Optional, Set
 
-from flask import jsonify
+from flask import jsonify, request
 
 from pydatalab.mongo import flask_mongo
 from pydatalab.routes.utils import get_default_permissions
 
 
-def get_graph_cy_format(item_id: Optional[str] = None):
+def get_graph_cy_format(item_id: Optional[str] = None, collection_id: Optional[str] = None):
+
+    collection_id = request.args.get("collection_id", type=str)
 
     if item_id is None:
+        if collection_id is not None:
+            collection_immutable_id = flask_mongo.db.collections.find_one(
+                {"collection_id": collection_id}, projection={"_id": 1}
+            )
+            if not collection_immutable_id:
+                raise RuntimeError("No collection {collection_id=} found.")
+            collection_immutable_id = collection_immutable_id["_id"]
+            query = {
+                "$and": [
+                    {"relationships.immutable_id": collection_immutable_id},
+                    {"relationships.type": "collections"},
+                ]
+            }
+        else:
+            query = {}
         all_documents = flask_mongo.db.items.find(
-            get_default_permissions(user_only=False),
+            {**query, **get_default_permissions(user_only=False)},
             projection={"item_id": 1, "name": 1, "type": 1, "relationships": 1},
         )
-        node_ids = {document["item_id"] for document in all_documents}
+        node_ids: Set[str] = {document["item_id"] for document in all_documents}
         all_documents.rewind()
 
     else:
@@ -27,56 +44,105 @@ def get_graph_cy_format(item_id: Optional[str] = None):
             )
         )
 
-        node_ids = {document["item_id"] for document in all_documents}
+        node_ids = {document["item_id"] for document in all_documents} | {
+            relationship.get("item_id")
+            for document in all_documents
+            for relationship in document.get("relationships", [])
+        }
         if len(node_ids) > 1:
+            or_query = [{"item_id": id} for id in node_ids if id != item_id]
             next_shell = flask_mongo.db.items.find(
                 {
-                    "$or": [
-                        *[{"item_id": id} for id in node_ids if id != item_id],
-                        *[{"relationships.item_id": id} for id in node_ids if id != item_id],
-                    ],
+                    "$or": or_query,
                     **get_default_permissions(user_only=False),
                 },
                 projection={"item_id": 1, "name": 1, "type": 1, "relationships": 1},
             )
 
-            node_ids = node_ids | {document["item_id"] for document in next_shell}
             all_documents.extend(next_shell)
+            node_ids = node_ids | {document["item_id"] for document in all_documents}
 
     nodes = []
     edges = []
-    for document in all_documents:
 
-        nodes.append(
-            {
-                "data": {
-                    "id": document["item_id"],
-                    "name": document["name"],
-                    "type": document["type"],
-                    "special": document["item_id"] == item_id,
-                }
-            }
-        )
+    # Collect the elements that have already been added to the graph, to avoid duplication
+    drawn_elements = set()
+    node_collections = set()
+    for document in all_documents:
 
-        if not document.get("relationships"):
-            continue
+        for relationship in document.get("relationships", []):
+            # only considering child-parent relationships
+            if relationship.get("type") == "collections" and not collection_id:
+                collection_data = flask_mongo.db.collections.find_one(
+                    {
+                        "_id": relationship["immutable_id"],
+                        **get_default_permissions(user_only=False),
+                    },
+                    projection={"collection_id": 1, "title": 1, "type": 1},
+                )
+                if collection_data:
+                    if relationship["immutable_id"] not in node_collections:
+                        _id = f'Collection: {collection_data["collection_id"]}'
+                        if _id not in drawn_elements:
+                            nodes.append(
+                                {
+                                    "data": {
+                                        "id": _id,
+                                        "name": collection_data["title"],
+                                        "type": collection_data["type"],
+                                        "shape": "triangle",
+                                    }
+                                }
+                            )
+                            node_collections.add(relationship["immutable_id"])
+                            drawn_elements.add(_id)
+
+                    source = f'Collection: {collection_data["collection_id"]}'
+                    target = document.get("item_id")
+                    edges.append(
+                        {
+                            "data": {
+                                "id": f"{source}->{target}",
+                                "source": source,
+                                "target": target,
+                                "value": 1,
+                            }
+                        }
+                    )
+                continue
 
-        for relationship in document["relationships"]:
+        for relationship in document.get("relationships", []):
             # only considering child-parent relationships:
-            if relationship["relation"] not in ("parent", "is_part_of"):
+            if relationship.get("relation") not in ("parent", "is_part_of"):
                 continue
 
             target = document["item_id"]
             source = relationship["item_id"]
             if source not in node_ids:
                 continue
-            edges.append(
+            edge_id = f"{source}->{target}"
+            if edge_id not in drawn_elements:
+                drawn_elements.add(edge_id)
+                edges.append(
+                    {
+                        "data": {
+                            "id": edge_id,
+                            "source": source,
+                            "target": target,
+                            "value": 1,
+                        }
+                    }
+                )
+
+        if document["item_id"] not in drawn_elements:
+            drawn_elements.add(document["item_id"])
+            nodes.append(
                 {
                     "data": {
-                        "id": f"{source}->{target}",
-                        "source": source,
-                        "target": target,
-                        "value": 1,
+                        "id": document["item_id"],
+                        "name": document["name"],
+                        "type": document["type"],
+                        "special": document["item_id"] == item_id,
                     }
                 }
             )

pydatalab/pydatalab/routes/v0_1/items.py

@@ -179,7 +179,7 @@ def creators_lookup() -> Dict:
             {
                 "$match": {
                     "$expr": {
-                        "$in": ["$_id", "$$creator_ids"],
+                        "$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}],
                     },
                 }
             },
@@ -211,7 +211,7 @@ def collections_lookup() -> Dict:
             {
                 "$match": {
                     "$expr": {
-                        "$in": ["$_id", "$$collection_ids"],
+                        "$in": ["$_id", {"$ifNull": ["$$collection_ids", []]}],
                     },
                     "type": "collections",
                 }