first pass at merging the explorer

application/database/db.py

@@ -736,7 +736,7 @@ def get_CREs(
             cres.append(cre)
         return cres
 
-    def export(self, dir: str = None, dry_run: bool = False) -> List[cre_defs.Document]:
+    def export(self, dir: str = "", dry_run: bool = False) -> List[cre_defs.Document]:
         """Exports the database to a CRE file collection on disk"""
         docs: Dict[str, cre_defs.Document] = {}
         cre, standard = None, None

application/frontend/src/const.ts

@@ -36,3 +36,4 @@ export const CRE = '/cre';
 export const GRAPH = '/graph';
 export const DEEPLINK = '/deeplink';
 export const BROWSEROOT = '/root_cres';
+export const EXPLORER = '/explorer';

application/frontend/src/pages/Explorer/explorer.scss

@@ -0,0 +1,105 @@
+#explorer-content {
+  font-family: Vollkorn, Ubuntu, Optima, Segoe, Segoe UI, Candara, Calibri, Arial, sans-serif;
+  margin: 40px;
+  text-align: left;
+}
+
+#explorer-content>.group {
+  border-top: 1px dotted lightgrey;
+  border-left: 6px solid lightgrey;
+  margin: 4px;
+  margin-left: 46px;
+  padding: 5px;
+  vertical-align: top;
+  background-color: rgba(200, 200, 200, 0.2);
+}
+
+#explorer-content>.doc-id {
+  display: inline-block;
+  border-radius: 6px;
+  margin: 3px;
+  padding: 3px;
+  background-color: #f8f8f8;
+  vertical-align: top;
+  font-size: 90%;
+}
+
+#explorer-content>.tag {
+  display: inline-block;
+  border: 1px solid lightgrey;
+  border-radius: 6px;
+  margin: 3px;
+  padding: 3px;
+  background-color: #f8f8f8;
+  vertical-align: top;
+  font-size: 90%;
+  max-width: 80px;
+  white-space: nowrap;
+  overflow: hidden;
+}
+
+#explorer-content>a {
+  text-decoration: none;
+}
+
+#explorer-content>.icon {
+  width: 140px;
+  height: 140px;
+  object-fit: cover;
+  border-radius: 4px;
+  margin-top: 26px;
+  margin-bottom: 20px;
+  filter: grayscale(100%);
+}
+
+#explorer-content>::placeholder {
+  color: lightgrey;
+  opacity: 1;
+}
+
+#explorer-content>:-ms-input-placeholder {
+  color: lightgrey;
+}
+
+#explorer-content>::-ms-input-placeholder {
+  color: lightgrey;
+}
+
+
+#explorer-content>div {
+  vertical-align: top;
+}
+
+#explorer-content>content {
+  margin-left: -20px;
+}
+
+#explorer-content>h1 {
+  font-size: 50px;
+  margin-bottom: 5px;
+  margin-left: 16px;
+}
+
+#explorer-content>img {
+  vertical-align: middle;
+  height: 80px;
+}
+
+b {
+  padding-top: 20px;
+  padding-left: 12px;
+}
+p{
+  color: grey; margin-left: 26px;
+}
+#explorer-wrapper{
+  margin-left: 24px; margin-top: 20px; margin-bottom: 20px; color: grey;
+}
+
+#filter{
+  font-size: 16px; height: 32px; width: 320px; margin-bottom: 10px;
+}
+#search-summary{
+  display: inline-block; vertical-align: middle;
+}
+#graphs{font-size: 80%; color: grey;}

application/frontend/src/pages/Explorer/explorer.tsx

@@ -0,0 +1,148 @@
+import './explorer.scss';
+
+import React, { useContext, useEffect, useMemo, useState } from 'react';
+import { useQuery } from 'react-query';
+import { useParams } from 'react-router-dom';
+
+import { DocumentNode } from '../../components/DocumentNode';
+import { ClearFilterButton, FilterButton } from '../../components/FilterButton/FilterButton';
+import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator';
+import { useEnvironment } from '../../hooks';
+import { applyFilters, filterContext } from '../../hooks/applyFilters';
+import { Document } from '../../types';
+import { groupLinksByType } from '../../utils';
+import { SearchResults } from '../Search/components/SearchResults';
+
+export const Explorer = () => {
+  const { apiUrl } = useEnvironment();
+  const [loading, setLoading] = useState<boolean>(false);
+  const [filter, setFilter] = useState("");
+  const [searchSummary, setSearchSummary] = useState(0);
+  const [data, setData] = useState<Document[]>()
+  const [rootCREs, setRootCREs] = useState<Document[]>()
+  const [filteredData, setFilteredData] = useState<Document[]>()
+
+  useQuery<{ data: Document }, string>(
+    'root_cres',
+    () =>
+      fetch(`${apiUrl}/root_cres`)
+        .then((res) => res.json())
+        .then((resjson) => {
+          setRootCREs(resjson.data);
+          return resjson;
+        }),
+    {
+      retry: false,
+      enabled: false,
+      onSettled: () => {
+        setLoading(false);
+      },
+    }
+  );
+  const docs = localStorage.getItem("documents")
+  useEffect(()=>{
+    if (docs != null) {
+      setData(JSON.parse(docs).sort((a, b) => (a.id + '').localeCompare(b.id + '')));
+      setFilteredData(data)
+    }
+  },[docs])
+
+  const query = useQuery(
+    'everything',
+    () => {
+      if (docs == null) {
+        fetch(`${apiUrl}/everything`)
+          .then((res) => { return res.json() })
+          .then((resjson) => {
+            return resjson.data
+          }).then((data) => {
+            if (data) {
+              localStorage.setItem("documents", JSON.stringify(data));
+              setData(data)
+            }
+          }),
+        {
+          retry: false,
+          enabled: false,
+          onSettled: () => {
+            setLoading(false);
+          },
+        }
+      }
+
+    }
+  );
+
+
+  useEffect(() => {
+    window.scrollTo(0, 0);
+    setLoading(true);
+    query.refetch();
+  }, []);
+
+
+  if (!data?.length) {
+    const docs = localStorage.getItem("documents")
+    if (docs) {
+      setData(JSON.parse(docs).sort((a, b) => (a.id + '').localeCompare(b.id + '')));
+      setFilteredData(data)
+    }
+  }
+
+  function processNode(item) {
+    if (!item) {
+      return (<></>)
+    }
+    return (
+      <div className="group" >
+        <div className="group-1">
+          <div className='group-2'>
+            <a target="_blank" href={"https://opencre.org/cre/" + item?.id}>
+              {item?.name}
+            </a>
+          </div>
+          <div /*style="font-size: 90%"*/>
+            {item?.links?.forEach(child => processNode(child))}
+          </div>
+        </div>
+      </div>
+    )
+  }
+  function update(event) {
+    setFilter(event.target.value)
+    setFilteredData(data?.filter(item => item.name.toLowerCase().includes(filter)))
+  }
+
+  return (<>
+    {/* <LoadingAndErrorIndicator loading={loading} error={query.error} /> */}
+    <div id="explorer-content">
+      <h1>
+        <img src="assets/logo.png" />
+        <b>Open CRE Explorer</b>
+      </h1>
+      <p>A visual explorer of Open Common Requirement Enumerations (CREs).
+        Data source: <a target="_blank" href="https://opencre.org/">opencre.org</a>.</p>
+
+      <div id="explorer-wrapper">
+        <div>
+          <input id="filter" type="text" placeholder="search..."
+            onKeyUp={update} />
+          <div id="search-summary"></div>
+        </div>
+        <div id="graphs">graphs (3D):
+          <a target="_blank" href="visuals/force-graph-3d-all.html">CRE dependencies</a> -
+          <a target="_blank" href="visuals/force-graph-3d-contains.html">hierarchy only</a> -
+          <a target="_blank" href="visuals/force-graph-3d-related.html">related only</a> |
+          <a target="_blank" href="visuals/force-graph-3d-linked.html">links to external standards</a> |
+          <a target="_blank" href="visuals/circles.html">zoomable circles</a>
+        </div>
+      </div>
+      <div id="content"></div>
+      {filteredData?.map(item => {
+        return processNode(item)
+      })}
+    </div>
+
+  </>
+  );
+};

application/frontend/src/routes.tsx

@@ -1,13 +1,14 @@
 import { ReactNode } from 'react';
 
-import { BROWSEROOT, CRE, DEEPLINK, GRAPH, INDEX, SEARCH, SECTION, SECTION_ID, STANDARD } from './const';
+import { BROWSEROOT, CRE, DEEPLINK, GRAPH, INDEX, SEARCH, SECTION, SECTION_ID, STANDARD, EXPLORER } from './const';
 import { CommonRequirementEnumeration, Graph, Search, Standard } from './pages';
 import { BrowseRootCres } from './pages/BrowseRootCres/browseRootCres';
 import { Chatbot } from './pages/chatbot/chatbot';
 import { Deeplink } from './pages/Deeplink/Deeplink';
 import { MembershipRequired } from './pages/MembershipRequired/MembershipRequired';
 import { SearchName } from './pages/Search/SearchName';
 import { StandardSection } from './pages/Standard/StandardSection';
+import { Explorer } from './pages/Explorer/explorer';
 
 export interface IRoute {
   path: string;
@@ -107,4 +108,10 @@ export const ROUTES: IRoute[] = [
     showHeader: true,
     showFilter: false,
   },
+  {
+    path: `${EXPLORER}`,
+    component: Explorer,
+    showHeader: true,
+    showFilter: false,
+  },
 ];

application/web/web_main.py

@@ -485,6 +485,16 @@ def logout():
     session.clear()
     return redirect("/")
 
+@app.route("/rest/v1/everything", methods=["GET"])
+def everything() -> Any:
+    database = db.Node_collection()
+    documents = database.export(dry_run=True)
+    if documents:
+        res = [doc.todict() for doc in documents]
+        result = {"data": res}
+        return jsonify(result)
+    abort(404)
+
 
 if __name__ == "__main__":
     app.run(use_reloader=False, debug=False)