diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7b0cbda8c..91392fdce 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -7,6 +7,9 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: '3.11.4' - name: Install python dependencies run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip chromium-browser libgbm1 && make install-deps - name: Test-e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94bb1ca15..65d07220c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,9 +7,10 @@ jobs: steps: - name: Check out code uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: '3.11.4' - name: Install python dependencies run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip && make install-deps - name: Test run: make test - - name: Test-e2e - run: make e2e \ No newline at end of file diff --git a/application/database/db.py b/application/database/db.py index 7f79a6b3d..ba495a62c 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -224,11 +224,14 @@ def add_cre(self, dbcre: CRE): if not self.connected: return self.driver.execute_query( - "MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})", - nid=dbcre.id, + "MERGE (n:CRE {id: $nid, name: $name, description: $description, doctype: $doctype, links: $links, metadata: $metadata, tags: $tags})", name=dbcre.name, + doctype="CRE", # dbcre.ntype, + nid=dbcre.id, description=dbcre.description, - external_id=dbcre.external_id, + links=[], # dbcre.links, + tags=dbcre.tags, + metadata="{}", # dbcre.metadata, database_="neo4j", ) @@ -236,20 +239,58 @@ def add_cre(self, dbcre: CRE): def add_dbnode(self, dbnode: Node): if not self.connected: return - # TODO: Add diffrent Node types - self.driver.execute_query( - "MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})", - nid=dbnode.id, - name=dbnode.name, - section=dbnode.section, - section_id=dbnode.section_id, - subsection=dbnode.subsection or "", - tags=dbnode.tags, - version=dbnode.version or "", - description=dbnode.description, - ntype=dbnode.ntype, - database_="neo4j", - ) + if dbnode.ntype == "Standard": + self.driver.execute_query( + "MERGE (n:Standard {id: $nid, name: $name, section: $section, sectionID: $sectionID, subsection: $subsection, tags: $tags, version: $version, description: $description, doctype: $doctype, links: $links, metadata: $metadata, hyperlink: $hyperlink})", + name=dbnode.name, + doctype=dbnode.ntype, + nid=dbnode.id, + description=dbnode.description, + links=[], # dbnode.links, + tags=dbnode.tags, + metadata="{}", # dbnode.metadata, + hyperlink="", # dbnode.hyperlink or "", + version=dbnode.version or "", + section=dbnode.section, + sectionID=dbnode.section_id, # dbnode.sectionID, + subsection=dbnode.subsection or "", + database_="neo4j", + ) + return + if dbnode.ntype == "Tool": + self.driver.execute_query( + "MERGE (n:Tool {id: $nid, name: $name, section: $section, sectionID: $sectionID, subsection: $subsection, tags: $tags, version: $version, description: $description, doctype: $doctype, links: $links, metadata: $metadata, hyperlink: $hyperlink, tooltype: $tooltype})", + name=dbnode.name, + doctype=dbnode.ntype, + nid=dbnode.id, + description=dbnode.description, + links=[], # dbnode.links, + tags=dbnode.tags, + metadata="{}", # dbnode.metadata, + hyperlink="", # dbnode.hyperlink or "", + version=dbnode.version or "", + section=dbnode.section, + sectionID=dbnode.section_id, # dbnode.sectionID, + subsection=dbnode.subsection or "", + tooltype="", # dbnode.tooltype, + database_="neo4j", + ) + return + if dbnode.ntype == "Code": + self.driver.execute_query( + "MERGE (n:Code {id: $nid, name: $name, section: $section, sectionID: $sectionID, subsection: $subsection, tags: $tags, version: $version, description: $description, doctype: $doctype, links: $links, metadata: $metadata, hyperlink: $hyperlink})", + name=dbnode.name, + doctype=dbnode.ntype, + nid=dbnode.id, + description=dbnode.description, + links=[], # dbnode.links, + tags=dbnode.tags, + metadata="{}", # dbnode.metadata, + hyperlink="", # dbnode.hyperlink or "", + version=dbnode.version or "", + ) + return + raise Exception(f"Unknown DB type: {dbnode.ntype}") @classmethod def link_CRE_to_CRE(self, id1, id2, link_type): @@ -272,7 +313,7 @@ def link_CRE_to_Node(self, CRE_id, node_id, link_type): if not self.connected: return self.driver.execute_query( - "MATCH (a:CRE), (b:Node) " + "MATCH (a:CRE), (b:Standard|Tool) " "WHERE a.id = $aID AND b.id = $bID " "CALL apoc.create.relationship(a,$relType, {},b) " "YIELD rel " @@ -289,7 +330,7 @@ def gap_analysis(self, name_1, name_2): return None, None base_standard, _, _ = self.driver.execute_query( """ - MATCH (BaseStandard:Node {name: $name1}) + MATCH (BaseStandard:Standard|Tool {name: $name1}) RETURN BaseStandard """, name1=name_1, @@ -298,8 +339,8 @@ def gap_analysis(self, name_1, name_2): path_records_all, _, _ = self.driver.execute_query( """ - OPTIONAL MATCH (BaseStandard:Node {name: $name1}) - OPTIONAL MATCH (CompareStandard:Node {name: $name2}) + OPTIONAL MATCH (BaseStandard:Standard|Tool {name: $name1}) + OPTIONAL MATCH (CompareStandard:Standard|Tool {name: $name2}) OPTIONAL MATCH p = shortestPath((BaseStandard)-[*..20]-(CompareStandard)) WITH p WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n.name = $name1 or n.name = $name2) @@ -311,8 +352,8 @@ def gap_analysis(self, name_1, name_2): ) path_records, _, _ = self.driver.execute_query( """ - OPTIONAL MATCH (BaseStandard:Node {name: $name1}) - OPTIONAL MATCH (CompareStandard:Node {name: $name2}) + OPTIONAL MATCH (BaseStandard:Standard|Tool {name: $name1}) + OPTIONAL MATCH (CompareStandard:Standard|Tool {name: $name2}) OPTIONAL MATCH p = shortestPath((BaseStandard)-[:(LINKED_TO|CONTAINS)*..20]-(CompareStandard)) WITH p WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n.name = $name1 or n.name = $name2) @@ -325,70 +366,94 @@ def gap_analysis(self, name_1, name_2): def format_segment(seg): return { - "start": { - "name": seg.start_node["name"], - "sectionID": seg.start_node["section_id"], - "section": seg.start_node["section"], - "subsection": seg.start_node["subsection"], - "description": seg.start_node["description"], - "id": seg.start_node["id"], - }, - "end": { - "name": seg.end_node["name"], - "sectionID": seg.end_node["section_id"], - "section": seg.end_node["section"], - "subsection": seg.end_node["subsection"], - "description": seg.end_node["description"], - "id": seg.end_node["id"], - }, + "start": NEO_DB.parse_node(seg.start_node), + "end": NEO_DB.parse_node(seg.end_node), "relationship": seg.type, } def format_path_record(rec): return { - "start": { - "name": rec.start_node["name"], - "sectionID": rec.start_node["section_id"], - "section": rec.start_node["section"], - "subsection": rec.start_node["subsection"], - "description": rec.start_node["description"], - "id": rec.start_node["id"], - }, - "end": { - "name": rec.end_node["name"], - "sectionID": rec.end_node["section_id"], - "section": rec.end_node["section"], - "subsection": rec.end_node["subsection"], - "description": rec.end_node["description"], - "id": rec.end_node["id"], - }, + "start": NEO_DB.parse_node(rec.start_node), + "end": NEO_DB.parse_node(rec.end_node), "path": [format_segment(seg) for seg in rec.relationships], } - def format_record(rec): - return { - "name": rec["name"], - "sectionID": rec["section_id"], - "section": rec["section"], - "subsection": rec["subsection"], - "description": rec["description"], - "id": rec["id"], - } - - return [format_record(rec["BaseStandard"]) for rec in base_standard], [ + return [NEO_DB.parse_node(rec["BaseStandard"]) for rec in base_standard], [ format_path_record(rec["p"]) for rec in (path_records + path_records_all) ] @classmethod - def standards(self): + def standards(self) -> List[str]: if not self.connected: return records, _, _ = self.driver.execute_query( - 'MATCH (n:Node {ntype: "Standard"}) ' "RETURN collect(distinct n.name)", + "MATCH (n:Standard|Tool) " "RETURN collect(distinct n.name)", database_="neo4j", ) return records[0][0] + @staticmethod + def parse_node(node: neo4j.graph.Node) -> cre_defs.Document: + name = node["name"] + id = node["id"] if "id" in node else None + description = node["description"] if "description" in node else None + # links = [self.parse_link(link) for link in node["links"]] + tags = node["tags"] + # metadata = node["metadata"] + if cre_defs.Credoctypes.Code.value in node.labels: + return cre_defs.Code( + name=name, + id=id, + description=description, + # links=links, + tags=tags, + # metadata=metadata, + # hyperlink=(node["hyperlink"] if "hyperlink" in node else None), + version=(node["version"] if "version" in node else None), + ) + if cre_defs.Credoctypes.Standard.value in node.labels: + return cre_defs.Standard( + name=name, + id=id, + description=description, + # links=links, + tags=tags, + # metadata=metadata, + # hyperlink=(node["hyperlink"] if "hyperlink" in node else None), + version=(node["version"] if "version" in node else None), + section=node["section"], + sectionID=node["sectionID"], + subsection=(node["subsection"] if "subsection" in node else None), + ) + if cre_defs.Credoctypes.Tool.value in node.labels: + return cre_defs.Tool( + name=name, + id=id, + description=description, + # links=links, + tags=tags, + # metadata=metadata, + # hyperlink=(node["hyperlink"] if "hyperlink" in node else None), + version=(node["version"] if "version" in node else None), + section=node["section"], + sectionID=node["sectionID"], + subsection=(node["subsection"] if "subsection" in node else None), + ) + if cre_defs.Credoctypes.CRE.value in node.labels: + return cre_defs.CRE( + name=name, + id=id, + description=description, + # links=links, + tags=tags, + # metadata=metadata, + ) + raise Exception(f"Unknown node {node.labels}") + + # @classmethod + # def parse_link(self, link): + # return cre_defs.Link(ltype=link["ltype"], tags=link["tags"]) + class CRE_Graph: graph: nx.Graph = None @@ -1298,20 +1363,18 @@ def find_path_between_nodes( return res def gap_analysis(self, node_names: List[str]): - if not self.neo_db.connected: - return None base_standard, paths = self.neo_db.gap_analysis(node_names[0], node_names[1]) if base_standard is None: return None grouped_paths = {} for node in base_standard: - key = node["id"] + key = node.id if key not in grouped_paths: grouped_paths[key] = {"start": node, "paths": {}} for path in paths: - key = path["start"]["id"] - end_key = path["end"]["id"] + key = path["start"].id + end_key = path["end"].id path["score"] = get_path_score(path) del path["start"] if end_key in grouped_paths[key]["paths"]: @@ -1321,7 +1384,7 @@ def gap_analysis(self, node_names: List[str]): grouped_paths[key]["paths"][end_key] = path return grouped_paths - def standards(self): + def standards(self) -> List[str]: return self.neo_db.standards() def text_search(self, text: str) -> List[Optional[cre_defs.Document]]: diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 833407f23..78d8d9aea 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -16,19 +16,29 @@ import { import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; import { useEnvironment } from '../../hooks'; +import { GapAnalysisPathStart } from '../../types'; +import { getDocumentDisplayName } from '../../utils'; const GetSegmentText = (segment, segmentID) => { let textPart = segment.end; let nextID = segment.end.id; - let arrow = '->'; + let arrow = ; if (segmentID !== segment.start.id) { textPart = segment.start; nextID = segment.start.id; - arrow = '<-'; + arrow = ; } - const text = `${arrow} ${segment.relationship} ${arrow} ${textPart.name} ${textPart.sectionID ?? ''} ${ - textPart.section ?? '' - } ${textPart.subsection ?? ''} ${textPart.description ?? ''}`; + const text = ( + <> +
+ {arrow}{' '} + + {segment.relationship.replace('_', ' ').toLowerCase()} + +
{getDocumentDisplayName(textPart, true)} {textPart.section ?? ''} {textPart.subsection ?? ''}{' '} + {textPart.description ?? ''} + + ); return { text, nextID }; }; @@ -48,7 +58,7 @@ export const GapAnalysis = () => { const [CompareStandard, setCompareStandard] = useState( searchParams.get('compare') ?? '' ); - const [gapAnalysis, setGapAnalysis] = useState(); + const [gapAnalysis, setGapAnalysis] = useState>(); const [activeIndex, SetActiveIndex] = useState(); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -66,6 +76,9 @@ export const GapAnalysis = () => { return 'Orange'; }; + const GetStrongPathsCount = (paths) => + Math.max(Object.values(paths).filter((x) => GetStrength(x.score) === 'Strong').length, 3); + useEffect(() => { const fetchData = async () => { const result = await axios.get(`${apiUrl}/standards`); @@ -182,43 +195,31 @@ export const GapAnalysis = () => {

- - {gapAnalysis[key].start.name} {gapAnalysis[key].start.section}{' '} - {gapAnalysis[key].start.subsection} - + {getDocumentDisplayName(gapAnalysis[key].start, true)}{' '} - {' '} -
- {gapAnalysis[key].start.sectionID} - {gapAnalysis[key].start.description} +

{Object.values(gapAnalysis[key].paths) .sort((a, b) => a.score - b.score) - .slice(0, 3) + .slice(0, GetStrongPathsCount(gapAnalysis[key].paths)) .map((path) => { let segmentID = gapAnalysis[key].start.id; return ( { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} trigger={ - {path.end.name} {path.end.sectionID} {path.end.section} {path.end.subsection}{' '} - {path.end.description} ( + {getDocumentDisplayName(path.end, true)} ( {GetStrength(path.score)}:{path.score} @@ -231,7 +232,16 @@ export const GapAnalysis = () => { } - /> + > + + {getDocumentDisplayName(gapAnalysis[key].start, true)} + {path.path.map((segment) => { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + })} + +
); @@ -248,25 +258,22 @@ export const GapAnalysis = () => { {Object.values(gapAnalysis[key].paths) .sort((a, b) => a.score - b.score) - .slice(3, gapAnalysis[key].paths.length) + .slice( + GetStrongPathsCount(gapAnalysis[key].paths), + Object.keys(gapAnalysis[key].paths).length + ) .map((path) => { let segmentID = gapAnalysis[key].start.id; return ( { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; - return text; - }) - .join('')} trigger={ - {path.end.name} {path.end.sectionID} {path.end.section}{' '} - {path.end.subsection} {path.end.description}( + {getDocumentDisplayName(path.end, true)} ( {GetStrength(path.score)}:{path.score} @@ -279,7 +286,16 @@ export const GapAnalysis = () => { } - /> + > + + {getDocumentDisplayName(gapAnalysis[key].start, true)} + {path.path.map((segment) => { + const { text, nextID } = GetSegmentText(segment, segmentID); + segmentID = nextID; + return text; + })} + +
); diff --git a/application/frontend/src/types.ts b/application/frontend/src/types.ts index c8b7cec72..64d73cdca 100644 --- a/application/frontend/src/types.ts +++ b/application/frontend/src/types.ts @@ -19,3 +19,19 @@ export interface LinkedDocument { document: Document; ltype: string; } + +interface GapAnalysisPathSegment { + start: Document; + end: Document; + relationship: string; +} + +interface GapAnalysisPath { + end: Document; + path: GapAnalysisPathSegment[]; +} + +export interface GapAnalysisPathStart { + start: Document; + paths: Record; +} diff --git a/application/frontend/src/utils/document.ts b/application/frontend/src/utils/document.ts index 07b5a784d..1e01d2b35 100644 --- a/application/frontend/src/utils/document.ts +++ b/application/frontend/src/utils/document.ts @@ -7,14 +7,14 @@ import { } from '../const'; import { Document, LinkedDocument } from '../types'; -export const getDocumentDisplayName = (document: Document) => { +export const getDocumentDisplayName = (document: Document, noID = false) => { // [document.doctype, document.id, document.name, document.section, document.subsection].filter(Boolean).join(' - '); // format: Standard - ASVS - V1.1 if (!document) { return ''; } return [ document.doctype, - document.id, + noID ? '' : document.id, document.name, document.version, document.sectionID, diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index 86fe6ea6a..f7de7bc82 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -175,6 +175,7 @@ def generate_embeddings( ) # cls.cre_embeddings[id] = embedding + class PromptHandler: def __init__(self, database: db.Node_collection) -> None: self.ai_client = None diff --git a/application/tests/db_test.py b/application/tests/db_test.py index 714b11423..d79671bc2 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -9,6 +9,7 @@ from pprint import pprint from pydoc import doc from typing import Any, Dict, List, Union +import neo4j import yaml from application import create_app, sqla # type: ignore @@ -1139,9 +1140,12 @@ def test_get_root_cres(self): self.maxDiff = None self.assertEqual(root_cres, [cres[0], cres[1], cres[7]]) - def test_gap_analysis_disconnected(self): + @patch.object(db.NEO_DB, "gap_analysis") + def test_gap_analysis_disconnected(self, gap_mock): collection = db.Node_collection() collection.neo_db.connected = False + gap_mock.return_value = (None, None) + self.assertEqual(collection.gap_analysis(["a", "b"]), None) @patch.object(db.NEO_DB, "gap_analysis") @@ -1157,9 +1161,10 @@ def test_gap_analysis_no_links(self, gap_mock): collection = db.Node_collection() collection.neo_db.connected = True - gap_mock.return_value = ([{"id": 1}], []) + gap_mock.return_value = ([defs.CRE(name="bob", id=1)], []) self.assertEqual( - collection.gap_analysis(["a", "b"]), {1: {"start": {"id": 1}, "paths": {}}} + collection.gap_analysis(["a", "b"]), + {1: {"start": defs.CRE(name="bob", id=1), "paths": {}}}, ) @patch.object(db.NEO_DB, "gap_analysis") @@ -1168,30 +1173,32 @@ def test_gap_analysis_one_link(self, gap_mock): collection.neo_db.connected = True path = [ { - "end": { - "id": 1, - }, + "end": defs.CRE(name="bob", id=1), "relationship": "LINKED_TO", - "start": { - "id": "a", - }, + "start": defs.CRE(name="bob", id="a"), }, { - "end": { - "id": 2, - }, + "end": defs.CRE(name="bob", id=2), "relationship": "LINKED_TO", - "start": {"id": "a"}, + "start": defs.CRE(name="bob", id="a"), }, ] gap_mock.return_value = ( - [{"id": 1}], - [{"start": {"id": 1}, "end": {"id": 2}, "path": path}], + [defs.CRE(name="bob", id=1)], + [ + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path, + } + ], ) expected = { 1: { - "start": {"id": 1}, - "paths": {2: {"end": {"id": 2}, "path": path, "score": 0}}, + "start": defs.CRE(name="bob", id=1), + "paths": { + 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} + }, } } self.assertEqual(collection.gap_analysis(["a", "b"]), expected) @@ -1202,51 +1209,49 @@ def test_gap_analysis_duplicate_link_path_existing_lower(self, gap_mock): collection.neo_db.connected = True path = [ { - "end": { - "id": 1, - }, + "end": defs.CRE(name="bob", id=1), "relationship": "LINKED_TO", - "start": { - "id": "a", - }, + "start": defs.CRE(name="bob", id="a"), }, { - "end": { - "id": 2, - }, + "end": defs.CRE(name="bob", id=2), "relationship": "LINKED_TO", - "start": {"id": "a"}, + "start": defs.CRE(name="bob", id="a"), }, ] path2 = [ { - "end": { - "id": 1, - }, + "end": defs.CRE(name="bob", id=1), "relationship": "LINKED_TO", - "start": { - "id": "a", - }, + "start": defs.CRE(name="bob", id="a"), }, { - "end": { - "id": 2, - }, + "end": defs.CRE(name="bob", id=2), "relationship": "RELATED", - "start": {"id": "a"}, + "start": defs.CRE(name="bob", id="a"), }, ] gap_mock.return_value = ( - [{"id": 1}], + [defs.CRE(name="bob", id=1)], [ - {"start": {"id": 1}, "end": {"id": 2}, "path": path}, - {"start": {"id": 1}, "end": {"id": 2}, "path": path2}, + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path, + }, + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path2, + }, ], ) expected = { 1: { - "start": {"id": 1}, - "paths": {2: {"end": {"id": 2}, "path": path, "score": 0}}, + "start": defs.CRE(name="bob", id=1), + "paths": { + 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} + }, } } self.assertEqual(collection.gap_analysis(["a", "b"]), expected) @@ -1257,55 +1262,202 @@ def test_gap_analysis_duplicate_link_path_existing_higher(self, gap_mock): collection.neo_db.connected = True path = [ { - "end": { - "id": 1, - }, + "end": defs.CRE(name="bob", id=1), "relationship": "LINKED_TO", - "start": { - "id": "a", - }, + "start": defs.CRE(name="bob", id="a"), }, { - "end": { - "id": 2, - }, + "end": defs.CRE(name="bob", id=2), "relationship": "LINKED_TO", - "start": {"id": "a"}, + "start": defs.CRE(name="bob", id="a"), }, ] path2 = [ { - "end": { - "id": 1, - }, + "end": defs.CRE(name="bob", id=1), "relationship": "LINKED_TO", - "start": { - "id": "a", - }, + "start": defs.CRE(name="bob", id="a"), }, { - "end": { - "id": 2, - }, + "end": defs.CRE(name="bob", id=2), "relationship": "RELATED", - "start": {"id": "a"}, + "start": defs.CRE(name="bob", id="a"), }, ] gap_mock.return_value = ( - [{"id": 1}], + [defs.CRE(name="bob", id=1)], [ - {"start": {"id": 1}, "end": {"id": 2}, "path": path2}, - {"start": {"id": 1}, "end": {"id": 2}, "path": path}, + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path2, + }, + { + "start": defs.CRE(name="bob", id=1), + "end": defs.CRE(name="bob", id=2), + "path": path, + }, ], ) expected = { 1: { - "start": {"id": 1}, - "paths": {2: {"end": {"id": 2}, "path": path, "score": 0}}, + "start": defs.CRE(name="bob", id=1), + "paths": { + 2: {"end": defs.CRE(name="bob", id=2), "path": path, "score": 0} + }, } } self.assertEqual(collection.gap_analysis(["a", "b"]), expected) + def test_neo_db_parse_node_code(self): + name = "name" + id = "id" + description = "description" + tags = "tags" + version = "version" + expected = defs.Code( + name=name, + id=id, + description=description, + tags=tags, + version=version, + ) + graph_node = neo4j.graph.Node( + None, + "123", + "id", + n_labels=[defs.Credoctypes.Code.value], + properties={ + "name": name, + "id": id, + "description": description, + "tags": tags, + "version": version, + }, + ) + self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + + def test_neo_db_parse_node_standard(self): + name = "name" + id = "id" + description = "description" + tags = "tags" + version = "version" + section = "section" + sectionID = "sectionID" + subsection = "subsection" + expected = defs.Standard( + name=name, + id=id, + description=description, + tags=tags, + version=version, + section=section, + sectionID=sectionID, + subsection=subsection, + ) + graph_node = neo4j.graph.Node( + None, + "123", + "id", + n_labels=[defs.Credoctypes.Standard.value], + properties={ + "name": name, + "id": id, + "description": description, + "tags": tags, + "version": version, + "section": section, + "sectionID": sectionID, + "subsection": subsection, + }, + ) + self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + + def test_neo_db_parse_node_tool(self): + name = "name" + id = "id" + description = "description" + tags = "tags" + version = "version" + section = "section" + sectionID = "sectionID" + subsection = "subsection" + expected = defs.Tool( + name=name, + id=id, + description=description, + tags=tags, + version=version, + section=section, + sectionID=sectionID, + subsection=subsection, + ) + graph_node = neo4j.graph.Node( + None, + "123", + "id", + n_labels=[defs.Credoctypes.Tool.value], + properties={ + "name": name, + "id": id, + "description": description, + "tags": tags, + "version": version, + "section": section, + "sectionID": sectionID, + "subsection": subsection, + }, + ) + self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + + def test_neo_db_parse_node_cre(self): + name = "name" + id = "id" + description = "description" + tags = "tags" + expected = defs.CRE( + name=name, + id=id, + description=description, + tags=tags, + ) + graph_node = neo4j.graph.Node( + None, + "123", + "id", + n_labels=[defs.Credoctypes.CRE.value], + properties={ + "name": name, + "id": id, + "description": description, + "tags": tags, + }, + ) + self.assertEqual(db.NEO_DB.parse_node(graph_node), expected) + + def test_neo_db_parse_node_unknown(self): + name = "name" + id = "id" + description = "description" + tags = "tags" + graph_node = neo4j.graph.Node( + None, + "123", + "id", + n_labels=["ABC"], + properties={ + "name": name, + "id": id, + "description": description, + "tags": tags, + }, + ) + with self.assertRaises(Exception) as cm: + db.NEO_DB.parse_node(graph_node) + + self.assertEqual(str(cm.exception), "Unknown node frozenset({'ABC'})") + def test_get_embeddings_by_doc_type_paginated(self): """Given: a range of embedding for Nodes and a range of embeddings for CREs when called with doc_type CRE return the cre embeddings diff --git a/application/tests/gap_analysis_test.py b/application/tests/gap_analysis_test.py index 396da8ee4..b4bf1a7ed 100644 --- a/application/tests/gap_analysis_test.py +++ b/application/tests/gap_analysis_test.py @@ -1,4 +1,5 @@ import unittest +from application.defs import cre_defs from application.utils.gap_analysis import ( get_path_score, @@ -16,47 +17,57 @@ def setUp(self) -> None: return None def test_get_relation_direction_UP(self): - step = {"start": {"id": "123"}, "end": {"id": "234"}} + step = { + "start": cre_defs.CRE(name="bob", id="123"), + "end": cre_defs.CRE(name="bob", id="234"), + } self.assertEqual(get_relation_direction(step, "123"), "UP") def test_get_relation_direction_DOWN(self): - step = {"start": {"id": "123"}, "end": {"id": "234"}} + step = { + "start": cre_defs.CRE(name="bob", id="123"), + "end": cre_defs.CRE(name="bob", id="234"), + } self.assertEqual(get_relation_direction(step, "234"), "DOWN") def test_get_next_id_start(self): - step = {"start": {"id": "123"}, "end": {"id": "234"}} + step = { + "start": cre_defs.CRE(name="bob", id="123"), + "end": cre_defs.CRE(name="bob", id="234"), + } self.assertEqual(get_next_id(step, "234"), "123") def test_get_next_id_end(self): - step = {"start": {"id": "123"}, "end": {"id": "234"}} + step = { + "start": cre_defs.CRE(name="bob", id="123"), + "end": cre_defs.CRE(name="bob", id="234"), + } self.assertEqual(get_next_id(step, "123"), "234") def test_get_path_score_direct_siblings_returns_zero(self): path = { - "start": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "start": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), + "end": cre_defs.CRE(name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d"), "path": [ { - "end": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, + "end": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), "relationship": "LINKED_TO", - "start": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "start": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), }, { - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "end": cre_defs.CRE( + name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d" + ), "relationship": "LINKED_TO", - "start": { - "id": "e2ac59b2-c1d8-4525-a6b3-155d480aecc9", - }, + "start": cre_defs.CRE( + name="bob", id="e2ac59b2-c1d8-4525-a6b3-155d480aecc9" + ), }, ], } @@ -64,39 +75,33 @@ def test_get_path_score_direct_siblings_returns_zero(self): def test_get_path_score_one_up_returns_one_up_penaltiy(self): path = { - "start": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "start": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), + "end": cre_defs.CRE(name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d"), "path": [ { - "end": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, + "end": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), "relationship": "LINKED_TO", - "start": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "start": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), }, { - "end": { - "id": "123", - }, + "end": cre_defs.CRE(name="bob", id="123"), "relationship": "CONTAINS", - "start": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "start": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), }, { - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "end": cre_defs.CRE( + name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d" + ), "relationship": "LINKED_TO", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, ], } @@ -104,39 +109,33 @@ def test_get_path_score_one_up_returns_one_up_penaltiy(self): def test_get_path_score_one_down_one_returns_one_down_penaltiy(self): path = { - "start": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "start": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), + "end": cre_defs.CRE(name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d"), "path": [ { - "end": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, + "end": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), "relationship": "LINKED_TO", - "start": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "start": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), }, { - "end": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "end": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), "relationship": "CONTAINS", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, { - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "end": cre_defs.CRE( + name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d" + ), "relationship": "LINKED_TO", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, ], } @@ -144,39 +143,33 @@ def test_get_path_score_one_down_one_returns_one_down_penaltiy(self): def test_get_path_score_related_returns_related_penalty(self): path = { - "start": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "start": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), + "end": cre_defs.CRE(name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d"), "path": [ { - "end": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, + "end": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), "relationship": "LINKED_TO", - "start": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "start": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), }, { - "end": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "end": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), "relationship": "RELATED", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, { - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "end": cre_defs.CRE( + name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d" + ), "relationship": "LINKED_TO", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, ], } @@ -184,57 +177,45 @@ def test_get_path_score_related_returns_related_penalty(self): def test_get_path_score_one_of_each_returns_penalty(self): path = { - "start": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "start": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), + "end": cre_defs.CRE(name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d"), "path": [ { - "end": { - "id": "029f7cd7-ef2f-4f25-b0d2-3227cde4b34b", - }, + "end": cre_defs.CRE( + name="bob", id="029f7cd7-ef2f-4f25-b0d2-3227cde4b34b" + ), "relationship": "LINKED_TO", - "start": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "start": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), }, { - "end": { - "id": "07bc9f6f-5387-4dc6-b277-0022ed76049f", - }, + "end": cre_defs.CRE( + name="bob", id="07bc9f6f-5387-4dc6-b277-0022ed76049f" + ), "relationship": "CONTAINS", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, { - "end": { - "id": "456", - }, + "end": cre_defs.CRE(name="bob", id="456"), "relationship": "RELATED", - "start": { - "id": "123", - }, + "start": cre_defs.CRE(name="bob", id="123"), }, { - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "end": cre_defs.CRE( + name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d" + ), "relationship": "CONTAINS", - "start": { - "id": "456", - }, + "start": cre_defs.CRE(name="bob", id="456"), }, { - "end": { - "id": "7d030730-14cc-4c43-8927-f2d0f5fbcf5d", - }, + "end": cre_defs.CRE( + name="bob", id="7d030730-14cc-4c43-8927-f2d0f5fbcf5d" + ), "relationship": "LINKED_TO", - "start": { - "id": "456", - }, + "start": cre_defs.CRE(name="bob", id="456"), }, ], } diff --git a/application/utils/gap_analysis.py b/application/utils/gap_analysis.py index 147fef3d3..3484f5fc1 100644 --- a/application/utils/gap_analysis.py +++ b/application/utils/gap_analysis.py @@ -9,7 +9,7 @@ def get_path_score(path): score = 0 - previous_id = path["start"]["id"] + previous_id = path["start"].id for step in path["path"]: penalty_type = step["relationship"] @@ -21,12 +21,12 @@ def get_path_score(path): def get_relation_direction(step, previous_id): - if step["start"]["id"] == previous_id: + if step["start"].id == previous_id: return "UP" return "DOWN" def get_next_id(step, previous_id): - if step["start"]["id"] == previous_id: - return step["end"]["id"] - return step["start"]["id"] + if step["start"].id == previous_id: + return step["end"].id + return step["start"].id diff --git a/requirements.txt b/requirements.txt index 8be294463..6659a57bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ compliance-trestle coverage dacite docx==0.2.4 +Flask==2.3.2 Flask_Caching==2.0.2 flask_compress==1.13 Flask_Cors==4.0.0 @@ -26,13 +27,13 @@ python_markdown_maker==1.0 scikit_learn scipy==1.11.2 semver -setuptools==66.1.1 +setuptools==68.2.2 simplify_docx==0.1.2 SQLAlchemy compliance-trestle nose==1.3.7 numpy==1.23.0 -neo4j==5.11.0 +neo4j openapi-schema-validator==0.3.4 openapi-spec-validator==0.5.1 openpyxl==3.1.0