Skip to content

Commit

Permalink
Merge pull request #377 from OWASP/neo4j-standard-object
Browse files Browse the repository at this point in the history 
Neo4j standard object
  • Loading branch information
@john681611
john681611 committed Sep 19, 2023
2 parents dca3961 + 32ae469 commit e224c9f
Show file tree
Hide file tree
Showing 11 changed files with 552 additions and 318 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/e2e.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: '3.11.4'
- name: Install python dependencies
run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip chromium-browser libgbm1 && make install-deps
- name: Test-e2e
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v2
- uses: actions/setup-python@v4
with:
python-version: '3.11.4'
- name: Install python dependencies
run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip && make install-deps
- name: Test
run: make test
- name: Test-e2e
run: make e2e
211 changes: 137 additions & 74 deletions application/database/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,32 +224,73 @@ def add_cre(self, dbcre: CRE):
if not self.connected:
return
self.driver.execute_query(
"MERGE (n:CRE {id: $nid, name: $name, description: $description, external_id: $external_id})",
nid=dbcre.id,
"MERGE (n:CRE {id: $nid, name: $name, description: $description, doctype: $doctype, links: $links, metadata: $metadata, tags: $tags})",
name=dbcre.name,
doctype="CRE", # dbcre.ntype,
nid=dbcre.id,
description=dbcre.description,
external_id=dbcre.external_id,
links=[], # dbcre.links,
tags=dbcre.tags,
metadata="{}", # dbcre.metadata,
database_="neo4j",
)

@classmethod
def add_dbnode(self, dbnode: Node):
if not self.connected:
return
# TODO: Add diffrent Node types
self.driver.execute_query(
"MERGE (n:Node {id: $nid, name: $name, section: $section, section_id: $section_id, subsection: $subsection, tags: $tags, version: $version, description: $description, ntype: $ntype})",
nid=dbnode.id,
name=dbnode.name,
section=dbnode.section,
section_id=dbnode.section_id,
subsection=dbnode.subsection or "",
tags=dbnode.tags,
version=dbnode.version or "",
description=dbnode.description,
ntype=dbnode.ntype,
database_="neo4j",
)
if dbnode.ntype == "Standard":
self.driver.execute_query(
"MERGE (n:Standard {id: $nid, name: $name, section: $section, sectionID: $sectionID, subsection: $subsection, tags: $tags, version: $version, description: $description, doctype: $doctype, links: $links, metadata: $metadata, hyperlink: $hyperlink})",
name=dbnode.name,
doctype=dbnode.ntype,
nid=dbnode.id,
description=dbnode.description,
links=[], # dbnode.links,
tags=dbnode.tags,
metadata="{}", # dbnode.metadata,
hyperlink="", # dbnode.hyperlink or "",
version=dbnode.version or "",
section=dbnode.section,
sectionID=dbnode.section_id, # dbnode.sectionID,
subsection=dbnode.subsection or "",
database_="neo4j",
)
return
if dbnode.ntype == "Tool":
self.driver.execute_query(
"MERGE (n:Tool {id: $nid, name: $name, section: $section, sectionID: $sectionID, subsection: $subsection, tags: $tags, version: $version, description: $description, doctype: $doctype, links: $links, metadata: $metadata, hyperlink: $hyperlink, tooltype: $tooltype})",
name=dbnode.name,
doctype=dbnode.ntype,
nid=dbnode.id,
description=dbnode.description,
links=[], # dbnode.links,
tags=dbnode.tags,
metadata="{}", # dbnode.metadata,
hyperlink="", # dbnode.hyperlink or "",
version=dbnode.version or "",
section=dbnode.section,
sectionID=dbnode.section_id, # dbnode.sectionID,
subsection=dbnode.subsection or "",
tooltype="", # dbnode.tooltype,
database_="neo4j",
)
return
if dbnode.ntype == "Code":
self.driver.execute_query(
"MERGE (n:Code {id: $nid, name: $name, section: $section, sectionID: $sectionID, subsection: $subsection, tags: $tags, version: $version, description: $description, doctype: $doctype, links: $links, metadata: $metadata, hyperlink: $hyperlink})",
name=dbnode.name,
doctype=dbnode.ntype,
nid=dbnode.id,
description=dbnode.description,
links=[], # dbnode.links,
tags=dbnode.tags,
metadata="{}", # dbnode.metadata,
hyperlink="", # dbnode.hyperlink or "",
version=dbnode.version or "",
)
return
raise Exception(f"Unknown DB type: {dbnode.ntype}")

@classmethod
def link_CRE_to_CRE(self, id1, id2, link_type):
Expand All @@ -272,7 +313,7 @@ def link_CRE_to_Node(self, CRE_id, node_id, link_type):
if not self.connected:
return
self.driver.execute_query(
"MATCH (a:CRE), (b:Node) "
"MATCH (a:CRE), (b:Standard|Tool) "
"WHERE a.id = $aID AND b.id = $bID "
"CALL apoc.create.relationship(a,$relType, {},b) "
"YIELD rel "
Expand All @@ -289,7 +330,7 @@ def gap_analysis(self, name_1, name_2):
return None, None
base_standard, _, _ = self.driver.execute_query(
"""
MATCH (BaseStandard:Node {name: $name1})
MATCH (BaseStandard:Standard|Tool {name: $name1})
RETURN BaseStandard
""",
name1=name_1,
Expand All @@ -298,8 +339,8 @@ def gap_analysis(self, name_1, name_2):

path_records_all, _, _ = self.driver.execute_query(
"""
OPTIONAL MATCH (BaseStandard:Node {name: $name1})
OPTIONAL MATCH (CompareStandard:Node {name: $name2})
OPTIONAL MATCH (BaseStandard:Standard|Tool {name: $name1})
OPTIONAL MATCH (CompareStandard:Standard|Tool {name: $name2})
OPTIONAL MATCH p = shortestPath((BaseStandard)-[*..20]-(CompareStandard))
WITH p
WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n.name = $name1 or n.name = $name2)
Expand All @@ -311,8 +352,8 @@ def gap_analysis(self, name_1, name_2):
)
path_records, _, _ = self.driver.execute_query(
"""
OPTIONAL MATCH (BaseStandard:Node {name: $name1})
OPTIONAL MATCH (CompareStandard:Node {name: $name2})
OPTIONAL MATCH (BaseStandard:Standard|Tool {name: $name1})
OPTIONAL MATCH (CompareStandard:Standard|Tool {name: $name2})
OPTIONAL MATCH p = shortestPath((BaseStandard)-[:(LINKED_TO|CONTAINS)*..20]-(CompareStandard))
WITH p
WHERE length(p) > 1 AND ALL(n in NODES(p) WHERE n:CRE or n.name = $name1 or n.name = $name2)
Expand All @@ -325,70 +366,94 @@ def gap_analysis(self, name_1, name_2):

def format_segment(seg):
return {
"start": {
"name": seg.start_node["name"],
"sectionID": seg.start_node["section_id"],
"section": seg.start_node["section"],
"subsection": seg.start_node["subsection"],
"description": seg.start_node["description"],
"id": seg.start_node["id"],
},
"end": {
"name": seg.end_node["name"],
"sectionID": seg.end_node["section_id"],
"section": seg.end_node["section"],
"subsection": seg.end_node["subsection"],
"description": seg.end_node["description"],
"id": seg.end_node["id"],
},
"start": NEO_DB.parse_node(seg.start_node),
"end": NEO_DB.parse_node(seg.end_node),
"relationship": seg.type,
}

def format_path_record(rec):
return {
"start": {
"name": rec.start_node["name"],
"sectionID": rec.start_node["section_id"],
"section": rec.start_node["section"],
"subsection": rec.start_node["subsection"],
"description": rec.start_node["description"],
"id": rec.start_node["id"],
},
"end": {
"name": rec.end_node["name"],
"sectionID": rec.end_node["section_id"],
"section": rec.end_node["section"],
"subsection": rec.end_node["subsection"],
"description": rec.end_node["description"],
"id": rec.end_node["id"],
},
"start": NEO_DB.parse_node(rec.start_node),
"end": NEO_DB.parse_node(rec.end_node),
"path": [format_segment(seg) for seg in rec.relationships],
}

def format_record(rec):
return {
"name": rec["name"],
"sectionID": rec["section_id"],
"section": rec["section"],
"subsection": rec["subsection"],
"description": rec["description"],
"id": rec["id"],
}

return [format_record(rec["BaseStandard"]) for rec in base_standard], [
return [NEO_DB.parse_node(rec["BaseStandard"]) for rec in base_standard], [
format_path_record(rec["p"]) for rec in (path_records + path_records_all)
]

@classmethod
def standards(self):
def standards(self) -> List[str]:
if not self.connected:
return
records, _, _ = self.driver.execute_query(
'MATCH (n:Node {ntype: "Standard"}) ' "RETURN collect(distinct n.name)",
"MATCH (n:Standard|Tool) " "RETURN collect(distinct n.name)",
database_="neo4j",
)
return records[0][0]

@staticmethod
def parse_node(node: neo4j.graph.Node) -> cre_defs.Document:
name = node["name"]
id = node["id"] if "id" in node else None
description = node["description"] if "description" in node else None
# links = [self.parse_link(link) for link in node["links"]]
tags = node["tags"]
# metadata = node["metadata"]
if cre_defs.Credoctypes.Code.value in node.labels:
return cre_defs.Code(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
# hyperlink=(node["hyperlink"] if "hyperlink" in node else None),
version=(node["version"] if "version" in node else None),
)
if cre_defs.Credoctypes.Standard.value in node.labels:
return cre_defs.Standard(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
# hyperlink=(node["hyperlink"] if "hyperlink" in node else None),
version=(node["version"] if "version" in node else None),
section=node["section"],
sectionID=node["sectionID"],
subsection=(node["subsection"] if "subsection" in node else None),
)
if cre_defs.Credoctypes.Tool.value in node.labels:
return cre_defs.Tool(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
# hyperlink=(node["hyperlink"] if "hyperlink" in node else None),
version=(node["version"] if "version" in node else None),
section=node["section"],
sectionID=node["sectionID"],
subsection=(node["subsection"] if "subsection" in node else None),
)
if cre_defs.Credoctypes.CRE.value in node.labels:
return cre_defs.CRE(
name=name,
id=id,
description=description,
# links=links,
tags=tags,
# metadata=metadata,
)
raise Exception(f"Unknown node {node.labels}")

# @classmethod
# def parse_link(self, link):
# return cre_defs.Link(ltype=link["ltype"], tags=link["tags"])


class CRE_Graph:
graph: nx.Graph = None
Expand Down Expand Up @@ -1298,20 +1363,18 @@ def find_path_between_nodes(
return res

def gap_analysis(self, node_names: List[str]):
if not self.neo_db.connected:
return None
base_standard, paths = self.neo_db.gap_analysis(node_names[0], node_names[1])
if base_standard is None:
return None
grouped_paths = {}
for node in base_standard:
key = node["id"]
key = node.id
if key not in grouped_paths:
grouped_paths[key] = {"start": node, "paths": {}}

for path in paths:
key = path["start"]["id"]
end_key = path["end"]["id"]
key = path["start"].id
end_key = path["end"].id
path["score"] = get_path_score(path)
del path["start"]
if end_key in grouped_paths[key]["paths"]:
Expand All @@ -1321,7 +1384,7 @@ def gap_analysis(self, node_names: List[str]):
grouped_paths[key]["paths"][end_key] = path
return grouped_paths

def standards(self):
def standards(self) -> List[str]:
return self.neo_db.standards()

def text_search(self, text: str) -> List[Optional[cre_defs.Document]]:
Expand Down
Loading

0 comments on commit e224c9f

Please sign in to comment.