Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add delete subgraph functionality #1053

Merged
merged 3 commits into from
Sep 28, 2023
Merged

add delete subgraph functionality #1053

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0ebd24f
readers: remove subgraph feature
zekemorton Jul 20, 2023
018d042
resource query: remove subgraph feature
zekemorton Jul 20, 2023
e9c4bfc
tests: add tests for remove subgraph
zekemorton Aug 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions resource/readers/resource_reader_base.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ class resource_reader_base_t {
virtual int unpack_at (resource_graph_t &g, resource_graph_metadata_t &m,
zekemorton marked this conversation as resolved.
Show resolved Hide resolved
vtx_t &vtx, const std::string &str, int rank = -1) = 0;

virtual int remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path) = 0;

/*! Update resource graph g with str.
*
* \param g resource graph
Expand Down
8 changes: 8 additions & 0 deletions resource/readers/resource_reader_grug.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,14 @@ int resource_reader_grug_t::unpack_at (resource_graph_t &g,
return -1;
}

int resource_reader_grug_t::remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path)
{
errno = ENOTSUP; // GRUG reader does not support remove
return -1;
}

int resource_reader_grug_t::update (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &str, int64_t jobid,
Expand Down
4 changes: 4 additions & 0 deletions resource/readers/resource_reader_grug.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ class resource_reader_grug_t : public resource_reader_base_t {
virtual int unpack_at (resource_graph_t &g, resource_graph_metadata_t &m,
vtx_t &vtx, const std::string &str, int rank = -1);

virtual int remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path);

/*! Update resource graph g with str.
*
* \param g resource graph
Expand Down
9 changes: 8 additions & 1 deletion resource/readers/resource_reader_hwloc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ int resource_reader_hwloc_t::walk_hwloc (resource_graph_t &g,
rc = -1;
break;
}
if (remap_id
if (remap_id
> static_cast<uint64_t> (std::numeric_limits<int>::max ())) {
errno = EOVERFLOW;
m_err_msg += "Remapped gpu id too large; ";
Expand Down Expand Up @@ -508,6 +508,13 @@ int resource_reader_hwloc_t::unpack_at (resource_graph_t &g,
return unpack_internal (g, m, vtx, str, rank);
}

int resource_reader_hwloc_t::remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path)
{
errno = ENOTSUP; // hwloc reader does not support remove
return -1;
}
int resource_reader_hwloc_t::update (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &str, int64_t jobid,
Expand Down
4 changes: 4 additions & 0 deletions resource/readers/resource_reader_hwloc.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ class resource_reader_hwloc_t : public resource_reader_base_t {
virtual int unpack_at (resource_graph_t &g, resource_graph_metadata_t &m,
vtx_t &vtx, const std::string &str, int rank = -1);

virtual int remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path);

/*! Update resource graph g with str.
*
* \param g resource graph
Expand Down
179 changes: 163 additions & 16 deletions resource/readers/resource_reader_jgf.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ int resource_reader_jgf_t::unpack_and_remap_vtx (fetch_helper_t &f,
m_err_msg += std::to_string (f.rank) + ".\n";
goto error;
}
if (remap_rank
if (remap_rank
> static_cast<uint64_t> (std::numeric_limits<int64_t>::max ())) {
errno = EOVERFLOW;
m_err_msg += __FUNCTION__;
Expand All @@ -298,7 +298,7 @@ int resource_reader_jgf_t::unpack_and_remap_vtx (fetch_helper_t &f,
m_err_msg += " rank=" + std::to_string (f.rank) + ".\n";
goto error;
}
if (remap_id
if (remap_id
> static_cast<uint64_t> (std::numeric_limits<int>::max ())) {
errno = EOVERFLOW;
m_err_msg += __FUNCTION__;
Expand Down Expand Up @@ -457,7 +457,7 @@ vtx_t resource_reader_jgf_t::create_vtx (resource_graph_t &g,

vtx_t resource_reader_jgf_t::vtx_in_graph (const resource_graph_t &g,
const resource_graph_metadata_t &m,
const std::map<std::string,
const std::map<std::string,
std::string> &paths,
int rank)
{
Expand Down Expand Up @@ -528,17 +528,75 @@ int resource_reader_jgf_t::add_graph_metadata (vtx_t v,
return rc;
}

int resource_reader_jgf_t::update_vmap (std::map<std::string,
int resource_reader_jgf_t::remove_graph_metadata (vtx_t v,
resource_graph_t &g,
resource_graph_metadata_t &m)
{
int rc = -1;
for (auto kv : g[v].paths) {
m.by_path.erase (kv.second);
}

m.by_outedges.erase (v);

for (auto it = m.by_type[g[v].type].begin (); it != m.by_type[g[v].type].end (); ++it) {
if (*it == v) {
m.by_type[g[v].type].erase (it);
break;
}
}

for (auto it = m.by_name[g[v].name].begin (); it != m.by_name[g[v].name].end (); ++it) {
if (*it == v) {
m.by_name[g[v].name].erase (it);
break;
}
}

for (auto it = m.by_rank[g[v].rank].begin (); it != m.by_rank[g[v].rank].end (); ++it) {
if (*it == v) {
m.by_rank[g[v].rank].erase (it);
break;
}
}

rc = 0;
return rc;
}

int resource_reader_jgf_t::remove_metadata_outedges (vtx_t source_vertex,
vtx_t dest_vertex,
resource_graph_t &g,
resource_graph_metadata_t &m)
{
int rc = -1;
std::vector<edg_t> remove_edges;
auto iter = m.by_outedges.find (source_vertex);
if (iter == m.by_outedges.end ())
return rc;
auto &outedges = iter->second;
for (auto kv = outedges.begin (); kv != outedges.end (); ++kv) {
if (boost::target (kv->second, g) == dest_vertex) {
kv = outedges.erase (kv);
// TODO: Consider adding break here
}
}

rc = 0;
return rc;
}

int resource_reader_jgf_t::update_vmap (std::map<std::string,
vmap_val_t> &vmap,
vtx_t v,
const std::map<std::string,
vtx_t v,
const std::map<std::string,
bool> &root_checks,
const fetch_helper_t &fetcher)
{
int rc = -1;
std::pair<std::map<std::string, vmap_val_t>::iterator, bool> ptr;
ptr = vmap.emplace (std::string (fetcher.vertex_id),
vmap_val_t{v, root_checks,
ptr = vmap.emplace (std::string (fetcher.vertex_id),
vmap_val_t{v, root_checks,
static_cast<unsigned int> (fetcher.size),
static_cast<unsigned int> (fetcher.exclusive)});
if (!ptr.second) {
Expand Down Expand Up @@ -776,7 +834,7 @@ int resource_reader_jgf_t::unpack_vertices (resource_graph_t &g,
resource_graph_metadata_t &m,
std::map<std::string,
vmap_val_t> &vmap,
json_t *nodes,
json_t *nodes,
std::unordered_set<std::string>
&added_vtcs)
{
Expand Down Expand Up @@ -894,7 +952,7 @@ int resource_reader_jgf_t::unpack_edges (resource_graph_t &g,
resource_graph_metadata_t &m,
std::map<std::string,
vmap_val_t> &vmap,
json_t *edges,
json_t *edges,
const std::unordered_set
<std::string> &added_vtcs)
{
Expand All @@ -914,7 +972,7 @@ int resource_reader_jgf_t::unpack_edges (resource_graph_t &g,
if ( (unpack_edge (element, vmap, source, target, &name)) != 0)
goto done;
// We only add the edge when it connects at least one newly added vertex
if ( (added_vtcs.count (source) == 1)
if ( (added_vtcs.count (source) == 1)
|| (added_vtcs.count (target) == 1)) {
tie (e, inserted) = add_edge (vmap[source].v, vmap[target].v, g);
if (inserted == false) {
Expand Down Expand Up @@ -1053,6 +1111,57 @@ int resource_reader_jgf_t::update_edges (resource_graph_t &g,
return rc;
}

int resource_reader_jgf_t::get_subgraph_vertices (resource_graph_t &g,
vtx_t vtx,
std::vector<vtx_t> &vtx_list)
{
vtx_t next_vtx;
boost::graph_traits<resource_graph_t>::out_edge_iterator ei, ei_end;
boost::tie (ei, ei_end) = boost::out_edges (vtx, g);

for (; ei != ei_end; ++ei) {
next_vtx = boost::target (*ei, g);

for (auto const &paths_it : g[next_vtx].paths) {
// check that we don't recurse on parent edges
if (paths_it.second.find (g[vtx].name) != std::string::npos &&
paths_it.second.find (g[vtx].name) < paths_it.second.find (g[next_vtx].name)) {
vtx_list.push_back (next_vtx);
get_subgraph_vertices (g, next_vtx, vtx_list);
break;
}
}
}

return 0;
}

int resource_reader_jgf_t::get_parent_vtx (resource_graph_t &g,
vtx_t vtx,
vtx_t &parent_vtx)

{
vtx_t next_vtx;
boost::graph_traits<resource_graph_t>::out_edge_iterator ei, ei_end;
boost::tie (ei, ei_end) = boost::out_edges (vtx, g);
int rc = -1;

for (; ei != ei_end; ++ei) {
next_vtx = boost::target (*ei, g);
for (auto const &paths_it : g[vtx].paths) {
// check that the parent's name exists in the child's path before the child's name
if (paths_it.second.find (g[next_vtx].name) != std::string::npos &&
paths_it.second.find (g[vtx].name) > paths_it.second.find (g[next_vtx].name)) {
parent_vtx = next_vtx;
rc = 0;
break;
}
}
}

return rc;
}


/********************************************************************************
* *
Expand Down Expand Up @@ -1099,12 +1208,12 @@ int resource_reader_jgf_t::unpack_at (resource_graph_t &g,
resource_graph_metadata_t &m, vtx_t &vtx,
const std::string &str, int rank)
{
/* This functionality is currently experimental, as resource graph
* growth causes a resize of the boost vecS vertex container type.
* Resizing the vecS results in lost job allocations and reservations
/* This functionality is currently experimental, as resource graph
* growth causes a resize of the boost vecS vertex container type.
* Resizing the vecS results in lost job allocations and reservations
* as there is no copy constructor for planner.
* vtx_t vtx is not implemented and may be used in the future
* for optimization.
* vtx_t vtx is not implemented and may be used in the future
* for optimization.
*/

return unpack (g, m, str, rank);
Expand Down Expand Up @@ -1144,6 +1253,44 @@ int resource_reader_jgf_t::update (resource_graph_t &g,
return rc;
}

int resource_reader_jgf_t::remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path)
{
vtx_t subgraph_root_vtx = boost::graph_traits<resource_graph_t>::null_vertex ();
vtx_t parent_vtx = boost::graph_traits<resource_graph_t>::null_vertex ();
std::vector<vtx_t> vtx_list;

auto iter = m.by_path.find (path);
if (iter == m.by_path.end ()) {
return -1;
}

for (auto &v : iter->second) {
subgraph_root_vtx = v;
}

vtx_list.push_back (subgraph_root_vtx);

get_subgraph_vertices (g, subgraph_root_vtx, vtx_list);

if ( get_parent_vtx (g, subgraph_root_vtx, parent_vtx) )
return -1;

if ( remove_metadata_outedges (parent_vtx, subgraph_root_vtx, g, m) )
return -1;

for (auto & vtx : vtx_list)
{
// clear vertex edges but don't delete vertex
boost::clear_vertex (vtx, g);
remove_graph_metadata (vtx, g, m);
}

return 0;

}

bool resource_reader_jgf_t::is_allowlist_supported ()
{
return false;
Expand Down
30 changes: 23 additions & 7 deletions resource/readers/resource_reader_jgf.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ class resource_reader_jgf_t : public resource_reader_base_t {
const std::string &str, int64_t jobid, int64_t at,
uint64_t dur, bool rsv, uint64_t trav_token);

virtual int remove_subgraph (resource_graph_t &g,
resource_graph_metadata_t &m,
const std::string &path);

/*! Is the selected reader format support allowlist
*
* \return false
Expand All @@ -100,7 +104,13 @@ class resource_reader_jgf_t : public resource_reader_base_t {
std::map<std::string, bool> &is_roots);
int add_graph_metadata (vtx_t v, resource_graph_t &g,
resource_graph_metadata_t &m);
int update_vmap (std::map<std::string, vmap_val_t> &vmap, vtx_t v,
int remove_graph_metadata (vtx_t v, resource_graph_t &g,
resource_graph_metadata_t &m);
int remove_metadata_outedges (vtx_t source_vertex,
vtx_t dest_vertex,
resource_graph_t &g,
resource_graph_metadata_t &m);
int update_vmap (std::map<std::string, vmap_val_t> &vmap, vtx_t v,
const std::map<std::string, bool> &root_checks,
const fetch_helper_t &fetcher);
int add_vtx (resource_graph_t &g, resource_graph_metadata_t &m,
Expand All @@ -121,19 +131,19 @@ class resource_reader_jgf_t : public resource_reader_base_t {
const fetch_helper_t &fetcher, uint64_t jobid, int64_t at,
uint64_t dur, bool rsv);
int unpack_vertices (resource_graph_t &g, resource_graph_metadata_t &m,
std::map<std::string, vmap_val_t> &vmap,
std::map<std::string, vmap_val_t> &vmap,
json_t *nodes,
std::unordered_set<std::string> &added_vtcs);
int undo_vertices (resource_graph_t &g,
std::map<std::string, vmap_val_t> &vmap,
uint64_t jobid, bool rsv);
int update_vertices (resource_graph_t &g, resource_graph_metadata_t &m,
std::map<std::string, vmap_val_t> &vmap,
json_t *nodes, int64_t jobid, int64_t at,
std::map<std::string, vmap_val_t> &vmap,
json_t *nodes, int64_t jobid, int64_t at,
uint64_t dur, bool rsv);
int update_vertices (resource_graph_t &g, resource_graph_metadata_t &m,
std::map<std::string, vmap_val_t> &vmap,
json_t *nodes, int64_t jobid, int64_t at,
std::map<std::string, vmap_val_t> &vmap,
json_t *nodes, int64_t jobid, int64_t at,
uint64_t dur);
int unpack_edge (json_t *element, std::map<std::string, vmap_val_t> &vmap,
std::string &source, std::string &target, json_t **name);
Expand All @@ -145,12 +155,18 @@ class resource_reader_jgf_t : public resource_reader_base_t {
std::string &source, std::string &target,
uint64_t token);
int unpack_edges (resource_graph_t &g, resource_graph_metadata_t &m,
std::map<std::string, vmap_val_t> &vmap,
std::map<std::string, vmap_val_t> &vmap,
json_t *edges,
const std::unordered_set<std::string> &added_vtcs);
int update_edges (resource_graph_t &g, resource_graph_metadata_t &m,
std::map<std::string, vmap_val_t> &vmap,
json_t *edges, uint64_t token);
int get_subgraph_vertices (resource_graph_t &g,
vtx_t node,
std::vector<vtx_t> &node_list);
int get_parent_vtx (resource_graph_t &g,
vtx_t node,
vtx_t &parent_node);
};

} // namespace resource_model
Expand Down
Loading
Loading