Skip to content

Commit

Permalink
Merge pull request #30 from bluescarni/pr/py_exc
Browse files Browse the repository at this point in the history
Attempt re-instating the special exception handling mechanism
  • Loading branch information
bluescarni authored Mar 4, 2020
2 parents 79c3a0b + a11487c commit 86b80b2
Show file tree
Hide file tree
Showing 13 changed files with 253 additions and 106 deletions.
12 changes: 6 additions & 6 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ jobs:
cmake --build . --config RelWithDebInfo --target install
cd ..
cd ..
python -c "import pygmo; pygmo.test.run_test_suite(0); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
python -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
displayName: "Configure, build and test"
- job: 'vs2015_clang_release_latest_36'
Expand Down Expand Up @@ -163,7 +163,7 @@ jobs:
cmake --build . --target install
cd ..
cd ..
python -c "import pygmo; pygmo.test.run_test_suite(0); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
python -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
displayName: "Configure, build and test"
- job: 'vs2017_release_latest_37'
Expand Down Expand Up @@ -234,7 +234,7 @@ jobs:
cmake --build . --config RelWithDebInfo --target install
cd ..
cd ..
python -c "import pygmo; pygmo.test.run_test_suite(0); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
python -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
displayName: "Configure, build and test"
- job: 'vs2015_clang_release_latest_37'
Expand Down Expand Up @@ -326,7 +326,7 @@ jobs:
cmake --build . --target install
cd ..
cd ..
python -c "import pygmo; pygmo.test.run_test_suite(0); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
python -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
displayName: "Configure, build and test"
- job: 'vs2017_release_latest_38'
Expand Down Expand Up @@ -397,7 +397,7 @@ jobs:
cmake --build . --config RelWithDebInfo --target install
cd ..
cd ..
python -c "import pygmo; pygmo.test.run_test_suite(0); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
python -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
displayName: "Configure, build and test"
- job: 'vs2015_clang_release_latest_38'
Expand Down Expand Up @@ -489,5 +489,5 @@ jobs:
cmake --build . --target install
cd ..
cd ..
python -c "import pygmo; pygmo.test.run_test_suite(0); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
python -c "import pygmo; pygmo.test.run_test_suite(1); pygmo.mp_island.shutdown_pool(); pygmo.mp_bfe.shutdown_pool()"
displayName: "Configure, build and test"
1 change: 1 addition & 0 deletions pygmo/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ YACMA_PYTHON_MODULE(core
expose_s_policies.cpp
topology.cpp
expose_topologies.cpp
handle_thread_py_exception.cpp
)

target_link_libraries(core PRIVATE Pagmo::pagmo Boost::boost Boost::serialization)
Expand Down
8 changes: 4 additions & 4 deletions pygmo/_island_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ def run_evolve(self, algo, pop):
isl = island(prob=rosenbrock(), udi=_udi_03(),
size=11, algo=de(), seed=15)
isl.evolve()
with self.assertRaises(ValueError) as cm:
with self.assertRaises(RuntimeError) as cm:
isl.wait_check()
err = cm.exception
self.assertTrue(
Expand Down Expand Up @@ -468,7 +468,7 @@ def run_basic_tests(self):
mp_island.init_pool()
mp_island.resize_pool(6)
isl.evolve(20)
isl.evolve(20)
isl.wait_check()
mp_island.resize_pool(4)
isl.wait_check()
isl.evolve(20)
Expand Down Expand Up @@ -584,7 +584,7 @@ def run_basic_tests(self):
isl.evolve()
isl.wait()
self.assertTrue("**error occurred**" in repr(isl))
self.assertRaises(ValueError, lambda: isl.wait_check())
self.assertRaises(RuntimeError, lambda: isl.wait_check())


class ipyparallel_island_test_case(_ut.TestCase):
Expand Down Expand Up @@ -687,5 +687,5 @@ def run_basic_tests(self):
isl.evolve()
isl.wait()
self.assertTrue("**error occurred**" in repr(isl))
self.assertRaises(ipyparallel.error.RemoteError,
self.assertRaises(RuntimeError,
lambda: isl.wait_check())
6 changes: 3 additions & 3 deletions pygmo/_r_policy_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,10 @@ class r(object):
def replace(self, inds, nx, nix, nobj, nec, nic, tol, mig):
return []
r_pol = r_policy(r())
self.assertRaises(ValueError, lambda: r_pol.replace(inds=([1, 2], [[.1, .2], [.3, .4]], [
self.assertRaises(RuntimeError, lambda: r_pol.replace(inds=([1, 2], [[.1, .2], [.3, .4]], [
[1.1], [2.2]]), nx=2, nix=0, nobj=1, nec=0, nic=0, tol=[], mig=([], np.zeros((0, 2)), np.zeros((0, 2)))))
# Try also flipping around the named argument.
self.assertRaises(ValueError, lambda: r_pol.replace(nx=2, inds=([1, 2], [[.1, .2], [.3, .4]], [
self.assertRaises(RuntimeError, lambda: r_pol.replace(nx=2, inds=([1, 2], [[.1, .2], [.3, .4]], [
[1.1], [2.2]]), nec=0, nix=0, nobj=1, nic=0, mig=([], np.zeros((0, 2)), np.zeros((0, 2))), tol=[]))

class r(object):
Expand Down Expand Up @@ -143,7 +143,7 @@ class r(object):
def replace(self, inds, nx, nix, nobj, nec, nic, tol, mig):
return 1
r_pol = r_policy(r())
self.assertRaises(TypeError, lambda: r_pol.replace(([1, 2], [[.1, .2], [.3, .4]], [
self.assertRaises(RuntimeError, lambda: r_pol.replace(([1, 2], [[.1, .2], [.3, .4]], [
[1.1], [2.2]]), 2, 0, 1, 0, 0, [], ([], np.zeros((0, 2)), np.zeros((0, 2)))))

class r(object):
Expand Down
6 changes: 3 additions & 3 deletions pygmo/_s_policy_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,10 +108,10 @@ class r(object):
def select(self, inds, nx, nix, nobj, nec, nic, tol):
return []
s_pol = s_policy(r())
self.assertRaises(ValueError, lambda: s_pol.select(inds=([1, 2], [[.1, .2], [.3, .4]], [
self.assertRaises(RuntimeError, lambda: s_pol.select(inds=([1, 2], [[.1, .2], [.3, .4]], [
[1.1], [2.2]]), nx=2, nix=0, nobj=1, nec=0, nic=0, tol=[]))
# Try also flipping around the named argument.
self.assertRaises(ValueError, lambda: s_pol.select(nx=2, inds=([1, 2], [[.1, .2], [.3, .4]], [
self.assertRaises(RuntimeError, lambda: s_pol.select(nx=2, inds=([1, 2], [[.1, .2], [.3, .4]], [
[1.1], [2.2]]), nec=0, nix=0, nobj=1, nic=0, tol=[]))

class r(object):
Expand Down Expand Up @@ -143,7 +143,7 @@ class r(object):
def select(self, inds, nx, nix, nobj, nec, nic, tol):
return 1
s_pol = s_policy(r())
self.assertRaises(TypeError, lambda: s_pol.select(([1, 2], [[.1, .2], [.3, .4]], [
self.assertRaises(RuntimeError, lambda: s_pol.select(([1, 2], [[.1, .2], [.3, .4]], [
[1.1], [2.2]]), 2, 0, 1, 0, 0, []))

class r(object):
Expand Down
2 changes: 1 addition & 1 deletion pygmo/_topology_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ def push_back(self):
def get_connections(self, n):
return []
topo = topology(t())
self.assertRaises(ValueError, lambda: topo.get_connections(0))
self.assertRaises(RuntimeError, lambda: topo.get_connections(0))

class t(object):

Expand Down
63 changes: 63 additions & 0 deletions pygmo/handle_thread_py_exception.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright 2020 PaGMO development team
//
// This file is part of the pygmo library.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <cassert>
#include <exception>
#include <stdexcept>
#include <string>

#include <pybind11/pybind11.h>

#include "handle_thread_py_exception.hpp"

namespace pygmo
{

namespace py = pybind11;

// Helper to handle Python exceptions thrown in a separate thread of execution not managed
// by Python.
void handle_thread_py_exception(const std::string &err, const py::error_already_set &eas)
{
// NOTE: this function must be called only in a catch block.
assert(std::current_exception());

// Make sure to clean up the Python error indicator
// for the current thread.
::PyErr_Clear();

// NOTE: this helper is needed because sometimes one of the attributes
// of eas is a default-constructed py::object(), which causes an error
// when passed to format_exception() below. In such a case, this helper
// will return None instead. See:
// https://github.com/pybind/pybind11/issues/1543
auto obj_or_none = [](const py::object &o) { return o ? o : py::none(); };

// Try to extract a string description of the exception using the "traceback" module.
std::string tmp(err);
try {
// NOTE: we are about to go back into the Python interpreter. Here Python could throw an exception
// and set again the error indicator. In case of any issue,
// we will give up any attempt of producing a meaningful error message, reset the error indicator,
// and throw a pure C++ exception with a generic error message.
tmp += py::cast<std::string>(
py::str("").attr("join")(py::module::import("traceback")
.attr("format_exception")(obj_or_none(eas.type()), obj_or_none(eas.value()),
obj_or_none(eas.trace()))));
} catch (const py::error_already_set &) {
// The block above threw from Python. There's not much we can do.
::PyErr_Clear();
throw std::runtime_error("While trying to analyze the error message of a Python exception raised in a "
"separate thread, another Python exception was raised. Giving up now.");
}

// Throw the C++ exception.
throw std::runtime_error(tmp);
}

} // namespace pygmo
25 changes: 25 additions & 0 deletions pygmo/handle_thread_py_exception.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright 2020 PaGMO development team
//
// This file is part of the pygmo library.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#ifndef PYGMO_HANDLE_THREAD_PY_EXCEPTION_HPP
#define PYGMO_HANDLE_THREAD_PY_EXCEPTION_HPP

#include <string>

#include <pybind11/pybind11.h>

namespace pygmo
{

namespace py = pybind11;

[[noreturn]] void handle_thread_py_exception(const std::string &, const py::error_already_set &);

} // namespace pygmo

#endif
92 changes: 53 additions & 39 deletions pygmo/island.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <pagmo/s11n.hpp>

#include "common_utils.hpp"
#include "handle_thread_py_exception.hpp"
#include "island.hpp"
#include "object_serialization.hpp"

Expand Down Expand Up @@ -58,50 +59,63 @@ void isl_inner<py::object>::run_evolve(island &isl) const
// doing anything with the interpreter (including the throws in the checks below).
pygmo::gil_thread_ensurer gte;

auto isl_name = get_name();

auto ret = m_value.attr("run_evolve")(isl.get_algorithm(), isl.get_population());

py::tuple ret_tup;
try {
ret_tup = py::cast<py::tuple>(ret);
} catch (const py::cast_error &) {
pygmo::py_throw(PyExc_TypeError, ("the 'run_evolve()' method of a user-defined island "
"must return a tuple, but it returned an object of type '"
+ pygmo::str(pygmo::type(ret)) + "' instead")
.c_str());
}
if (py::len(ret_tup) != 2) {
pygmo::py_throw(PyExc_ValueError, ("the tuple returned by the 'run_evolve()' method of a user-defined island "
"must have 2 elements, but instead it has "
+ std::to_string(py::len(ret_tup)) + " element(s)")
.c_str());
}

algorithm ret_algo;
// NOTE: every time we call into the Python interpreter from a separate thread, we need to
// handle Python exceptions in a special way.
std::string isl_name;
try {
ret_algo = py::cast<algorithm>(ret_tup[0]);
} catch (const py::cast_error &) {
pygmo::py_throw(PyExc_TypeError,
("the first value returned by the 'run_evolve()' method of a user-defined island "
"must be an algorithm, but an object of type '"
+ pygmo::str(pygmo::type(ret_tup[0])) + "' was returned instead")
.c_str());
isl_name = get_name();
} catch (const py::error_already_set &eas) {
pygmo::handle_thread_py_exception("Could not fetch the name of a pythonic island. The error is:\n", eas);
}

population ret_pop;
try {
ret_pop = py::cast<population>(ret_tup[1]);
} catch (const py::cast_error &) {
pygmo::py_throw(PyExc_TypeError,
("the second value returned by the 'run_evolve()' method of a user-defined island "
"must be a population, but an object of type '"
+ pygmo::str(pygmo::type(ret_tup[1])) + "' was returned instead")
.c_str());
auto ret = m_value.attr("run_evolve")(isl.get_algorithm(), isl.get_population());

py::tuple ret_tup;
try {
ret_tup = py::cast<py::tuple>(ret);
} catch (const py::cast_error &) {
pygmo::py_throw(PyExc_TypeError, ("the 'run_evolve()' method of a user-defined island "
"must return a tuple, but it returned an object of type '"
+ pygmo::str(pygmo::type(ret)) + "' instead")
.c_str());
}
if (py::len(ret_tup) != 2) {
pygmo::py_throw(PyExc_ValueError,
("the tuple returned by the 'run_evolve()' method of a user-defined island "
"must have 2 elements, but instead it has "
+ std::to_string(py::len(ret_tup)) + " element(s)")
.c_str());
}

algorithm ret_algo;
try {
ret_algo = py::cast<algorithm>(ret_tup[0]);
} catch (const py::cast_error &) {
pygmo::py_throw(PyExc_TypeError,
("the first value returned by the 'run_evolve()' method of a user-defined island "
"must be an algorithm, but an object of type '"
+ pygmo::str(pygmo::type(ret_tup[0])) + "' was returned instead")
.c_str());
}

population ret_pop;
try {
ret_pop = py::cast<population>(ret_tup[1]);
} catch (const py::cast_error &) {
pygmo::py_throw(PyExc_TypeError,
("the second value returned by the 'run_evolve()' method of a user-defined island "
"must be a population, but an object of type '"
+ pygmo::str(pygmo::type(ret_tup[1])) + "' was returned instead")
.c_str());
}

isl.set_algorithm(ret_algo);
isl.set_population(ret_pop);
} catch (const py::error_already_set &eas) {
pygmo::handle_thread_py_exception(
"The asynchronous evolution of a pythonic island of type '" + isl_name + "' raised an error:\n", eas);
}

isl.set_algorithm(ret_algo);
isl.set_population(ret_pop);
}

std::string isl_inner<py::object>::get_name() const
Expand Down
29 changes: 22 additions & 7 deletions pygmo/r_policy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <pagmo/types.hpp>

#include "common_utils.hpp"
#include "handle_thread_py_exception.hpp"
#include "object_serialization.hpp"
#include "r_policy.hpp"

Expand Down Expand Up @@ -71,14 +72,28 @@ r_pol_inner<py::object>::replace(const individuals_group_t &inds, const vector_d
// doing anything with the interpreter (including the throws in the checks below).
pygmo::gil_thread_ensurer gte;

auto r_pol_name = get_name();

// Fetch the new individuals in Python form.
auto o = m_value.attr("replace")(pygmo::inds_to_tuple(inds), nx, nix, nobj, nec, nic,
pygmo::vector_to_ndarr<py::array_t<double>>(tol), pygmo::inds_to_tuple(mig));
// NOTE: every time we call into the Python interpreter from a separate thread, we need to
// handle Python exceptions in a special way.
std::string r_pol_name;
try {
r_pol_name = get_name();
} catch (const py::error_already_set &eas) {
pygmo::handle_thread_py_exception("Could not fetch the name of a pythonic replacement policy. The error is:\n",
eas);
}

// Convert back to C++ form and return.
return pygmo::iterable_to_inds(o);
try {
// Fetch the new individuals in Python form.
auto o = m_value.attr("replace")(pygmo::inds_to_tuple(inds), nx, nix, nobj, nec, nic,
pygmo::vector_to_ndarr<py::array_t<double>>(tol), pygmo::inds_to_tuple(mig));

// Convert back to C++ form and return.
return pygmo::iterable_to_inds(o);
} catch (const py::error_already_set &eas) {
pygmo::handle_thread_py_exception("The replace() method of a pythonic replacement policy of type '" + r_pol_name
+ "' raised an error:\n",
eas);
}
}

std::string r_pol_inner<py::object>::get_name() const
Expand Down
Loading

0 comments on commit 86b80b2

Please sign in to comment.