diff --git a/server/result/tests/test_pk.py b/server/result/tests/test_pk.py new file mode 100644 index 000000000..3083c5a4c --- /dev/null +++ b/server/result/tests/test_pk.py @@ -0,0 +1,328 @@ +from rest_framework import status + +from medperf.tests import MedPerfTest + +from parameterized import parameterized, parameterized_class + + +class ResultsTest(MedPerfTest): + def generic_setup(self): + # setup users + data_owner = "data_owner" + mlcube_owner = "mlcube_owner" + bmk_owner = "bmk_owner" + bmk_prep_mlcube_owner = "bmk_prep_mlcube_owner" + ref_mlcube_owner = "ref_mlcube_owner" + eval_mlcube_owner = "eval_mlcube_owner" + other_user = "other_user" + + self.create_user(data_owner) + self.create_user(mlcube_owner) + self.create_user(bmk_owner) + self.create_user(bmk_prep_mlcube_owner) + self.create_user(ref_mlcube_owner) + self.create_user(eval_mlcube_owner) + self.create_user(other_user) + + # create benchmark + prep, _, _, benchmark = self.shortcut_create_benchmark( + bmk_prep_mlcube_owner, + ref_mlcube_owner, + eval_mlcube_owner, + bmk_owner, + ) + + # create dataset + self.set_credentials(data_owner) + dataset = self.mock_dataset( + data_preparation_mlcube=prep["id"], state="OPERATION" + ) + dataset = self.create_dataset(dataset).data + + # create dataset assoc + assoc = self.mock_dataset_association( + benchmark["id"], dataset["id"], approval_status="APPROVED" + ) + self.create_dataset_association(assoc, data_owner, bmk_owner) + + # create model mlcube + self.set_credentials(mlcube_owner) + mlcube = self.mock_mlcube(state="OPERATION") + mlcube = self.create_mlcube(mlcube).data + + # create mlcube assoc + assoc = self.mock_mlcube_association( + benchmark["id"], mlcube["id"], approval_status="APPROVED" + ) + self.create_mlcube_association(assoc, mlcube_owner, bmk_owner) + + # setup globals + self.data_owner = data_owner + self.mlcube_owner = mlcube_owner + self.bmk_owner = bmk_owner + self.bmk_prep_mlcube_owner = bmk_prep_mlcube_owner + self.ref_mlcube_owner = ref_mlcube_owner + self.eval_mlcube_owner = eval_mlcube_owner + self.other_user = other_user + + self.bmk_id = benchmark["id"] + self.dataset_id = dataset["id"] + self.mlcube_id = mlcube["id"] + + self.url = self.api_prefix + "/results/{0}/" + self.set_credentials(None) + + +@parameterized_class( + [ + {"actor": "data_owner"}, + {"actor": "bmk_owner"}, + ] +) +class ResultGetTest(ResultsTest): + """Test module for GET /results/""" + + def setUp(self): + super(ResultGetTest, self).setUp() + self.generic_setup() + self.set_credentials(self.actor) + + def test_generic_get_result(self): + # Arrange + result = self.mock_result( + self.bmk_id, self.mlcube_id, self.dataset_id, results={"r": 1} + ) + self.set_credentials(self.data_owner) + result = self.create_result(result).data + self.set_credentials(self.actor) + + url = self.url.format(result["id"]) + + # Act + response = self.client.get(url) + + # Assert + self.assertEqual(response.status_code, status.HTTP_200_OK) + for k, v in response.data.items(): + if k in result: + self.assertEqual(result[k], v, f"Unexpected value for {k}") + + def test_result_not_found(self): + # Arrange + invalid_id = 9999 + url = self.url.format(invalid_id) + + # Act + response = self.client.get(url) + + # Assert + # TODO: fixme after refactoring permissions. should be 404 + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + +@parameterized_class( + [ + {"actor": "data_owner"}, + ] +) +class ResultPutTest(ResultsTest): + """Test module for PUT /results/""" + + def setUp(self): + super(ResultPutTest, self).setUp() + self.generic_setup() + self.set_credentials(self.actor) + + def test_put_does_not_modify_readonly_fields(self): + # Arrange + result = self.mock_result( + self.bmk_id, self.mlcube_id, self.dataset_id, results={"r": 1} + ) + self.set_credentials(self.data_owner) + result = self.create_result(result).data + self.set_credentials(self.actor) + + newtestresult = { + "owner": 10, + "approved_at": "some time", + "created_at": "some time", + "modified_at": "some time", + "benchmark": 44, + "model": 444, + "dataset": 55, + "results": {"new": 111}, + "approval_status": "APPROVED", + } + url = self.url.format(result["id"]) + + # Act + response = self.client.put(url, newtestresult, format="json") + + # Assert + self.assertEqual(response.status_code, status.HTTP_200_OK) + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + for k, v in newtestresult.items(): + self.assertNotEqual(v, response.data[k], f"{k} was modified") + + +@parameterized_class( + [ + {"actor": "api_admin"}, + {"actor": "data_owner"}, + ] +) +class ResultDeleteTest(ResultsTest): + """Test module for DELETE /results/""" + + # TODO: for all DELETE tests, we should revisit when we allow users + # to delete. We should test the effects of model.CASCADE and model.PROTECT + + def setUp(self): + super(ResultDeleteTest, self).setUp() + self.generic_setup() + self.set_credentials(self.actor) + + def test_deletion_works_as_expected(self): + # Arrange + result = self.mock_result( + self.bmk_id, self.mlcube_id, self.dataset_id, results={"r": 1} + ) + self.set_credentials(self.data_owner) + result = self.create_result(result).data + self.set_credentials(self.actor) + + url = self.url.format(result["id"]) + + # Act + response = self.client.delete(url) + + # Assert + self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + response = self.client.get(url) + + # TODO: fixme after refactoring permissions. should just like this: + # self.assertEqual(response.status_code, status.HTTP_404_FORBIDDEN) + if self.actor == self.data_owner: + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + else: + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + +class PermissionTest(ResultsTest): + """Test module for permissions of /results/{pk} endpoint + Non-permitted actions: + GET: for all users except bmk_owner, data_owner, and admin + DELETE: for all users except data_owner and admin + PUT: for all users except data_owner and admin + """ + + def setUp(self): + super(PermissionTest, self).setUp() + self.generic_setup() + result = self.mock_result( + self.bmk_id, self.mlcube_id, self.dataset_id, results={"r": 1} + ) + self.set_credentials(self.data_owner) + result = self.create_result(result).data + self.url = self.url.format(result["id"]) + + self.result = result + self.set_credentials(None) + + @parameterized.expand( + [ + ("mlcube_owner", status.HTTP_403_FORBIDDEN), + ("bmk_prep_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("ref_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("eval_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("other_user", status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), + ] + ) + def test_get_permissions(self, user, expected_status): + # Arrange + self.set_credentials(user) + + # Act + response = self.client.get(self.url) + + # Assert + self.assertEqual(response.status_code, expected_status) + + @parameterized.expand( + [ + ("bmk_owner", status.HTTP_403_FORBIDDEN), + ("mlcube_owner", status.HTTP_403_FORBIDDEN), + ("bmk_prep_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("ref_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("eval_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("other_user", status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), + ] + ) + def test_put_permissions(self, user, expected_status): + # Arrange + + # create new assets to edit with + prep, refmodel, _, newbenchmark = self.shortcut_create_benchmark( + self.bmk_prep_mlcube_owner, + self.ref_mlcube_owner, + self.eval_mlcube_owner, + self.bmk_owner, + prep_mlcube_kwargs={"name": "newprep", "mlcube_hash": "newprephash"}, + ref_mlcube_kwargs={"name": "newref", "mlcube_hash": "newrefhash"}, + eval_mlcube_kwargs={"name": "neweval", "mlcube_hash": "newevalhash"}, + name="newbmk", + ) + self.set_credentials(self.data_owner) + newdataset = self.mock_dataset(prep["id"], generated_uid="newgen") + newdataset = self.create_dataset(newdataset).data + + newtestresult = { + "name": "new", + "owner": 55, + "benchmark": newbenchmark["id"], + "model": refmodel["id"], + "dataset": newdataset["id"], + "results": {"new": "t"}, + "metadata": {"new": "t"}, + "approval_status": "APPROVED", + "approved_at": "time", + "created_at": "time", + "modified_at": "time", + } + + self.set_credentials(user) + + for key in newtestresult: + # Act + response = self.client.put( + self.url, {key: newtestresult[key]}, format="json" + ) + + # Assert + self.assertEqual( + response.status_code, expected_status, f"{key} was modified" + ) + + @parameterized.expand( + [ + ("bmk_owner", status.HTTP_403_FORBIDDEN), + ("mlcube_owner", status.HTTP_403_FORBIDDEN), + ("bmk_prep_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("ref_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("eval_mlcube_owner", status.HTTP_403_FORBIDDEN), + ("other_user", status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), + ] + ) + def test_delete_permissions(self, user, expected_status): + # Arrange + self.set_credentials(user) + + # Act + response = self.client.delete(self.url) + + # Assert + self.assertEqual(response.status_code, expected_status) diff --git a/server/utils/tests.py b/server/utils/tests.py new file mode 100644 index 000000000..c1d17b5ca --- /dev/null +++ b/server/utils/tests.py @@ -0,0 +1,311 @@ +from rest_framework import status + +from medperf.tests import MedPerfTest + + +class UserTest(MedPerfTest): + def test_me_returns_current_user(self): + url = self.api_prefix + "/me/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + self.assertEqual(response1.data["username"], user1) + self.assertEqual(response2.data["username"], user2) + + +class BenchmarksTest(MedPerfTest): + def __create_asset(self, user): + _, _, _, benchmark = self.shortcut_create_benchmark( + user, + user, + user, + user, + prep_mlcube_kwargs={"name": f"{user}prep", "mlcube_hash": f"{user}prep"}, + ref_mlcube_kwargs={"name": f"{user}ref", "mlcube_hash": f"{user}ref"}, + eval_mlcube_kwargs={"name": f"{user}eval", "mlcube_hash": f"{user}eval"}, + name=f"{user}name", + ) + return benchmark + + def test_endpoint_returns_current_user_assets(self): + url = self.api_prefix + "/me/benchmarks/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # create an asset for each user + benchmark1 = self.__create_asset(user1) + benchmark2 = self.__create_asset(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + resp1 = response1.data["results"] + resp2 = response2.data["results"] + self.assertEqual(len(resp1), 1) + self.assertEqual(len(resp2), 1) + self.assertEqual(resp1[0]["id"], benchmark1["id"]) + self.assertEqual(resp2[0]["id"], benchmark2["id"]) + + +class DatasetsTest(MedPerfTest): + def __create_asset(self, user): + self.set_credentials(user) + prep = self.mock_mlcube(name=f"{user}name", mlcube_hash=f"{user}hash") + prep = self.create_mlcube(prep).data + dataset = self.mock_dataset(prep["id"], generated_uid=f"{user}genid") + dataset = self.create_dataset(dataset).data + return dataset + + def test_endpoint_returns_current_user_assets(self): + url = self.api_prefix + "/me/datasets/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # create an asset for each user + dataset1 = self.__create_asset(user1) + dataset2 = self.__create_asset(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + resp1 = response1.data["results"] + resp2 = response2.data["results"] + self.assertEqual(len(resp1), 1) + self.assertEqual(len(resp2), 1) + self.assertEqual(resp1[0]["id"], dataset1["id"]) + self.assertEqual(resp2[0]["id"], dataset2["id"]) + + +class MlCubesTest(MedPerfTest): + def __create_asset(self, user): + self.set_credentials(user) + mlcube = self.mock_mlcube(name=f"{user}name", mlcube_hash=f"{user}hash") + mlcube = self.create_mlcube(mlcube).data + return mlcube + + def test_endpoint_returns_current_user_assets(self): + url = self.api_prefix + "/me/mlcubes/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # create an asset for each user + mlcube1 = self.__create_asset(user1) + mlcube2 = self.__create_asset(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + resp1 = response1.data["results"] + resp2 = response2.data["results"] + self.assertEqual(len(resp1), 1) + self.assertEqual(len(resp2), 1) + self.assertEqual(resp1[0]["id"], mlcube1["id"]) + self.assertEqual(resp2[0]["id"], mlcube2["id"]) + + +class ResultsTest(MedPerfTest): + def setUp(self): + super(ResultsTest, self).setUp() + bmk_owner = "bmk_owner" + self.create_user(bmk_owner) + prep, refmodel, _, benchmark = self.shortcut_create_benchmark( + bmk_owner, bmk_owner, bmk_owner, bmk_owner + ) + self.bmk_owner = bmk_owner + self.benchmark = benchmark + self.prep = prep + self.refmodel = refmodel + + def __create_asset(self, user): + self.set_credentials(user) + dataset = self.mock_dataset( + self.prep["id"], generated_uid=f"{user}genid", state="OPERATION" + ) + dataset = self.create_dataset(dataset).data + assoc = self.mock_dataset_association( + self.benchmark["id"], dataset["id"], approval_status="APPROVED" + ) + self.create_dataset_association(assoc, user, self.bmk_owner) + result = self.mock_result( + self.benchmark["id"], self.refmodel["id"], dataset["id"] + ) + result = self.create_result(result).data + return result + + def test_endpoint_returns_current_user_assets(self): + url = self.api_prefix + "/me/results/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # create an asset for each user + result1 = self.__create_asset(user1) + result2 = self.__create_asset(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + resp1 = response1.data["results"] + resp2 = response2.data["results"] + self.assertEqual(len(resp1), 1) + self.assertEqual(len(resp2), 1) + self.assertEqual(resp1[0]["id"], result1["id"]) + self.assertEqual(resp2[0]["id"], result2["id"]) + + +class BenchmarkDatasetTest(MedPerfTest): + def __create_asset(self, user): + prep, _, _, benchmark = self.shortcut_create_benchmark( + user, + user, + user, + user, + prep_mlcube_kwargs={"name": f"{user}prep", "mlcube_hash": f"{user}prep"}, + ref_mlcube_kwargs={"name": f"{user}ref", "mlcube_hash": f"{user}ref"}, + eval_mlcube_kwargs={"name": f"{user}eval", "mlcube_hash": f"{user}eval"}, + name=f"{user}name", + ) + self.set_credentials(user) + dataset = self.mock_dataset( + prep["id"], generated_uid=f"{user}genuid", state="OPERATION" + ) + dataset = self.create_dataset(dataset).data + + assoc = self.mock_dataset_association(benchmark["id"], dataset["id"]) + assoc = self.create_dataset_association(assoc, user, user).data + + return assoc + + def test_endpoint_returns_current_user_assets(self): + url = self.api_prefix + "/me/datasets/associations/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # create an asset for each user + assoc1 = self.__create_asset(user1) + assoc2 = self.__create_asset(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + resp1 = response1.data["results"] + resp2 = response2.data["results"] + self.assertEqual(len(resp1), 1) + self.assertEqual(len(resp2), 1) + self.assertEqual(resp1[0]["id"], assoc1["id"]) + self.assertEqual(resp2[0]["id"], assoc2["id"]) + + +class BenchmarkMlCubeTest(MedPerfTest): + def __create_asset(self, user): + _, _, _, benchmark = self.shortcut_create_benchmark( + user, + user, + user, + user, + prep_mlcube_kwargs={"name": f"{user}prep", "mlcube_hash": f"{user}prep"}, + ref_mlcube_kwargs={"name": f"{user}ref", "mlcube_hash": f"{user}ref"}, + eval_mlcube_kwargs={"name": f"{user}eval", "mlcube_hash": f"{user}eval"}, + name=f"{user}name", + ) + self.set_credentials(user) + mlcube = self.mock_mlcube( + name=f"{user}name", mlcube_hash=f"{user}hash", state="OPERATION" + ) + mlcube = self.create_mlcube(mlcube).data + + assoc = self.mock_mlcube_association(benchmark["id"], mlcube["id"]) + assoc = self.create_mlcube_association(assoc, user, user).data + + return assoc + + def test_endpoint_returns_current_user_assets(self): + url = self.api_prefix + "/me/mlcubes/associations/" + + # setup users + user1 = "user1" + user2 = "user2" + self.create_user(user1) + self.create_user(user2) + + # create an asset for each user + assoc1 = self.__create_asset(user1) + assoc2 = self.__create_asset(user2) + + # Act + self.set_credentials(user1) + response1 = self.client.get(url) + self.set_credentials(user2) + response2 = self.client.get(url) + + # Assert + self.assertEqual(response1.status_code, status.HTTP_200_OK) + self.assertEqual(response2.status_code, status.HTTP_200_OK) + resp1 = response1.data["results"] + resp2 = response2.data["results"] + self.assertEqual(len(resp1), 1) + self.assertEqual(len(resp2), 1) + self.assertEqual(resp1[0]["id"], assoc1["id"]) + self.assertEqual(resp2[0]["id"], assoc2["id"])