From 5e7cafa2d20c4450d9cf04f366a5706a5f8d792b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 20:52:25 +0200 Subject: [PATCH 01/27] Update node.py --- packages/syft/src/syft/node/node.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 2340ca33ba2..318f37fdc9b 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,10 +221,9 @@ def in_kubernetes() -> bool: def get_venv_packages() -> str: - res = subprocess.getoutput( - "pip list --format=freeze", - ) - return res + process = subprocess.Popen(["pip", "list", "--format=freeze"], stdout=subprocess.PIPE) + output, _ = process.communicate() + return output.decode() def get_syft_worker() -> bool: From 1611968edf45f0613fdb3633ff3183a46af499d9 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 20:55:25 +0200 Subject: [PATCH 02/27] Update node.py --- packages/syft/src/syft/node/node.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 318f37fdc9b..7626283630a 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,7 +221,9 @@ def in_kubernetes() -> bool: def get_venv_packages() -> str: - process = subprocess.Popen(["pip", "list", "--format=freeze"], stdout=subprocess.PIPE) + process = subprocess.Popen( + ["pip", "list", "--format=freeze"], stdout=subprocess.PIPE + ) output, _ = process.communicate() return output.decode() From ddeff10857cf27bc26875a8382032877d895d98c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:06:56 +0200 Subject: [PATCH 03/27] Update node.py --- packages/syft/src/syft/node/node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 7626283630a..d1f7b47869a 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,8 +221,11 @@ def in_kubernetes() -> bool: def get_venv_packages() -> str: + pip_path = shutil.which("pip") + if pip_path is None: + raise Exception("pip not found") process = subprocess.Popen( - ["pip", "list", "--format=freeze"], stdout=subprocess.PIPE + [pip_path, "list", "--format=freeze"], stdout=subprocess.PIPE ) output, _ = process.communicate() return output.decode() From 30f5851cdf24d4282bdf0444caa44c4081300279 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:18:37 +0200 Subject: [PATCH 04/27] Update node.py --- packages/syft/src/syft/node/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index d1f7b47869a..9d84447f794 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -225,7 +225,7 @@ def get_venv_packages() -> str: if pip_path is None: raise Exception("pip not found") process = subprocess.Popen( - [pip_path, "list", "--format=freeze"], stdout=subprocess.PIPE + [pip_path, "list", "--format=freeze"], stdout=subprocess.PIPE, shell=False ) output, _ = process.communicate() return output.decode() From 48793d81329d03b357114a18b078bc762454c241 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:24:31 +0200 Subject: [PATCH 05/27] Update node.py --- packages/syft/src/syft/node/node.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 9d84447f794..3d5170ef74f 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -224,11 +224,10 @@ def get_venv_packages() -> str: pip_path = shutil.which("pip") if pip_path is None: raise Exception("pip not found") - process = subprocess.Popen( - [pip_path, "list", "--format=freeze"], stdout=subprocess.PIPE, shell=False + process = subprocess.run( + [pip_path, "list", "--format=freeze"], capture_output=True, text=True ) - output, _ = process.communicate() - return output.decode() + return process.stdout def get_syft_worker() -> bool: From 63822410d9d9197f492db49beee094fb87ccacd8 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:27:29 +0200 Subject: [PATCH 06/27] Update node.py --- packages/syft/src/syft/node/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 3d5170ef74f..43ef69f3114 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -224,10 +224,10 @@ def get_venv_packages() -> str: pip_path = shutil.which("pip") if pip_path is None: raise Exception("pip not found") - process = subprocess.run( - [pip_path, "list", "--format=freeze"], capture_output=True, text=True + output = subprocess.check_output( + [pip_path, "list", "--format=freeze"], text=True ) - return process.stdout + return output def get_syft_worker() -> bool: From ad6550a2e50139d1a34d7dc1cd47452dfb8f68de Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:30:14 +0200 Subject: [PATCH 07/27] Update node.py --- packages/syft/src/syft/node/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 43ef69f3114..3d5170ef74f 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -224,10 +224,10 @@ def get_venv_packages() -> str: pip_path = shutil.which("pip") if pip_path is None: raise Exception("pip not found") - output = subprocess.check_output( - [pip_path, "list", "--format=freeze"], text=True + process = subprocess.run( + [pip_path, "list", "--format=freeze"], capture_output=True, text=True ) - return output + return process.stdout def get_syft_worker() -> bool: From 7660c0e90912db764130e41721ee96c109ebbd4c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:33:52 +0200 Subject: [PATCH 08/27] Update node.py --- packages/syft/src/syft/node/node.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 3d5170ef74f..be02543f84e 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,13 +221,17 @@ def in_kubernetes() -> bool: def get_venv_packages() -> str: - pip_path = shutil.which("pip") - if pip_path is None: - raise Exception("pip not found") - process = subprocess.run( - [pip_path, "list", "--format=freeze"], capture_output=True, text=True - ) - return process.stdout + try: + result = subprocess.run( + ["pip", "list", "--format=freeze"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + text=True + ) + return result.stdout + except subprocess.CalledProcessError as e: + return f"An error occurred: {e.stderr}" def get_syft_worker() -> bool: From 771c93cf3c84ed35ea8081a630926a0a8bbf7763 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:36:44 +0200 Subject: [PATCH 09/27] Update node.py --- packages/syft/src/syft/node/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index be02543f84e..7c526e539a6 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -11,6 +11,7 @@ from pathlib import Path import shutil import subprocess # nosec +import sys import tempfile from time import sleep import traceback @@ -223,7 +224,7 @@ def in_kubernetes() -> bool: def get_venv_packages() -> str: try: result = subprocess.run( - ["pip", "list", "--format=freeze"], + [sys.executable, "-m", "pip", "list", "--format=freeze"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, From 0f721c65666f05b76248f15537c29c16d08d5d70 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:42:08 +0200 Subject: [PATCH 10/27] Update setup.cfg --- packages/syft/setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 4224239d091..9c3219c4849 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -199,6 +199,7 @@ ignore = B027 B026 B028 + B603 max-line-length = 120 exclude = From 38961ee9be175aabe61b1614730ad18339d4c7b8 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:44:11 +0200 Subject: [PATCH 11/27] Update node.py --- packages/syft/src/syft/node/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 7c526e539a6..082d630e509 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -228,7 +228,7 @@ def get_venv_packages() -> str: stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, - text=True + text=True, ) return result.stdout except subprocess.CalledProcessError as e: From 81b7b4d19039c1fcec67d59c2533b64fd61fb1bb Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:50:43 +0200 Subject: [PATCH 12/27] Update node.py --- packages/syft/src/syft/node/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 082d630e509..4c812ffc82f 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -223,12 +223,12 @@ def in_kubernetes() -> bool: def get_venv_packages() -> str: try: + # subprocess call is safe because it uses a fully qualified path and fixed arguments result = subprocess.run( [sys.executable, "-m", "pip", "list", "--format=freeze"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=True, - text=True, + text=True ) return result.stdout except subprocess.CalledProcessError as e: From 892473bb36b958646b228f2d0eba864c03ce0aa5 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:52:30 +0200 Subject: [PATCH 13/27] Update node.py --- packages/syft/src/syft/node/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 4c812ffc82f..6ddcb5bcc3f 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -228,7 +228,7 @@ def get_venv_packages() -> str: [sys.executable, "-m", "pip", "list", "--format=freeze"], capture_output=True, check=True, - text=True + text=True, ) return result.stdout except subprocess.CalledProcessError as e: From 4a857a24f67032377e8033cda9a7f30ffab3090f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:56:30 +0200 Subject: [PATCH 14/27] Update setup.cfg From e88a9483047ab4b5c9b99e04f8297fc6047b4c3f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:56:56 +0200 Subject: [PATCH 15/27] Update setup.cfg --- packages/syft/setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 9c3219c4849..4224239d091 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -199,7 +199,6 @@ ignore = B027 B026 B028 - B603 max-line-length = 120 exclude = From 0c95a59c06918648d63a74d39ab255027e6dd199 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:59:11 +0200 Subject: [PATCH 16/27] Update ruff.toml --- ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/ruff.toml b/ruff.toml index 6d8e8a2f93a..70a4dbb850d 100644 --- a/ruff.toml +++ b/ruff.toml @@ -19,6 +19,7 @@ select = [ ignore = [ "B904", # check for raise statements in exception handlers that lack a from clause "B905", # zip() without an explicit strict= parameter + "B603", # start_process_with_partial_path] Starting a process with a partial executable path ] [lint.per-file-ignores] From 2fd3329c4e8585d2197f36d6900ed68491a1033a Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 21:59:39 +0200 Subject: [PATCH 17/27] Update ruff.toml --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 70a4dbb850d..dcee7c8ac31 100644 --- a/ruff.toml +++ b/ruff.toml @@ -17,9 +17,9 @@ select = [ "UP", # pyupgrade ] ignore = [ + "B603", # start_process_with_partial_path] Starting a process with a partial executable path "B904", # check for raise statements in exception handlers that lack a from clause "B905", # zip() without an explicit strict= parameter - "B603", # start_process_with_partial_path] Starting a process with a partial executable path ] [lint.per-file-ignores] From 66ff40903b0170d55b5ba21ff8f4fd78277e8950 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 22:02:46 +0200 Subject: [PATCH 18/27] Update ruff.toml --- ruff.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index dcee7c8ac31..6d8e8a2f93a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -17,7 +17,6 @@ select = [ "UP", # pyupgrade ] ignore = [ - "B603", # start_process_with_partial_path] Starting a process with a partial executable path "B904", # check for raise statements in exception handlers that lack a from clause "B905", # zip() without an explicit strict= parameter ] From 5139c04d10fb936549ae1d6f4da4927aeb7a0d77 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 22:08:24 +0200 Subject: [PATCH 19/27] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c374b12ff36..79c0830d701 100644 --- a/tox.ini +++ b/tox.ini @@ -232,7 +232,7 @@ changedir = {toxinidir}/packages/syft deps = {[testenv:syft]deps} commands = - bandit -r src + bandit -r src --skip B607 # restrictedpython 6.2 safety check -i 60840 -i 54229 -i 54230 -i 42923 -i 54230 -i 54229 -i 62044 -i 65213 -i 54564 From 0f395ef8f95a71cab1b935a7b1c4469f2df9f4fc Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 22:13:12 +0200 Subject: [PATCH 20/27] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 79c0830d701..294c2bfcf9b 100644 --- a/tox.ini +++ b/tox.ini @@ -232,7 +232,7 @@ changedir = {toxinidir}/packages/syft deps = {[testenv:syft]deps} commands = - bandit -r src --skip B607 + bandit -r src --exclude src/syft/node/node.py # restrictedpython 6.2 safety check -i 60840 -i 54229 -i 54230 -i 42923 -i 54230 -i 54229 -i 62044 -i 65213 -i 54564 From bd3bff93c6dfcb3e3a73ef781e3f420174ae182f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 30 May 2024 22:57:32 +0200 Subject: [PATCH 21/27] Update setup.cfg --- packages/syft/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 4224239d091..7ee32b1a691 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -92,7 +92,7 @@ data_science = dev = %(test_plugins)s %(telemetry)s - bandit==1.7.7 + bandit==1.7.8 ruff==0.3.0 importlib-metadata==6.8.0 isort==5.13.2 From 7eaec0fa9f2432048c5750c1fa96990d5e019878 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 31 May 2024 06:15:51 +0200 Subject: [PATCH 22/27] Update setup.cfg --- packages/syft/setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 7ee32b1a691..a0fae87fc7b 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -93,11 +93,11 @@ dev = %(test_plugins)s %(telemetry)s bandit==1.7.8 - ruff==0.3.0 - importlib-metadata==6.8.0 + ruff==0.4.6 + importlib-metadata==7.1.0 isort==5.13.2 - mypy==1.7.1 - pre-commit==3.6.2 + mypy==1.10.0 + pre-commit==3.7.1 safety>=2.4.0b2 telemetry = From a45f6d1e84fc991bae8e8eba687e35e410f9a65c Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 31 May 2024 08:50:33 +0200 Subject: [PATCH 23/27] revert --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 294c2bfcf9b..c374b12ff36 100644 --- a/tox.ini +++ b/tox.ini @@ -232,7 +232,7 @@ changedir = {toxinidir}/packages/syft deps = {[testenv:syft]deps} commands = - bandit -r src --exclude src/syft/node/node.py + bandit -r src # restrictedpython 6.2 safety check -i 60840 -i 54229 -i 54230 -i 42923 -i 54230 -i 54229 -i 62044 -i 65213 -i 54564 From f162e775240c8da3d29e4ab7ecb230d4957e1943 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 31 May 2024 08:52:29 +0200 Subject: [PATCH 24/27] revert --- packages/syft/setup.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index f7f6e145db0..ac91f01c061 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -92,12 +92,12 @@ data_science = dev = %(test_plugins)s %(telemetry)s - bandit==1.7.8 - ruff==0.4.6 - importlib-metadata==7.1.0 + bandit==1.7.7 + ruff==0.3.0 + importlib-metadata==6.8.0 isort==5.13.2 - mypy==1.10.0 - pre-commit==3.7.1 + mypy==1.7.1 + pre-commit==3.6.2 safety>=2.4.0b2 telemetry = From 411f3e5fefe77d1c2830bc7a87db91aaf354ed26 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 31 May 2024 08:58:30 +0200 Subject: [PATCH 25/27] Update node.py --- packages/syft/src/syft/node/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 6ddcb5bcc3f..87dca1e1a85 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,7 +221,7 @@ def in_kubernetes() -> bool: return get_container_host() == "k8s" -def get_venv_packages() -> str: +def get_venv_packages() -> str: # nosec try: # subprocess call is safe because it uses a fully qualified path and fixed arguments result = subprocess.run( From 4f23b8efd5fbea84b6e640cc074ac83d34a0e491 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 31 May 2024 09:04:38 +0200 Subject: [PATCH 26/27] Update node.py --- packages/syft/src/syft/node/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 87dca1e1a85..663bbad6fc8 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,7 +221,7 @@ def in_kubernetes() -> bool: return get_container_host() == "k8s" -def get_venv_packages() -> str: # nosec +def get_venv_packages() -> str: # nosec try: # subprocess call is safe because it uses a fully qualified path and fixed arguments result = subprocess.run( From 2a8a304c1d0698344065f7c5d48053fd79e54d67 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 31 May 2024 09:05:50 +0200 Subject: [PATCH 27/27] Update node.py --- packages/syft/src/syft/node/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 663bbad6fc8..dafe86409c0 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -221,11 +221,11 @@ def in_kubernetes() -> bool: return get_container_host() == "k8s" -def get_venv_packages() -> str: # nosec +def get_venv_packages() -> str: try: # subprocess call is safe because it uses a fully qualified path and fixed arguments result = subprocess.run( - [sys.executable, "-m", "pip", "list", "--format=freeze"], + [sys.executable, "-m", "pip", "list", "--format=freeze"], # nosec capture_output=True, check=True, text=True,