Skip to content

Commit

Permalink
Merge pull request OpenMined#9304 from OpenMined/bschell/add-l0-worke…
Browse files Browse the repository at this point in the history 
…r-pool-testing

Add L0 worker pool testing notebook
  • Loading branch information
@BrendanSchell
BrendanSchell authored Sep 19, 2024
2 parents 8604131 + bca09d9 commit 8873210
Show file tree
Hide file tree
Showing 6 changed files with 387 additions and 13 deletions.
6 changes: 3 additions & 3 deletions ...ry/sync/01-setup-high-low-datasites.ipynb → ...y/sync/000-setup-high-low-datasites.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@
"source": [
"# stdlib\n",
"environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n",
"high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"auto\")\n",
"low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"auto\")\n",
"high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n",
"low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n",
"print(environment, high_port, low_port)"
]
},
Expand Down Expand Up @@ -327,7 +327,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
"version": "3.12.4"
}
},
"nbformat": 4,
Expand Down
374 changes: 374 additions & 0 deletions notebooks/scenarios/bigquery/sync/001-scale-delete-worker-pools.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,374 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0",
"metadata": {},
"outputs": [],
"source": [
"# stdlib\n",
"import os\n",
"\n",
"# Testing works over 4 possibilities\n",
"# 1. (python/in-memory workers and using tox commands)\n",
"# 2. (python/in-memory workers and manually running notebooks)\n",
"# 3. (using k8s and using tox commands)\n",
"# 4. (using k8s and manually running notebooks)\n",
"# Uncomment the lines below if in the 4th possibility\n",
"\n",
"# os.environ[\"ORCHESTRA_DEPLOYMENT_TYPE\"] = \"remote\"\n",
"# os.environ[\"DEV_MODE\"] = \"True\"\n",
"# os.environ[\"TEST_EXTERNAL_REGISTRY\"] = \"k3d-registry.localhost:5800\"\n",
"# os.environ[\"CLUSTER_HTTP_PORT_HIGH\"] = \"9081\"\n",
"# os.environ[\"CLUSTER_HTTP_PORT_LOW\"] = \"9083\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1",
"metadata": {},
"outputs": [],
"source": [
"# stdlib\n",
"environment = os.environ.get(\"ORCHESTRA_DEPLOYMENT_TYPE\", \"python\")\n",
"high_port = os.environ.get(\"CLUSTER_HTTP_PORT_HIGH\", \"9081\")\n",
"low_port = os.environ.get(\"CLUSTER_HTTP_PORT_LOW\", \"9083\")\n",
"print(environment, high_port, low_port)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2",
"metadata": {},
"outputs": [],
"source": [
"# stdlib\n",
"import os\n",
"\n",
"# syft absolute\n",
"import syft as sy\n",
"from syft.util.test_helpers.email_helpers import Timeout\n",
"from syft.util.test_helpers.email_helpers import get_email_server"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3",
"metadata": {},
"outputs": [],
"source": [
"num_workers = int(os.environ.get(\"NUM_TEST_WORKERS\", 1))\n",
"\n",
"# ROOT_EMAIL = \"[email protected]\"\n",
"# ROOT_PASSWORD = \"bqpw\"\n",
"environment"
]
},
{
"cell_type": "markdown",
"id": "4",
"metadata": {},
"source": [
"### Launch server & login"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5",
"metadata": {},
"outputs": [],
"source": [
"server_low = sy.orchestra.launch(\n",
" name=\"bigquery-low\",\n",
" server_side_type=\"low\",\n",
" dev_mode=True,\n",
" n_consumers=1,\n",
" create_producer=True,\n",
" port=low_port,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6",
"metadata": {},
"outputs": [],
"source": [
"email_server, smtp_server = get_email_server(reset=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7",
"metadata": {},
"outputs": [],
"source": [
"low_client = server_low.login(email=\"[email protected]\", password=\"changethis\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8",
"metadata": {},
"outputs": [],
"source": [
"assert len(low_client.worker_pools.get_all()) == 2"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9",
"metadata": {},
"outputs": [],
"source": [
"default_worker_pool = low_client.worker_pools.get_by_name(\"default-pool\")\n",
"default_worker_pool"
]
},
{
"cell_type": "markdown",
"id": "10",
"metadata": {},
"source": [
"### Scale Worker pool"
]
},
{
"cell_type": "markdown",
"id": "11",
"metadata": {},
"source": [
"##### Scale up"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "12",
"metadata": {},
"outputs": [],
"source": [
"# Scale to 1\n",
"if environment == \"remote\":\n",
" low_client.api.worker_pool.scale(\n",
" number=num_workers, pool_name=default_worker_pool.name\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "13",
"metadata": {},
"outputs": [],
"source": [
"low_client.api.services.worker_pool[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "14",
"metadata": {},
"outputs": [],
"source": [
"# Scale up workers\n",
"if environment == \"remote\":\n",
" scale_up_result = low_client.api.worker_pool.scale(\n",
" number=5, pool_name=default_worker_pool.name\n",
" )\n",
" if environment == \"remote\":\n",
" assert scale_up_result, scale_up_result\n",
"\n",
" assert (\n",
" low_client.api.services.worker_pool[default_worker_pool.name].max_count == 5\n",
" )"
]
},
{
"cell_type": "markdown",
"id": "15",
"metadata": {},
"source": [
"##### Scale down"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "16",
"metadata": {},
"outputs": [],
"source": [
"# Scale down workers, this gracefully shutdowns the consumers\n",
"if environment == \"remote\":\n",
" scale_down_result = low_client.api.worker_pool.scale(\n",
" number=num_workers, pool_name=default_worker_pool.name\n",
" )\n",
" assert scale_down_result, scale_down_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17",
"metadata": {},
"outputs": [],
"source": [
"if environment == \"remote\":\n",
"\n",
" def has_worker_scaled_down():\n",
" return (\n",
" low_client.api.worker_pool[default_worker_pool.name].max_count\n",
" == num_workers\n",
" )\n",
"\n",
" worker_scale_timeout = Timeout(timeout_duration=20)\n",
" worker_scale_timeout.run_with_timeout(has_worker_scaled_down)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18",
"metadata": {},
"outputs": [],
"source": [
"if environment == \"remote\":\n",
" assert (\n",
" low_client.api.services.worker_pool[default_worker_pool.name].max_count\n",
" == num_workers\n",
" )"
]
},
{
"cell_type": "markdown",
"id": "19",
"metadata": {},
"source": [
"#### Delete Worker Pool"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "20",
"metadata": {},
"outputs": [],
"source": [
"pool_delete_result = low_client.api.services.worker_pool.delete(\n",
" pool_name=default_worker_pool.name\n",
")\n",
"pool_delete_result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "21",
"metadata": {},
"outputs": [],
"source": [
"with sy.raises(KeyError):\n",
" _ = low_client.api.services.worker_pool[default_worker_pool.name]"
]
},
{
"cell_type": "markdown",
"id": "22",
"metadata": {},
"source": [
"#### Re-launch the default worker pool"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "23",
"metadata": {},
"outputs": [],
"source": [
"default_worker_image = default_worker_pool.image"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "24",
"metadata": {},
"outputs": [],
"source": [
"launch_result = low_client.api.services.worker_pool.launch(\n",
" pool_name=default_worker_pool.name,\n",
" image_uid=default_worker_image.id,\n",
" num_workers=num_workers,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "25",
"metadata": {},
"outputs": [],
"source": [
"assert low_client.api.services.worker_pool[default_worker_pool.name]\n",
"assert (\n",
" low_client.api.services.worker_pool[default_worker_pool.name].max_count\n",
" == num_workers\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "26",
"metadata": {},
"outputs": [],
"source": [
"smtp_server.stop()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "27",
"metadata": {},
"outputs": [],
"source": [
"if environment != \"remote\":\n",
" server_low.land()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "syft",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit 8873210

Please sign in to comment.