From 587ef626b7d265e3501c2930c1133396c67d6dda Mon Sep 17 00:00:00 2001 From: John Kitchin Date: Tue, 8 Oct 2024 22:24:46 -0400 Subject: [PATCH] initial commit for claude-light (#86) * initial commit for clade-light * rm unused imports --- examples/claude-light/claude_alab/__init__.py | 17 ++ examples/claude-light/claude_alab/config.toml | 43 +++ .../claude_alab/devices/__init__.py | 1 + .../claude_alab/devices/claude.py | 43 +++ .../claude_alab/tasks/__init__.py | 1 + .../claude-light/claude_alab/tasks/claude.py | 15 + examples/claude-light/example.ipynb | 256 ++++++++++++++++++ examples/claude-light/pyproject.toml | 12 + examples/claude-light/readme.org | 170 ++++++++++++ 9 files changed, 558 insertions(+) create mode 100644 examples/claude-light/claude_alab/__init__.py create mode 100644 examples/claude-light/claude_alab/config.toml create mode 100644 examples/claude-light/claude_alab/devices/__init__.py create mode 100644 examples/claude-light/claude_alab/devices/claude.py create mode 100644 examples/claude-light/claude_alab/tasks/__init__.py create mode 100644 examples/claude-light/claude_alab/tasks/claude.py create mode 100644 examples/claude-light/example.ipynb create mode 100644 examples/claude-light/pyproject.toml create mode 100644 examples/claude-light/readme.org diff --git a/examples/claude-light/claude_alab/__init__.py b/examples/claude-light/claude_alab/__init__.py new file mode 100644 index 00000000..8613df5e --- /dev/null +++ b/examples/claude-light/claude_alab/__init__.py @@ -0,0 +1,17 @@ +import os +from pathlib import Path + +# keep this line at the top of the file +os.putenv("ALABOS_CONFIG_PATH", Path(__file__).parent.absolute() / "config.toml") + +from alab_management import add_device, add_task + +from .devices.claude import ClaudeLight +from .tasks.claude import MeasureRGB + +# set the config path to the default config file + + +add_device(ClaudeLight(name="rgb")) +add_task(MeasureRGB) + diff --git a/examples/claude-light/claude_alab/config.toml b/examples/claude-light/claude_alab/config.toml new file mode 100644 index 00000000..54de04b1 --- /dev/null +++ b/examples/claude-light/claude_alab/config.toml @@ -0,0 +1,43 @@ +[general] +name = 'claude_alab' # Put the name of the lab here, it will be used as the DB name +working_dir = "." # the working directory of the lab, where the device and task definitions are stored + +[mongodb] # the MongoDB configuration +host = 'localhost' +password = '' +port = 27017 +username = '' + +# all the completed experiments are stored in this database +# the db name will be the lab name + '_completed' +[mongodb_completed] +host = "localhost" +password = "" +port = 27017 +username = "" + +[rabbitmq] # the RabbitMQ configuration +host = "localhost" +port = 5672 + +# the user notification configuration, currently only email and slack are supported +# if you don't want to use them, just leave them empty +[alarm] +# the email configuration. All the user notification will be sent to all the email_receivers in the list +# the email_sender is the email address of the sender, e.g. alabos@xxx.com +email_receivers = [] +email_sender = " " +email_password = " " + +# the slack configuration. All the user notification will be sent to the slack_channel_id +# the slack_bot_token is the token of the slack bot, you can get it from https://api.slack.com/apps +slack_bot_token = " " +slack_channel_id = " " + +[large_result_storage] +# the default storage configuration for tasks that generate large results +# (>16 MB, cannot be contained in MongoDB) +# currently only gridfs is supported +# storage_type is defined by using LargeResult class located in alab_management/task_view/task.py +# you can override this default configuration by setting the storage_type in the task definition +default_storage_type = "gridfs" diff --git a/examples/claude-light/claude_alab/devices/__init__.py b/examples/claude-light/claude_alab/devices/__init__.py new file mode 100644 index 00000000..ce94388e --- /dev/null +++ b/examples/claude-light/claude_alab/devices/__init__.py @@ -0,0 +1 @@ +"""You don't have to modify this file. It is used to make Python treat the directory as containing packages.""" diff --git a/examples/claude-light/claude_alab/devices/claude.py b/examples/claude-light/claude_alab/devices/claude.py new file mode 100644 index 00000000..64750591 --- /dev/null +++ b/examples/claude-light/claude_alab/devices/claude.py @@ -0,0 +1,43 @@ +import requests +from typing import ClassVar + +from alab_management.device_view import BaseDevice +from alab_management.sample_view import SamplePosition + + +class ClaudeLight(BaseDevice): + """Claude Light definition""" + description: ClassVar[str] = "Claude-Light device" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.running = None + + def __str__(self): + return 'An RGB Claude Light' + + @property + def sample_positions(self): + """Sample positions define the sample positions related to this device.""" + return [ + SamplePosition( + name="DefaultSamplePosition", + number=1, + description="Default sample position", + ) + ] + + def connect(self): + self.running = True + + def disconnect(self): + self.running = False + + def is_running(self): + return self.running + + def measure(self, R=0, G=0, B=0): + resp = requests.get('https://claude-light.cheme.cmu.edu/api', + params={'R': R, 'G': G, 'B': B}) + data = resp.json() + return data diff --git a/examples/claude-light/claude_alab/tasks/__init__.py b/examples/claude-light/claude_alab/tasks/__init__.py new file mode 100644 index 00000000..ce94388e --- /dev/null +++ b/examples/claude-light/claude_alab/tasks/__init__.py @@ -0,0 +1 @@ +"""You don't have to modify this file. It is used to make Python treat the directory as containing packages.""" diff --git a/examples/claude-light/claude_alab/tasks/claude.py b/examples/claude-light/claude_alab/tasks/claude.py new file mode 100644 index 00000000..28e3f6ee --- /dev/null +++ b/examples/claude-light/claude_alab/tasks/claude.py @@ -0,0 +1,15 @@ +from alab_management.task_view.task import BaseTask + +class MeasureRGB(BaseTask): + """Measurement task definition.""" + def __init__(self, R=0, G=0, B=0, **kwargs): + super().__init__(**kwargs) + self.R = R + self.G = G + self.B = B + + def run(self): + print('Running a job') + with self.lab_view.request_resources({"rgb": {}}) as (devices, _): + print(devices) + return devices["rgb"].measure(self.R, self.G, self.B) diff --git a/examples/claude-light/example.ipynb b/examples/claude-light/example.ipynb new file mode 100644 index 00000000..ffc93fdd --- /dev/null +++ b/examples/claude-light/example.ipynb @@ -0,0 +1,256 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "07e6b35b-6fde-4bb8-961e-2f2cd7faee16", + "metadata": {}, + "source": [ + "# Setting up\n", + "\n", + "Run these commands in this directory. This will install the lab and setup your database .\n", + "\n", + "\n", + " pip install -e .\n", + "\n", + "To start the server, run these commands.\n", + "\n", + "\n", + " export ALABOS_CONFIG_PATH=`pwd`/claude_alab/config.toml\n", + " export SIM_MODE_FLAG=FALSE\n", + " alabos setup\n", + " alabos launch &\n", + " alabos launch_worker &" + ] + }, + { + "cell_type": "markdown", + "id": "0fa7bd38-8367-41b7-a372-53165592f32e", + "metadata": {}, + "source": [ + "# setup and submit an experiment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "63d09232-7f62-4182-8483-d7cd62729aa8", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ObjectId('6705d63f9fa06af11c9361bc')" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from alab_management.builders import ExperimentBuilder\n", + "from claude_alab.tasks.claude import MeasureRGB\n", + "\n", + "# you need a name for the experiment\n", + "# you can also provide tags for better organization\n", + "# You can also provide arbitrary kwargs to the builder\n", + "# it will be stored as metadata (e.g., description)\n", + "exp = ExperimentBuilder(name=\"MyExperiment\",\n", + " tags=[\"tag1\", \"tag2\"],\n", + " description=\"My first experiment\")\n", + "\n", + "sample = exp.add_sample(name=\"my_sample_1\",\n", + " tags=[\"project\"],\n", + " description=\"My first sample\")\n", + "\n", + "\n", + "task1 = MeasureRGB(R=0, G=0, B=0)\n", + "task2 = MeasureRGB(R=0, G=1, B=0)\n", + "# you can also do task.add_to([sample1, sample2, ...]) to add multiple samples to the task\n", + "task1.add_to(sample)\n", + "task2.add_to(sample)\n", + "exp_id = exp.submit(address=\"http://localhost:8895\")\n", + "exp_id" + ] + }, + { + "cell_type": "markdown", + "id": "549ffdc6-7141-4a3c-9b5b-2009a5cce382", + "metadata": {}, + "source": [ + "# get the experiment status" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1f9d8e48-af24-4282-9bd0-8ec7375b50ef", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '6705d63f9fa06af11c9361bc',\n", + " 'name': 'MyExperiment',\n", + " 'progress': 1.0,\n", + " 'samples': [{'id': '6705d63f9fa06af11c9361bd',\n", + " 'name': 'my_sample_1',\n", + " 'position': None}],\n", + " 'status': 'COMPLETED',\n", + " 'submitted_at': 'Tue, 08 Oct 2024 21:02:55 GMT',\n", + " 'tasks': [{'id': '6705d63f9fa06af11c9361be',\n", + " 'message': '',\n", + " 'status': 'COMPLETED',\n", + " 'type': 'MeasureRGB'},\n", + " {'id': '6705d63f9fa06af11c9361bf',\n", + " 'message': '',\n", + " 'status': 'COMPLETED',\n", + " 'type': 'MeasureRGB'}]}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from alab_management import get_experiment_status, get_experiment_result\n", + "\n", + "get_experiment_status(exp_id)" + ] + }, + { + "cell_type": "markdown", + "id": "de0259e5-8b27-4be9-a1e4-dcff28aa2e3e", + "metadata": {}, + "source": [ + "# Get the results" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "60d4cced-f55b-4c30-93e1-4b3c49e9bba3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'completed_at': '2024-10-08T21:03:00.003000',\n", + " 'id': '6705d63f9fa06af11c9361bc',\n", + " 'metadata': {'description': 'My first experiment'},\n", + " 'name': 'MyExperiment',\n", + " 'progress': 1.0,\n", + " 'samples': [{'id': '6705d63f9fa06af11c9361bd',\n", + " 'metadata': {'description': 'My first sample'},\n", + " 'name': 'my_sample_1',\n", + " 'tags': ['project']}],\n", + " 'status': 'COMPLETED',\n", + " 'submitted_at': '2024-10-08T21:02:55.943000',\n", + " 'tags': ['tag1', 'tag2'],\n", + " 'tasks': [{'completed_at': '2024-10-08T21:02:57.863000',\n", + " 'id': '6705d63f9fa06af11c9361be',\n", + " 'message': '',\n", + " 'parameters': {'B': 0, 'G': 0, 'R': 0},\n", + " 'result': {'in': [0.0, 0.0, 0.0],\n", + " 'out': {'415nm': 0,\n", + " '445nm': 0,\n", + " '480nm': 0,\n", + " '515nm': 2,\n", + " '555nm': 5,\n", + " '590nm': 2,\n", + " '630nm': 5,\n", + " '680nm': 0,\n", + " 'clear': 18,\n", + " 'nir': 0}},\n", + " 'samples': ['my_sample_1'],\n", + " 'started_at': '2024-10-08T21:02:56.118000',\n", + " 'status': 'COMPLETED',\n", + " 'type': 'MeasureRGB'},\n", + " {'completed_at': '2024-10-08T21:02:59.847000',\n", + " 'id': '6705d63f9fa06af11c9361bf',\n", + " 'message': '',\n", + " 'parameters': {'B': 0, 'G': 1, 'R': 0},\n", + " 'result': {'in': [0.0, 1.0, 0.0],\n", + " 'out': {'415nm': 923,\n", + " '445nm': 329,\n", + " '480nm': 11541,\n", + " '515nm': 62894,\n", + " '555nm': 9972,\n", + " '590nm': 1298,\n", + " '630nm': 1305,\n", + " '680nm': 1243,\n", + " 'clear': 50552,\n", + " 'nir': 1294}},\n", + " 'samples': ['my_sample_1'],\n", + " 'started_at': '2024-10-08T21:02:58.103000',\n", + " 'status': 'COMPLETED',\n", + " 'type': 'MeasureRGB'}]}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results = get_experiment_result(exp_id)\n", + "results" + ] + }, + { + "cell_type": "markdown", + "id": "9be8cdd2-81dc-4363-9f18-1ee498f90112", + "metadata": {}, + "source": [ + "Here we subtract the background from the measurement. Maybe there is a better way to do this?" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a39fda51-2ea7-4ada-8589-7ebd379a273c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "62892" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r0 = results['tasks'][0]['result']['out']['515nm']\n", + "r1 = results['tasks'][1]['result']['out']['515nm']\n", + "r1 - r0" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/claude-light/pyproject.toml b/examples/claude-light/pyproject.toml new file mode 100644 index 00000000..5faf70a7 --- /dev/null +++ b/examples/claude-light/pyproject.toml @@ -0,0 +1,12 @@ +[build-system] +requires = ["setuptools>=42", "setuptools-git-versioning", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "claude_alab" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["E402"] diff --git a/examples/claude-light/readme.org b/examples/claude-light/readme.org new file mode 100644 index 00000000..76fbe735 --- /dev/null +++ b/examples/claude-light/readme.org @@ -0,0 +1,170 @@ +#+title: Claude_alab + +* Setup the Alab + +Run these commands in this directory. This will install the lab and setup your database. + +#+BEGIN_SRC sh +pip install -e . +alabos setup +#+END_SRC + +To start the server, run these commands. + +#+BEGIN_SRC sh +export ALABOS_CONFIG_PATH=`pwd`/claude_alab/config.toml +export SIM_MODE_FLAG=FALSE + +alabos launch & +alabos launch_worker & +#+END_SRC + +Note: if you add a device, you have to rerun the setup to update your database. + +* Create and submit an experiment + +In this experiment, we run two tasks. One for the background, and one for the measurement in the green channel. + +Note that every time you run this, you get a new experiment. + +Note: there is probably a way to do this with no sample, but most experiments probably involve a sample, so for now I think this is ok., + +#+BEGIN_SRC jupyter-python :tangle submit.py +from alab_management.builders import ExperimentBuilder +from claude_alab.tasks.claude import MeasureRGB + +exp = ExperimentBuilder(name="Claude [2024-10-08 Tue]", + description="My first experiment") + +sample = exp.add_sample(name="sample_1", + tags=["project"], + description="There is not really a sample here, it just holds tasks.") + +# get the background +task1 = MeasureRGB(R=0, G=0, B=0) + +# Measure at G=1 +task2 = MeasureRGB(R=0, G=1, B=0) + +task1.add_to(sample) +task2.add_to(sample) + +exp_id = exp.submit(address="http://localhost:8895") +print(exp_id) +#+END_SRC + +#+RESULTS: +: 6705736aa419b8c2271a7701 + +The output is an experiment id. + +See http://localhost:8895 for the browser UI. + +* Checking experiment status + +You retrieve results by an ID. That means you need to have saved that ID in a variable, or to a file, or printed it somewhere. + +#+BEGIN_SRC jupyter-python +from alab_management import get_experiment_status, get_experiment_result + +get_experiment_status(exp_id) +#+END_SRC + +#+RESULTS: +| id | : | 6705736aa419b8c2271a7701 | name | : | Claude [2024-10-08 Tue] | progress | : | 0.0 | samples | : | ((id : 6705736ba419b8c2271a7702 name : sample_1 position : None)) | status | : | RUNNING | submitted_at | : | Tue, 08 Oct 2024 14:01:14 GMT | tasks | : | ((id : 6705736ba419b8c2271a7703 message : status : RUNNING type : MeasureRGB) (id : 6705736ba419b8c2271a7704 message : status : WAITING type : MeasureRGB)) | + +You can also use a string id. + +#+BEGIN_SRC jupyter-python +get_experiment_status('67056aeba419b8c2271a76ef') +#+END_SRC + +#+RESULTS: +| id | : | 67056aeba419b8c2271a76ef | name | : | Claude [2024-10-08 Tue] | progress | : | 1.0 | samples | : | ((id : 67056aeba419b8c2271a76f0 name : sample_1 position : None)) | status | : | COMPLETED | submitted_at | : | Tue, 08 Oct 2024 13:24:59 GMT | tasks | : | ((id : 67056aeba419b8c2271a76f1 message : status : COMPLETED type : MeasureRGB) (id : 67056aeba419b8c2271a76f2 message : status : COMPLETED type : MeasureRGB)) | + + + +* Retrieving results + +You retrieve results by their ID. + +#+BEGIN_SRC jupyter-python +results = get_experiment_result(exp_id) +results +#+END_SRC + +#+RESULTS: +| completed_at | : | 2024-10-08T14:01:20.197000 | id | : | 6705736aa419b8c2271a7701 | metadata | : | (description : My first experiment) | name | : | Claude [2024-10-08 Tue] | progress | : | 1.0 | samples | : | ((id : 6705736ba419b8c2271a7702 metadata : (description : There is not really a sample here, it just holds tasks.) name : sample_1 tags : (project))) | status | : | COMPLETED | submitted_at | : | 2024-10-08T14:01:14.542000 | tags | : | nil | tasks | : | ((completed_at : 2024-10-08T14:01:17.573000 id : 6705736ba419b8c2271a7703 message : parameters : (B : 0 G : 0 R : 0) result : (in : (0.0 0.0 0.0) out : (415nm : 1033 445nm : 1714 480nm : 2083 515nm : 2485 555nm : 2679 590nm : 2684 630nm : 2546 680nm : 3036 clear : 22124 nir : 10013)) samples : (sample_1) started_at : 2024-10-08T14:01:15.681000 status : COMPLETED type : MeasureRGB) (completed_at : 2024-10-08T14:01:20.026000 id : 6705736ba419b8c2271a7704 message : parameters : (B : 0 G : 1 R : 0) result : (in : (0.0 1.0 0.0) out : (415nm : 1962 445nm : 2054 480nm : 13624 515nm : 65397 555nm : 12640 590nm : 4001 630nm : 3853 680nm : 4309 clear : 65535 nir : 11294)) samples : (sample_1) started_at : 2024-10-08T14:01:17.730000 status : COMPLETED type : MeasureRGB)) | + +Here we subtract our background measurement to get the result we want. + +#+BEGIN_SRC jupyter-python +results['tasks'][1]['result']['out']['515nm'] - results['tasks'][0]['result']['out']['515nm'] +#+END_SRC + +#+RESULTS: +: 62912 + +Maybe that could have been built into the task, but we show the explicit path here. + + +* Get a list of known devices + +It would be a good idea to make this a little richer in output. These seem to be the registered names. + +#+BEGIN_SRC jupyter-python +from alab_management.device_view.device import get_all_devices +for device in get_all_devices(): + print(device) +#+END_SRC + +#+RESULTS: +: rgb + +This is a little clunkier, but it does use the __str__ method defined on ClaudeLight. + +#+BEGIN_SRC jupyter-python +from alab_management.device_view.device import _device_registry +for name in _device_registry: + print(_device_registry[name]) +#+END_SRC + +#+RESULTS: +: An RGB Claude Light + +This would be used with a completion backend + +* Get a list of experiments + +#+BEGIN_SRC jupyter-python +from alab_management.utils.data_objects import get_collection +ec = get_collection("experiment") +for exp in ec.find(): + print(exp['name']) +#+END_SRC + +#+RESULTS: +: MyExperiment +: MyExperiment +: MyExperiment +: Claude [2024-10-08 Tue] +: Claude [2024-10-08 Tue] +: Claude [2024-10-08 Tue] + +Obviously I should do a better job naming my experiments... + +* search for experiments + +You might want to search by sample id (see https://github.com/CederGroupHub/alabos/blob/b5618d25c8fa176d4f5716668be2fadf33fe8c31/alab_management/experiment_view/experiment_view.py#L175), parameters, etc. + + +It looks like you can do something like this with a MongoDB like search syntax. + +#+BEGIN_SRC jupyter-python +print(ec.find_one({'name': 'Claude [2024-10-08 Tue]'})) +#+END_SRC + +#+RESULTS: +: {'_id': ObjectId('67056aeba419b8c2271a76ef'), 'name': 'Claude [2024-10-08 Tue]', 'samples': [{'name': 'sample_1', 'sample_id': ObjectId('67056aeba419b8c2271a76f0'), 'tags': ['project'], 'metadata': {'description': 'There is not really a sample here, it just holds tasks.'}}], 'tasks': [{'type': 'MeasureRGB', 'parameters': {'R': 0, 'G': 0, 'B': 0}, 'prev_tasks': [], 'samples': ['sample_1'], 'task_id': ObjectId('67056aeba419b8c2271a76f1')}, {'type': 'MeasureRGB', 'parameters': {'R': 0, 'G': 1, 'B': 0}, 'prev_tasks': [0], 'samples': ['sample_1'], 'task_id': ObjectId('67056aeba419b8c2271a76f2')}], 'tags': [], 'metadata': {'description': 'My first experiment'}, 'submitted_at': datetime.datetime(2024, 10, 8, 13, 24, 59, 550000), 'status': 'COMPLETED', 'completed_at': datetime.datetime(2024, 10, 8, 13, 25, 5, 962000)} + +