Relational DB support for ParamStore (#326)

* WIP: Refactoring persistence out of ParamStore

* WIP: Refactoring persistence out of ParamStore

* Refactoring commit() and checkout()

* WIP: Cont'd refactoring TinyDB persistence out of  InProcessParamStore impl.

* list_values() refactored to use search_commits() (by key)

* Refactoring continued

* Down to 4 failing tests. Hoorah!

* Fixing migration tests

* Removed `checkout(move_by)` functionality

* Tidied up in_process_param_store.py. Migration code move to migrations.py

* Removed self.__is_dirty

* Persistence ABC

* Organized persistence classes under params/persistence dir

* Bringing in SqlAlchemy model for paramstore persistence

* Moving persistence files into sub-dirs (tindyb, sqlalchemy)

* Simplifying persistence class names

* Cont'd refactoring of params package structure

* SqlAlchemy Commit class renamed CommitTable

* Metadata is now a dataclass

* Implemented SqlAlchemyPersistence.get_commit()

* SqlAlchemyPersistence.commit()

* Fixing broken tests

* SqlAlchemyPersistence.search_commits()

* Integrated SqlAlchemyPersistence into InProcessParamStore

* Workaround for saving commit timestamps as pd.Timestamp

* woot! All param_store tests are green for both TinyDB and SqlAlchemy

* Commit and Metadata timestamps changed from int to pd.Timestamp

* SqlAlchemyPersistence load_temp() and save_temp()

* New prop for Commit: local_timestamp

* Removed ParamStore ABC

* InProcessParamStore class renamed to ParamStore

* Moved ParamStore to entropylab.pipeline.params

* WIP: migrations of PS JSON 0.2->0.3

* Revert "WIP: migrations of PS JSON 0.2->0.3"

This reverts commit 217fd9b.

* Migration from ParamStore JSON 0.2 to 0.3

* Alembic migration to migrate ParamStore JSON from v0.2 to v0.3

* Linting and formatting

* Moved param_store tests to the right folder

* ParamStore now tries to read path/url from settings if not passed as args

* ParamStore now creates DB schema if url points to empty DB

* Setting up Alembic migrations for ParamStore SqlAlchemy Persistence

* Initial Alembic migration for ParamStore

* Updated README file for Alembic migrations

* ParamStore SqlAlchemyPersistence now stamps new DBs with head rev

* Linting and fixing imports

entropylab/__init__.py

@@ -1,3 +1,4 @@
+from entropylab.components.lab_topology import ExperimentResources, LabResources
 from entropylab.pipeline.api.data_reader import ExperimentReader
 from entropylab.pipeline.api.data_writer import RawResultData
 from entropylab.pipeline.api.execution import EntropyContext
@@ -8,10 +9,9 @@
     SubGraphNode,
     pynode,
 )
-from entropylab.components.lab_topology import ExperimentResources, LabResources
+from entropylab.pipeline.params.param_store import ParamStore
 from entropylab.pipeline.results_backend.sqlalchemy.db import SqlAlchemyDB
 from entropylab.pipeline.script_experiment import Script, script_experiment
-from entropylab.pipeline.api.in_process_param_store import InProcessParamStore
 from entropylab.quam.core import QuAMManager
 
 __all__ = [
@@ -28,7 +28,6 @@
     "SqlAlchemyDB",
     "Script",
     "script_experiment",
-    "InProcessParamStore",
-    "QuAM",
+    "ParamStore",
     "QuAMManager",
 ]

entropylab/config.py

@@ -2,7 +2,7 @@
 
 settings = Dynaconf(
     envvar_prefix="ENTROPY",
-    settings_files=["settings.toml", ".secrets.toml"],
+    settings_files=[".entropy/settings.toml", "settings.toml", ".secrets.toml"],
 )
 
 # `envvar_prefix` = export envvars with `export DYNACONF_FOO=bar`.

entropylab/dashboard/app.py

@@ -9,7 +9,7 @@
     theme_stylesheet,
 )
 from entropylab.logger import logger
-from entropylab.pipeline.api.in_process_param_store import InProcessParamStore
+from entropylab.pipeline.params.param_store import ParamStore
 from entropylab.pipeline.results_backend.sqlalchemy.project import (
     project_name,
     project_path,
@@ -42,7 +42,7 @@ def build_dashboard_app(proj_path):
     # noinspection PyBroadException
     try:
         param_store_file = param_store_file_path(proj_path)
-        param_store = InProcessParamStore(param_store_file)
+        param_store = ParamStore(param_store_file)
     except BaseException as e:
         logger.exception(
             f"Exception when loading ParamStore file at '{param_store_file}'"

entropylab/dashboard/pages/params/callbacks.py

@@ -1,7 +1,7 @@
 from dash import callback_context
 from dash.dependencies import Input, Output, State
 
-from entropylab.pipeline.api.param_store import ParamStore
+from entropylab import ParamStore
 from entropylab.dashboard.pages.params.utils import (
     param_store_to_df,
     param_store_to_commits_df,

entropylab/dashboard/pages/params/layout.py

@@ -14,7 +14,7 @@
     table_style_cell,
     table_active_cell_conditional,
 )
-from entropylab.pipeline.api.param_store import ParamStore
+from entropylab.pipeline.params.param_store import ParamStore
 
 REFRESH_INTERVAL_IN_MILLIS = 3000
 

entropylab/dashboard/pages/params/utils.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-from entropylab.pipeline.api.param_store import ParamStore
+from entropylab.pipeline.params.param_store import ParamStore
 
 
 def param_store_to_df(ps: ParamStore):