refactor: SettingsManager with TOML and better practices (#258)

pyproject.toml

@@ -11,15 +11,14 @@ include = ["version_constraint_key"]
 python = ">=3.10.0,<4.0.0"
 celery = "^5.1.2"
 rich = "^10.12.0"
-typer = {extras = ["all"], version = "^0.6.1"}
+typer = {extras = ["rich"], version = "^0.7.0"}
 pyfiglet = "^0.8.post1"
 redis = "^3.5.3"
 requests = "^2.26.0"
 "ruamel.yaml" = "^0.17.17"
 schema = "^0.7.5"
 deepdiff = "^5.7.0"
 pymediainfo = "^5.1.0"
-commonregex = "^1.5.3"
 yaspin = "^2.1.0"
 shortuuid = "^1.0.9"
 termynal = "^0.2.0"
@@ -30,6 +29,8 @@ click = "^8.1.3"
 shellingham = "^1.5.0.post1"
 notify-py = "^0.3.42"
 pydavinci = {git = "https://github.com/in03/pydavinci"}
+pydantic = {extras = ["dotenv"], version = "^1.10.4"}
+rtoml = "^0.9.0"
 
 [tool.poetry.group.dev.dependencies]
 isort = "^5.10.1"

requirements.txt

@@ -1,57 +1,59 @@
 amqp==5.1.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-attrs==22.1.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+attrs==22.2.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 billiard==3.6.4.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 celery==5.2.7 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-certifi==2022.6.15 ; python_full_version >= "3.10.0" and python_version < "4"
+certifi==2022.12.7 ; python_full_version >= "3.10.0" and python_version < "4"
 charset-normalizer==2.1.1 ; python_full_version >= "3.10.0" and python_version < "4"
 click-didyoumean==0.3.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 click-plugins==1.1.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 click-repl==0.2.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 click==8.1.3 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-colorama==0.4.5 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0" or python_full_version >= "3.10.0" and python_version < "4.0" and sys_platform == "win32"
+colorama==0.4.6 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0" or python_full_version >= "3.10.0" and python_version < "4.0" and sys_platform == "win32"
 commonmark==0.9.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-commonregex==1.5.3 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 contextlib2==21.6.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 deepdiff==5.8.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-exceptiongroup==1.0.0 ; python_full_version >= "3.10.0" and python_version < "3.11"
-gitdb==4.0.9 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+exceptiongroup==1.1.0 ; python_full_version >= "3.10.0" and python_version < "3.11"
+gitdb==4.0.10 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 gitpython==3.1.30 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-idna==3.3 ; python_full_version >= "3.10.0" and python_version < "4"
-iniconfig==1.1.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+idna==3.4 ; python_full_version >= "3.10.0" and python_version < "4"
+iniconfig==2.0.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 jeepney==0.8.0 ; python_full_version >= "3.10.0" and python_version < "4.0" and sys_platform == "linux"
 kombu==5.2.4 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 loguru==0.6.0 ; python_full_version >= "3.10.0" and python_version < "4.0"
-markdown==3.3.7 ; python_full_version >= "3.10.0" and python_version < "4.0"
+markdown==3.3.7 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 notify-py==0.3.42 ; python_full_version >= "3.10.0" and python_version < "4.0"
 ordered-set==4.1.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 packaging==23.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 pluggy==1.0.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-prompt-toolkit==3.0.31 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-pydantic==1.9.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+prompt-toolkit==3.0.36 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+pydantic==1.10.4 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+pydantic[dotenv]==1.10.4 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 pydavinci @ git+https://github.com/in03/pydavinci@HEAD ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 pyfiglet==0.8.post1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-pygments==2.13.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+pygments==2.14.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 pymediainfo==5.1.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-pytest==7.2.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-pytz==2022.2.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+pytest==7.2.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+python-dotenv==0.21.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+pytz==2022.7.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 redis==3.5.3 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-requests==2.28.1 ; python_full_version >= "3.10.0" and python_version < "4"
+requests==2.28.2 ; python_full_version >= "3.10.0" and python_version < "4"
 rich==10.16.2 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-ruamel-yaml-clib==0.2.6 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_full_version >= "3.10.0"
+rtoml==0.9.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+ruamel-yaml-clib==0.2.7 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_full_version >= "3.10.0"
 ruamel-yaml==0.17.21 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 schema==0.7.5 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 semver==2.13.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 shellingham==1.5.0.post1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-shortuuid==1.0.9 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+shortuuid==1.0.11 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 six==1.16.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 smmap==5.0.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-termcolor-whl==1.1.2 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-termynal==0.2.0 ; python_full_version >= "3.10.0" and python_version < "4.0"
-tomli==1.2.3 ; python_full_version >= "3.10.0" and python_version < "3.11"
-typer[all]==0.6.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-typing-extensions==4.1.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-urllib3==1.26.12 ; python_full_version >= "3.10.0" and python_version < "4"
+termcolor==2.2.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+termynal==0.2.1 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+tomli==2.0.1 ; python_full_version >= "3.10.0" and python_version < "3.11"
+typer[rich]==0.7.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+typing-extensions==4.4.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+urllib3==1.26.14 ; python_full_version >= "3.10.0" and python_version < "4"
 vine==5.0.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
-wcwidth==0.2.5 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+wcwidth==0.2.6 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
 win32-setctime==1.1.0 ; python_full_version >= "3.10.0" and sys_platform == "win32" and python_version < "4.0"
-yaspin==2.2.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"
+yaspin==2.3.0 ; python_full_version >= "3.10.0" and python_full_version < "4.0.0"

src/proxima/__init__.py

@@ -1,14 +1,37 @@
 __version__ = "1.0.5"
 
+import logging
 import os
-
 import semver
 
+from rich.logging import RichHandler
+
+
+def setup_rich_logging():
+    """Set logger to rich, allowing for console markup."""
+
+    FORMAT = "%(message)s"
+    logging.basicConfig(
+        level="WARNING",
+        format=FORMAT,
+        datefmt="[%X]",
+        handlers=[
+            RichHandler(
+                rich_tracebacks=True,
+                tracebacks_extra_lines=1,
+                markup=True,
+            )
+        ],
+    )
+
+
 _semver = semver.VersionInfo.parse(__version__)
 
 os.environ["PROXIMA_VERSION"] = __version__
 os.environ["PROXIMA_VC_KEY"] = f"{_semver.major}.{_semver.minor}"
 
+setup_rich_logging()
+
 # TODO: Fix fragile relative imports
 # This import order really matters!
 # isort is configured not to touch __init__ files

src/proxima/app/__init__.py

@@ -1 +0,0 @@
-from proxima.app.package import build_info

src/proxima/app/checks.py

@@ -10,7 +10,7 @@
 from proxima.app import core
 from proxima.app.package import build_info
 from proxima.celery import celery_app
-from proxima.settings import settings
+from proxima.settings.manager import settings
 
 core.install_rich_tracebacks()
 logger = logging.getLogger("proxima")
@@ -178,7 +178,7 @@ def status_panel(self):
         )
 
     def update_status(self):
-        if not settings["app"]["check_for_updates"]:
+        if not settings.app.check_for_updates:
             self.status_text += "[yellow]Update check disabled\n"
             return
 
@@ -194,14 +194,15 @@ def build_status(self):
         self.status_text += "[bold]\nBuild\n[/]"
 
         if build_info.is_git_repo:
-            self.status_text += f"[magenta]Git: {build_info.git_version[:7:]}[/] | "
+            if build_info.git_version:
+                self.status_text += f"[magenta]Git: {build_info.git_version[:7:]}[/] | "
 
         self.status_text += (
             f"[green]Release: {build_info.version}[/] | "
             f'[cyan]VC key: "{self.vc_key}"\n'
         )
 
-        if settings["app"]["disable_version_constrain"]:
+        if not settings.app.version_constrain:
             self.status_text += "[yellow]Version constrain is disabled! :dragon_face:\n"
 
     def worker_status(self):
@@ -232,5 +233,5 @@ def worker_status(self):
                 f"{incompatible_host_string}"
             )
 
-            if settings["app"]["disable_version_constrain"]:
+            if not settings.app.version_constrain:
                 self.status_text += "\n\n[yellow]WARNING: Jobs will be queued to incompatible workers anyway.\n"

src/proxima/app/link.py

@@ -9,7 +9,7 @@
 
 from proxima import exceptions
 from proxima.app import core
-from proxima.settings import settings
+from proxima.settings.manager import settings
 from proxima.types.job import Job
 from proxima.types.media_pool_index import media_pool_index
 
@@ -18,7 +18,7 @@
 
 core.install_rich_tracebacks()
 logger = logging.getLogger("proxima")
-logger.setLevel(settings["app"]["loglevel"])
+logger.setLevel(settings.app.loglevel)
 
 
 class ProxyLinker:

src/proxima/app/package.py

@@ -10,10 +10,10 @@
 from git.exc import InvalidGitRepositoryError
 from git.repo import Repo
 
-from proxima.settings import settings
+from proxima.settings.manager import settings
 
 logger = logging.getLogger("proxima")
-logger.setLevel(settings["app"]["loglevel"])
+logger.setLevel(settings.app.loglevel)
 
 
 def get_script_from_package(script_name: str) -> str:

src/proxima/app/resolve.py

@@ -2,14 +2,13 @@
 import os
 
 from pydavinci import davinci
-from pydavinci.wrappers.mediapool import MediaPool
 from pydavinci.wrappers.mediapoolitem import MediaPoolItem
 from pydavinci.wrappers.project import Project
 from pydavinci.wrappers.timeline import Timeline
 from pydavinci.wrappers.timelineitem import TimelineItem
 
 from proxima.app import core
-from proxima.settings import SettingsManager, settings
+from proxima.settings.manager import Settings, settings
 from proxima.types.batch import Batch
 from proxima.types.job import Job, ProjectMetadata, SourceMetadata
 from proxima.types.media_pool_index import media_pool_index
@@ -18,7 +17,7 @@
 
 core.install_rich_tracebacks()
 logger = logging.getLogger("proxima")
-logger.setLevel(settings["app"]["loglevel"])
+logger.setLevel(settings.app.loglevel)
 
 
 def get_timeline_items(timeline: Timeline) -> list[TimelineItem]:
@@ -140,21 +139,21 @@ def filter_queueable(media_pool_items: list[MediaPoolItem]) -> list[MediaPoolIte
         source_ext = os.path.splitext(source_path)[1].lower()
 
         # Filter extension
-        if settings["filters"]["extension_whitelist"]:
-            if source_ext not in settings["filters"]["extension_whitelist"]:
+        if settings.filters.extension_whitelist:
+            if source_ext not in settings.filters.extension_whitelist:
                 logger.warning(
                     f"[yellow]Ignoring file with extension not in whitelist: '{source_ext}'\n"
                     + f"from '{mpi.properties['File Path']}'[/]\n"
                 )
                 continue
 
         # Filter framerate
-        if settings["filters"]["framerate_whitelist"]:
+        if settings.filters.framerate_whitelist:
             # Make int to avoid awkward extra zeros.
             if float(mpi.properties["FPS"]).is_integer():
                 mpi.properties["FPS"] = int(float(mpi.properties["FPS"]))
 
-            if mpi.properties["FPS"] not in settings["filters"]["framerate_whitelist"]:
+            if mpi.properties["FPS"] not in settings.filters.framerate_whitelist:
                 logger.warning(
                     f"[yellow]Ignoring file with framerate not in whitelist: '{mpi.properties['FPS']}'\n"
                     + f"from '{mpi.properties['File Path']}' [/]\n"
@@ -164,9 +163,7 @@ def filter_queueable(media_pool_items: list[MediaPoolItem]) -> list[MediaPoolIte
     return media_pool_items
 
 
-def generate_batch(
-    media_pool_items: list[MediaPoolItem], settings: SettingsManager
-) -> Batch:
+def generate_batch(media_pool_items: list[MediaPoolItem], settings: Settings) -> Batch:
     logger.info("[cyan]Generating batch of jobs...")
 
     job_list = []

src/proxima/celery/celery.py

@@ -6,15 +6,15 @@
 
 from celery import Celery
 
-from proxima.settings import settings
+from proxima.settings.manager import settings
 
 # QUEUE - Celery routing queue using version constraint key
 celery_queue = os.getenv("PROXIMA_VC_KEY")
-if settings["app"]["disable_version_constrain"]:
+if not settings.app.version_constrain:
     celery_queue = "all"
 
 logger = logging.getLogger("proxima")
-logger.setLevel(settings["app"]["loglevel"])
+logger.setLevel(settings.app.loglevel)
 
 # Windows can't fork processes. It'll choke if you make it try.
 if sys.platform == "win32":
@@ -30,15 +30,17 @@
 
 # Remap terms
 broker_settings = {
-    "broker_url": settings["broker"]["url"],
-    "result_backend": settings["broker"]["url"],
-    "result_expires": settings["broker"]["result_expires"],
+    "broker_url": settings.broker.broker_url,
+    "result_backend": settings.broker.broker_url,
+    "result_expires": settings.broker.result_expires,
 }
 
 try:
     app.config_from_object(broker_settings)
 except Exception as e:
-    logger.error(f"[red]Couldn't load settings from YAML![/]\n{e}")
+    logger.error(
+        f"[red]Couldn't configure Celery with settings {broker_settings}[/]\n{e}"
+    )
 
 # Fragile! Moved from user settings to here.
 app.conf.update(

src/proxima/celery/ffmpeg/ffmpeg_process.py

@@ -14,13 +14,13 @@
 from rich.prompt import Confirm
 
 from proxima.app import core
-from proxima.settings import settings
+from proxima.settings.manager import settings
 
 from .utils import ffprobe
 
 core.install_rich_tracebacks()
 logger = logging.getLogger("proxima")
-logger.setLevel(settings["worker"]["loglevel"])
+logger.setLevel(settings.worker.loglevel)
 
 
 class FfmpegProcess:
@@ -101,7 +101,7 @@ def run(self, celery_task_object, logfile=None):
 
         # Don't continue if we don't have the process
         if not process:
-            logger.critical(f"Couldn't start ffmpeg process!")
+            logger.critical("Couldn't start ffmpeg process!")
             core.app_exit(1, -1)
 
         try:

src/proxima/celery/ffmpeg/utils.py

@@ -3,11 +3,9 @@
 import os
 import shlex
 import subprocess
-import sys
 from fractions import Fraction
 
 from proxima.app import core
-from proxima.settings import settings
 
 core.install_rich_tracebacks()
 logger = logging.getLogger("proxima")
@@ -19,8 +17,8 @@ def cleanup_working_dir(dir):
     if os.path.exists(dir):
         try:
             os.rmdir(dir)
-        except:
-            print(f"Couldn't remove {dir}. In use?")
+        except Exception as e:
+            print(f"Couldn't remove {dir}. In use?\n{e}")
             return False
     else:
         print(f"File: '{dir}' already deleted. No action taken.")