Logging setup

gouline · Dec 15, 2023 · 36d4256 · 36d4256
1 parent f45e6e8
commit 36d4256
Show file tree

Hide file tree

Showing 6 changed files with 92 additions and 120 deletions.
diff --git a/dbtmetabase/__init__.py b/dbtmetabase/__init__.py
@@ -4,10 +4,12 @@
 from .dbt import DbtReader
 from .metabase import MetabaseClient
 
+logger = logging.getLogger(__name__)
+
 __all__ = ["DbtReader", "MetabaseClient"]
 
 try:
     __version__ = importlib.metadata.version("dbt-metabase")
 except importlib.metadata.PackageNotFoundError:
+    logger.warning("No version found in metadata")
     __version__ = "0.0.0-UNKONWN"
-    logging.warning("No version found in metadata")
diff --git a/dbtmetabase/__main__.py b/dbtmetabase/__main__.py
@@ -1,16 +1,58 @@
 import functools
 import logging
+from logging.handlers import RotatingFileHandler
 from pathlib import Path
 from typing import Callable, Iterable, List, Optional, Union
 
 import click
 import yaml
+from rich.logging import RichHandler
 from typing_extensions import cast
 
 from .dbt import DbtReader
-from .logger import logging as package_logger
 from .metabase import MetabaseClient
 
+LOG_PATH = Path.home().absolute() / ".dbt-metabase" / "logs" / "dbtmetabase.log"
+
+logger = logging.getLogger(__name__)
+
+
+def setup_logger(level: int = logging.INFO):
+    """Basic logger configuration for the CLI.
+
+    Args:
+        level (int, optional): Logging level. Defaults to logging.INFO.
+    """
+
+    LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    file_handler = RotatingFileHandler(
+        filename=LOG_PATH,
+        maxBytes=int(1e6),
+        backupCount=3,
+    )
+    file_handler.setFormatter(
+        logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
+    )
+    file_handler.setLevel(logging.WARNING)
+
+    rich_handler = RichHandler(
+        level=level,
+        rich_tracebacks=True,
+        markup=True,
+        show_time=False,
+    )
+
+    logging.basicConfig(
+        level=level,
+        format="%(asctime)s — %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S %z",
+        handlers=[file_handler, rich_handler],
+        force=True,
+    )
+
+
+setup_logger()
+
 
 def _comma_separated_list_callback(
     ctx: click.Context,
@@ -204,7 +246,7 @@ def wrapper(
         **kwargs,
     ):
         if verbose:
-            package_logger.LOGGING_LEVEL = logging.DEBUG
+            setup_logger(logging.DEBUG)
 
         return func(
             dbt_reader=DbtReader(

diff --git a/dbtmetabase/dbt.py b/dbtmetabase/dbt.py
@@ -1,11 +1,12 @@
 import dataclasses
 import json
+import logging
 import re
 from enum import Enum
 from pathlib import Path
 from typing import Iterable, List, Mapping, MutableMapping, Optional, Sequence
 
-from .logger.logging import logger
+logger = logging.getLogger(__name__)
 
 # Allowed metabase.* fields
 _METABASE_COMMON_META_FIELDS = [
@@ -143,17 +144,17 @@ def read_models(
             model_database = node["database"].upper()
 
             if node["resource_type"] != "model":
-                logger().debug("Skipping %s not of resource type model", model_name)
+                logger.debug("Skipping %s not of resource type model", model_name)
                 continue
 
             if node["config"]["materialized"] == "ephemeral":
-                logger().debug(
+                logger.debug(
                     "Skipping ephemeral model %s not manifested in database", model_name
                 )
                 continue
 
             if model_database != self.database:
-                logger().debug(
+                logger.debug(
                     "Skipping %s in database %s, not in target database %s",
                     model_name,
                     model_database,
@@ -162,7 +163,7 @@ def read_models(
                 continue
 
             if self.schema and model_schema != self.schema:
-                logger().debug(
+                logger.debug(
                     "Skipping %s in schema %s not in target schema %s",
                     model_name,
                     model_schema,
@@ -171,15 +172,15 @@ def read_models(
                 continue
 
             if model_schema in self.schema_excludes:
-                logger().debug(
+                logger.debug(
                     "Skipping %s in schema %s marked for exclusion",
                     model_name,
                     model_schema,
                 )
                 continue
 
             if not self.model_selected(model_name):
-                logger().debug(
+                logger.debug(
                     "Skipping %s not included in includes or excluded by excludes",
                     model_name,
                 )
@@ -202,17 +203,17 @@ def read_models(
             source_database = node["database"].upper()
 
             if node["resource_type"] != "source":
-                logger().debug("Skipping %s not of resource type source", source_name)
+                logger.debug("Skipping %s not of resource type source", source_name)
                 continue
 
             if source_database != self.database:
-                logger().debug(
+                logger.debug(
                     "Skipping %s not in target database %s", source_name, self.database
                 )
                 continue
 
             if self.schema and source_schema != self.schema:
-                logger().debug(
+                logger.debug(
                     "Skipping %s in schema %s not in target schema %s",
                     source_name,
                     source_schema,
@@ -221,15 +222,15 @@ def read_models(
                 continue
 
             if source_schema in self.schema_excludes:
-                logger().debug(
+                logger.debug(
                     "Skipping %s in schema %s marked for exclusion",
                     source_name,
                     source_schema,
                 )
                 continue
 
             if not self.model_selected(source_name):
-                logger().debug(
+                logger.debug(
                     "Skipping %s not included in includes or excluded by excludes",
                     source_name,
                 )
@@ -330,7 +331,7 @@ def _read_model_relationships(
                 # Note, sometimes only the referenced model is returned.
                 depends_on_nodes = list(child["depends_on"][model_type])
                 if len(depends_on_nodes) > 2:
-                    logger().warning(
+                    logger.warning(
                         "Expected at most two nodes, got %d {} nodes, skipping %s {}",
                         len(depends_on_nodes),
                         unique_id,
@@ -341,7 +342,7 @@ def _read_model_relationships(
                 # Otherwise, the primary key of the current model would be (incorrectly) determined to be a foreign key.
                 is_incoming_relationship_test = depends_on_nodes[1] != unique_id
                 if len(depends_on_nodes) == 2 and is_incoming_relationship_test:
-                    logger().debug(
+                    logger.debug(
                         "Skip this incoming relationship test, concerning nodes %s.",
                         depends_on_nodes,
                     )
@@ -353,7 +354,7 @@ def _read_model_relationships(
                     depends_on_nodes.remove(unique_id)
 
                 if len(depends_on_nodes) != 1:
-                    logger().warning(
+                    logger.warning(
                         "Expected single node after filtering, got %d nodes, skipping %s",
                         len(depends_on_nodes),
                         unique_id,
@@ -369,7 +370,7 @@ def _read_model_relationships(
                 )
 
                 if not fk_target_table_alias:
-                    logger().debug(
+                    logger.debug(
                         "Could not resolve depends on model id %s to a model in manifest",
                         depends_on_id,
                     )
@@ -447,7 +448,7 @@ def set_column_foreign_key(
 
         if not table or not field:
             if table or field:
-                logger().warning(
+                logger.warning(
                     "Foreign key requires table and field for column %s",
                     metabase_column.name,
                 )
@@ -462,7 +463,7 @@ def set_column_foreign_key(
             [x.strip('"').upper() for x in table_path]
         )
         metabase_column.fk_target_field = field.strip('"').upper()
-        logger().debug(
+        logger.debug(
             "Relation from %s to %s.%s",
             metabase_column.name,
             metabase_column.fk_target_table,
@@ -503,6 +504,6 @@ def parse_ref(text: str) -> Optional[str]:
         # We are catching the rightmost argument of either source or ref which is ultimately the table name
         matches = re.findall(r"['\"]([\w\_\-\ ]+)['\"][ ]*\)$", text.strip())
         if matches:
-            logger().debug("%s -> %s", text, matches[0])
+            logger.debug("%s -> %s", text, matches[0])
             return matches[0]
         return None
diff --git a/dbtmetabase/logger/__init__.py b/dbtmetabase/logger/__init__.py
diff --git a/dbtmetabase/logger/logging.py b/dbtmetabase/logger/logging.py