feat: soft deps + catalog option (#46)

* feat: top pin dbt-artifiacts-parser to < 1.0.0 and better warning message * test: improve cov * docs: Update readme * docs: add catalog option * fix: pin poetry 1.4 in PRCI
datnguye · Sep 2, 2023 · 2cb01ee · 2cb01ee
1 parent 7aed610
commit 2cb01ee
Show file tree

Hide file tree

Showing 11 changed files with 363 additions and 73 deletions.
diff --git a/.github/workflows/ci_pr.yml b/.github/workflows/ci_pr.yml
@@ -21,7 +21,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install poetry
+          pip install "poetry~=1.4.0"
           poetry config virtualenvs.in-project true
 
       - name: Cache the virtualenv

diff --git a/README.md b/README.md
@@ -13,6 +13,7 @@ pip install dbterd --upgrade
 ```
 
 Verify installation:
+
 ```bash
 dbterd --version
 ```

diff --git a/dbterd/adapters/base.py b/dbterd/adapters/base.py
@@ -37,20 +37,21 @@ def __read_manifest(self, mp: str, mv: int = None):
         cli_messaging.check_existence(mp, self.filename_manifest)
         conditional = f" or provided version {mv} is incorrect" if mv else ""
         with cli_messaging.handle_read_errors(self.filename_manifest, conditional):
-            return file_handlers.read_manifest(mp, mv)
+            return file_handlers.read_manifest(path=mp, version=mv)
 
-    def __read_catalog(self, cp: str):
+    def __read_catalog(self, cp: str, cv: int = None):
         """Read the Catalog content
 
         Args:
             cp (str): catalog.json file path
+            cv (int, optional): Catalog version. Defaults to None.
 
         Returns:
             dict: Catalog dict
         """
         cli_messaging.check_existence(cp, self.filename_catalog)
         with cli_messaging.handle_read_errors(self.filename_catalog):
-            return file_handlers.read_catalog(cp)
+            return file_handlers.read_catalog(path=cp, version=cv)
 
     def __run_by_strategy(self, **kwargs):
         """Read artifacts and export the diagram file following the target"""
@@ -66,7 +67,8 @@ def __run_by_strategy(self, **kwargs):
             mv=kwargs["manifest_version"],
         )
         catalog = self.__read_catalog(
-            cp=kwargs.get("manifest_path") or kwargs["artifacts_dir"]
+            cp=kwargs.get("manifest_path") or kwargs["artifacts_dir"],
+            cv=kwargs["catalog_version"],
         )
 
         result = operation(manifest=manifest, catalog=catalog, **kwargs)

diff --git a/dbterd/cli/params.py b/dbterd/cli/params.py
@@ -61,6 +61,13 @@ def common_params(func):
         default=None,
         type=click.STRING,
     )
+    @click.option(
+        "--catalog-version",
+        "-cv",
+        help="Specified dbt catalog.json version",
+        default=None,
+        type=click.STRING,
+    )
     @click.option(
         "--resource-type",
         "-rt",

diff --git a/dbterd/helpers/file.py b/dbterd/helpers/file.py
@@ -4,6 +4,8 @@
 
 from dbt_artifacts_parser import parser
 
+from dbterd.helpers.log import logger
+
 
 def get_sys_platform():  # pragma: no cover
     return sys.platform
@@ -118,7 +120,16 @@ def read_manifest(path: str, version: int = None):
         dict: Manifest dict
     """
     _dict = open_json(f"{path}/manifest.json")
-    parser_version = f"parse_manifest_v{version}" if version else "parse_manifest"
+    default_parser = "parse_manifest"
+    parser_version = f"parse_manifest_v{version}" if version else default_parser
+    if not hasattr(parser, parser_version):
+        logger.warning(
+            "Manifest version is NOT SUPPORTED in current `dbt-artifacts-parser` package. \n"
+            "Please help to try `-mv {version}` option with other value, OR upgrade the package:\n"
+            "\tpip install dbt-artifacts-parser --upgrade\n"
+            "Try falling back to the latest one..."
+        )
+        parser_version = default_parser
     parse_func = getattr(parser, parser_version)
     return parse_func(manifest=_dict)
 
@@ -134,6 +145,15 @@ def read_catalog(path: str, version: int = None):
         dict: Catalog dict
     """
     _dict = open_json(f"{path}/catalog.json")
-    parser_version = f"parse_catalog_v{version}" if version else "parse_catalog"
+    default_parser = "parse_catalog"
+    parser_version = f"parse_catalog_v{version}" if version else default_parser
+    if not hasattr(parser, parser_version):
+        logger.warning(
+            "Catalog version is NOT SUPPORTED in current `dbt-artifacts-parser` package. \n"
+            "Please help to try `-mv {version}` option with other value, OR upgrade the package:\n"
+            "\tpip install dbt-artifacts-parser --upgrade\n"
+            "Try falling back to the latest one..."
+        )
+        parser_version = default_parser
     parse_func = getattr(parser, parser_version)
     return parse_func(catalog=_dict)
diff --git a/docs/index.md b/docs/index.md
@@ -1,6 +1,6 @@
 # dbterd
 
-CLI to generate Diagram-as-a-code file ([DBML](https://dbdiagram.io/d), [Mermaid](https://mermaid-js.github.io/mermaid-live-editor/), [PlantUML](https://plantuml.com/ie-diagram), [GraphViz](https://graphviz.org/), [D2](https://d2lang.com/)) from dbt artifact files (required: [![dbt](https://img.shields.io/badge/manifest.json-upto--v9-3776AB.svg?style=flat&logo=dbt&logoColor=orange)](https://schemas.getdbt.com/) [![dbt](https://img.shields.io/badge/catalog.json-upto--v1-3776AB.svg?style=flat&logo=dbt&logoColor=orange)](https://schemas.getdbt.com/))
+CLI to generate Diagram-as-a-code file ([DBML](https://dbdiagram.io/d), [Mermaid](https://mermaid-js.github.io/mermaid-live-editor/), [PlantUML](https://plantuml.com/ie-diagram), [GraphViz](https://graphviz.org/), [D2](https://d2lang.com/)) from dbt artifact files (required: [![dbt](https://img.shields.io/badge/manifest.json-upto--v10-3776AB.svg?style=flat&logo=dbt&logoColor=orange)](https://schemas.getdbt.com/) [![dbt](https://img.shields.io/badge/catalog.json-upto--v1-3776AB.svg?style=flat&logo=dbt&logoColor=orange)](https://schemas.getdbt.com/))
 
 [![PyPI version](https://badge.fury.io/py/dbterd.svg)](https://pypi.org/project/dbterd/)
 ![python-cli](https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white)
@@ -15,42 +15,46 @@ CLI to generate Diagram-as-a-code file ([DBML](https://dbdiagram.io/d), [Mermaid
     <a href="#" data-terminal-control="">restart ↻</a>
 </div>
 
-Verify installed version:
+Verify installation:
 
 ```bash
 dbterd --version
 ```
 
 ## Quick examine with existing samples
 
-```bash
-# select all models in dbt_resto
-dbterd run -ad samples/dbtresto -o target
-# select all models in dbt_resto, Select multiple dbt resources
-dbterd run -ad samples/dbtresto -o target -rt model -rt source
-# select only models in dbt_resto excluding staging
-dbterd run -ad samples/dbtresto -o target -s model.dbt_resto -ns model.dbt_resto.staging
-# select only models in schema name mart excluding staging
-dbterd run -ad samples/dbtresto -o target -s schema:mart -ns model.dbt_resto.staging
-# select only models in schema full name dbt.mart excluding staging
-dbterd run -ad samples/dbtresto -o target -s schema:dbt.mart -ns model.dbt_resto.staging
-
-# other samples
-dbterd run -ad samples/fivetranlog -o target
-dbterd run -ad samples/fivetranlog -o target -rt model -rt source
-
-dbterd run -ad samples/facebookad -o target
-dbterd run -ad samples/facebookad -o target -rt model -rt source
-
-dbterd run -ad samples/shopify -o target
-dbterd run -ad samples/shopify -o target -rt model -rt source
-
-dbterd run -ad samples/dbt-constraints \
-    -a "test_relationship:(name:foreign_key|c_from:fk_column_name|c_to:pk_column_name)"
-
-# your own sample without commiting to repo
-dbterd run -ad samples/local -o target -rt model -rt source
-```
+<details>
+  <summary>Click me</summary>
+
+  ```bash
+  # select all models in dbt_resto
+  dbterd run -ad samples/dbtresto
+  # select all models in dbt_resto, Select multiple dbt resources
+  dbterd run -ad samples/dbtresto -rt model -rt source
+  # select only models in dbt_resto excluding staging
+  dbterd run -ad samples/dbtresto -s model.dbt_resto -ns model.dbt_resto.staging
+  # select only models in schema name mart excluding staging
+  dbterd run -ad samples/dbtresto -s schema:mart -ns model.dbt_resto.staging
+  # select only models in schema full name dbt.mart excluding staging
+  dbterd run -ad samples/dbtresto -s schema:dbt.mart -ns model.dbt_resto.staging
+
+  # other samples
+  dbterd run -ad samples/fivetranlog
+  dbterd run -ad samples/fivetranlog -rt model -rt source
+
+  dbterd run -ad samples/facebookad
+  dbterd run -ad samples/facebookad -rt model -rt source
+
+  dbterd run -ad samples/shopify -s wildcard:*shopify.shopify__*
+  dbterd run -ad samples/shopify -rt model -rt source
+
+  dbterd run -ad samples/dbt-constraints -a "test_relationship:(name:foreign_key|c_from:fk_column_name|c_to:pk_column_name)"
+
+  # your own sample without commiting to repo
+  dbterd run -ad samples/local -rt model -rt source
+  ```
+
+</details>
 
 ## Quick DEMO
 

diff --git a/docs/nav/guide/cli-references.md b/docs/nav/guide/cli-references.md
@@ -51,6 +51,7 @@ Command to generate diagram-as-a-code file
                                         diagram connectors  [default:
                                         test_relationship]
         -mv, --manifest-version TEXT  Specified dbt manifest.json version
+        -cv, --catalog-version TEXT   Specified dbt catalog.json version
         -rt, --resource-type TEXT     Specified dbt resource type(seed, model,
                                         source, snapshot),default:model, use examples,
                                         -rt model -rt source
@@ -229,6 +230,23 @@ Specified dbt manifest.json version
     dbterd run -mv 7
     ```
 
+### --catalog-version (-cv)
+
+Specified dbt catalog.json version
+> Auto detect if not specified
+
+**Examples:**
+=== "CLI"
+
+    ```bash
+    dbterd run --catalog-version 7
+    ```
+=== "CLI (long style)"
+
+    ```bash
+    dbterd run -cv 7
+    ```
+
 ### --resource-type (-rt)
 
 Specified dbt resource type(seed, model, source, snapshot).