Feature/#2 unit test (#12)

* Init branch

* Some simple tests

* Converage 67%

* Coverage 81%

* Coverage 90%

* feat: Coverage 100%

* feat: Add CI pipeline

* feat: Add CI pipeline

* feat: Add CI pipeline

* feat: Add CI  (debug)

* fix: pytest incompatible with python 3.10

* docs: add badges [skip ci]

* feat: export xml coverage report

.flake8

@@ -1,6 +1,6 @@
 [flake8]
 extend-ignore = E203, E711
-max-line-length = 88
+max-line-length = 130
 exclude =
     .venv,
     venv,

.github/workflows/ci_pr.yml

@@ -0,0 +1,48 @@
+name: CI PR
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-and-test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry config virtualenvs.in-project true
+
+      # - name: Cache the virtualenv
+      #   uses: actions/cache@v2
+      #   with:
+      #     path: ./.venv
+      #     key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        run: |
+          poetry install
+
+      - name: Code quality
+        run: |
+          poetry run poe lint
+
+      - name: Run tests
+        run: |
+          poetry run poe test-cov
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     hooks:
     - id: trailing-whitespace
     - id: end-of-file-fixer
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:
     - id: flake8

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

CONTRIBUTING.md

@@ -40,7 +40,7 @@ There are some tools that will be helpful to you in developing locally. While th
 
 ### Tools
 
-We will buy `poertry` in `dbterd` development and testing.
+We will buy `poetry` in `dbterd` development and testing.
 
 So first install poetry via pip:
 ```bash
@@ -51,6 +51,7 @@ then, start installing the local environment:
 ```bash
 python3 -m poetry install
 python3 -m poetry shell
+poe git-hooks
 pip install -e .
 dbterd -h
 ```
@@ -68,7 +69,12 @@ Once you're able to manually test that your code change is working as expected,
 Finally, you can also run a specific test or group of tests using [`pytest`](https://docs.pytest.org/en/latest/) directly. With a virtualenv active and dev dependencies installed you can do things like:
 
 ```bash
-pytest .
+poe test
+```
+
+Run test with coverage report:
+```bash
+poe test-cov
 ```
 
 > See [pytest usage docs](https://docs.pytest.org/en/6.2.x/usage.html) for an overview of useful command-line options.

README.md

@@ -5,7 +5,8 @@ CLI to generate DBML file from dbt artifact files (required: `manifest.json`, `c
 [![PyPI version](https://badge.fury.io/py/dbterd.svg)](https://pypi.org/project/dbterd/)
 ![python-cli](https://img.shields.io/badge/CLI-Python-FFCE3E?labelColor=14354C&logo=python&logoColor=white)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-
+[![python](https://img.shields.io/badge/Python-3.9|3.10|3.11-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org)
+[![codecov](https://codecov.io/gh/datnguye/dbterd/branch/main/graph/badge.svg?token=N7DMQBLH4P)](https://codecov.io/gh/datnguye/dbterd)
 
 ```
 pip install dbterd --upgrade
@@ -36,14 +37,14 @@ Commands:
 
 ## Quick examine with existing samples
 ```bash
-# select all models in dbt_resto 
+# select all models in dbt_resto
 dbterd run -ad "samples/dbtresto" -o "target"
 # select only models in dbt_resto excluding staging
 dbterd run -ad "samples/dbtresto" -o "target" -s model.dbt_resto -ns model.dbt_resto.staging
 # select only models in schema name "mart" excluding staging
 dbterd run -ad "samples/dbtresto" -o "target" -s schema:mart -ns model.dbt_resto.staging
 # select only models in schema full name "dbt.mart" excluding staging
-dbterd run -ad "samples/v4-dbtresto" -o "target" -s schema:dbt.mart -ns model.dbt_resto.staging
+dbterd run -ad "samples/dbtresto" -o "target" -s schema:dbt.mart -ns model.dbt_resto.staging
 
 # other samples
 dbterd run -ad "samples/fivetranlog" -o "target"
@@ -78,9 +79,9 @@ In your dbt project (I am using dbt-resto/[integration_tests](https://github.com
 ```bash
 dbt docs generate
 ```
-    
+
 #### 2. Generate DBML
-Copy `manifest.json` into a specific folder, and run 
+Copy `manifest.json` into a specific folder, and run
 ```
 dbterd run -mp "/path/to/dbt/target" -o "/path/to/output"
 # dbterd run -mp "./target/v4-dbtresto" -o "./target" -s model.dbt_resto -ns model.dbt_resto.staging
@@ -115,4 +116,3 @@ Result after applied Model Selection:
 If you've ever wanted to contribute to this tool, and a great cause, now is your chance!
 
 See the contributing docs [CONTRIBUTING.md](https://github.com/datnguye/dbterd/blob/main/CONTRIBUTING.md) for more information
-

dbterd/adapters/targets/dbml/engine/engine.py

@@ -70,7 +70,7 @@ def parse(manifest, catalog, **kwargs):
 
 
 def get_tables(manifest, catalog):
-
+    """Extract tables from dbt artifacts"""
     tables = [
         Table(
             name=x,
@@ -91,7 +91,8 @@ def get_tables(manifest, catalog):
             for column, metadata in cat_columns.items():
                 table.columns.append(
                     Column(
-                        name=str(column).lower(), data_type=str(metadata.type).lower()
+                        name=str(column).lower(),
+                        data_type=str(metadata.type).lower(),
                     )
                 )
 
@@ -119,6 +120,7 @@ def get_tables(manifest, catalog):
 
 
 def get_relationships(manifest):
+    """Extract relationships from dbt artifacts based on test relationship"""
     refs = [
         Ref(
             name=x,
@@ -167,12 +169,7 @@ def get_compiled_sql(manifest_node):
             {columns}
         from {table}
         """.format(
-            columns="\n".join(
-                [
-                    f"{x} as {manifest_node.columns[x].data_type or 'varchar'},"
-                    for x in manifest_node.columns
-                ]
-            ),
+            columns=",\n".join([f"{x}" for x in manifest_node.columns]),
             table=f"{manifest_node.database}.{manifest_node.schema}.undefined",
         )
 

dbterd/adapters/targets/dbml/strategies/dbml_test_relationship.py

@@ -1,5 +1,5 @@
 from dbterd.adapters.targets.dbml.engine import engine
 
 
-def run(manifest, **kwargs):
-    return ("output.dbml", engine.parse(manifest, **kwargs))
+def run(manifest, catalog, **kwargs):
+    return ("output.dbml", engine.parse(manifest, catalog, **kwargs))

dbterd/cli/main.py

@@ -1,4 +1,5 @@
 import importlib.metadata
+from typing import List
 
 import click
 
@@ -10,6 +11,24 @@
 __version__ = importlib.metadata.version("dbterd")
 
 
+# Programmatic invocation
+class dbterdRunner:
+    def __init__(self) -> None:
+        pass
+
+    def invoke(self, args: List[str]):
+        try:
+            dbt_ctx = dbterd.make_context(dbterd.name, args)
+            return dbterd.invoke(dbt_ctx)
+        except click.exceptions.Exit as e:
+            # 0 exit code, expected for --version early exit
+            if str(e) == "0":
+                return [], True
+            raise Exception(f"unhandled exit code {str(e)}")
+        except (click.NoSuchOption, click.UsageError) as e:
+            raise Exception(e.message)
+
+
 # dbterd
 @click.group(
     context_settings={"help_option_names": ["-h", "--help"]},

dbterd/cli/params.py

@@ -1,5 +1,7 @@
 import functools
+
 import click
+
 from dbterd import default
 
 
@@ -66,6 +68,6 @@ def common_params(func):
     )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
+        return func(*args, **kwargs)  # pragma: no cover
 
     return wrapper

dbterd/helpers/file.py

@@ -4,11 +4,16 @@
 
 from dbt_artifacts_parser import parser
 
-if sys.platform == "win32":
+
+def get_sys_platform():  # pragma: no cover
+    return sys.platform
+
+
+if get_sys_platform() == "win32":  # pragma: no cover
     from ctypes import WinDLL, c_bool
 else:
-    WinDLL = None
-    c_bool = None
+    WinDLL = None  # pragma: no cover
+    c_bool = None  # pragma: no cover
 
 
 def load_file_contents(path: str, strip: bool = True) -> str:
@@ -36,15 +41,15 @@ def convert_path(path: str) -> str:
     # sorry.
     if len(path) < 250:
         return path
-    if _supports_long_paths():
+    if supports_long_paths():
         return path
 
     prefix = "\\\\?\\"
     # Nothing to do
     if path.startswith(prefix):
         return path
 
-    path = _win_prepare_path(path)
+    path = win_prepare_path(path)
 
     # add the prefix. The check is just in case os.getcwd() does something
     # unexpected - I believe this if-state should always be True though!
@@ -53,8 +58,8 @@ def convert_path(path: str) -> str:
     return path
 
 
-def _supports_long_paths() -> bool:
-    if sys.platform != "win32":
+def supports_long_paths(windll_name="ntdll") -> bool:  # pragma: no cover
+    if get_sys_platform() != "win32":
         return True
     # Eryk Sun says to use `WinDLL('ntdll')` instead of `windll.ntdll` because
     # of pointer caching in a comment here:
@@ -63,18 +68,18 @@ def _supports_long_paths() -> bool:
     # he's pretty active on Python windows bugs!
     else:
         try:
-            dll = WinDLL("ntdll")
+            dll = WinDLL(windll_name)
         except OSError:  # I don't think this happens? you need ntdll to run python
             return False
         # not all windows versions have it at all
         if not hasattr(dll, "RtlAreLongPathsEnabled"):
-            return False
+            return False  # pragma: no cover
         # tell windows we want to get back a single unsigned byte (a bool).
         dll.RtlAreLongPathsEnabled.restype = c_bool
         return dll.RtlAreLongPathsEnabled()
 
 
-def _win_prepare_path(path: str) -> str:
+def win_prepare_path(path: str) -> str:  # pragma: no cover
     """Given a windows path, prepare it for use by making sure it is absolute
     and normalized.
     """
@@ -99,18 +104,17 @@ def _win_prepare_path(path: str) -> str:
     return path
 
 
-def read_manifest(manifest_path: str, manifest_version: int):
-    """Reads in the manifest file, with optional version specification"""
-    manifest_dict = open_json(f"{manifest_path}/manifest.json")
-    parser_version = (
-        f"parse_manifest_v{manifest_version}" if manifest_version else "parse_manifest"
-    )
+def read_manifest(path: str, version: int = None):
+    """Reads in the manifest.json file, with optional version specification"""
+    _dict = open_json(f"{path}/manifest.json")
+    parser_version = f"parse_manifest_v{version}" if version else "parse_manifest"
     parse_func = getattr(parser, parser_version)
-    manifest_obj = parse_func(manifest=manifest_dict)
-    return manifest_obj
+    return parse_func(manifest=_dict)
 
 
-def read_catalog(catalog_path):
-    """reads and parses the catalog file"""
-    catalog_dict = open_json(f"{catalog_path}/catalog.json")
-    return parser.parse_catalog(catalog=catalog_dict)
+def read_catalog(path: str, version: int = None):
+    """Reads in the catalog.json file, with optional version specification"""
+    _dict = open_json(f"{path}/catalog.json")
+    parser_version = f"parse_catalog_v{version}" if version else "parse_catalog"
+    parse_func = getattr(parser, parser_version)
+    return parse_func(catalog=_dict)

dbterd/helpers/jsonify.py

@@ -2,7 +2,7 @@
 import json
 
 
-class EnhancedJSONEncoder(json.JSONEncoder):
+class EnhancedJSONEncoder(json.JSONEncoder):  # pragma: no cover
     def default(self, o):
         if dataclasses.is_dataclass(o):
             return dataclasses.asdict(o)
@@ -11,21 +11,22 @@ def default(self, o):
         return super().default(o)
 
 
-def __mask(obj: str, masks: list = []):
+def mask(obj: str, mask_keys: list = ["password", "secret"]):
     obj_dict = json.loads(obj)
     for key, value in obj_dict.items():
-        if key in masks:
-            obj_dict[key] = value[0:5] + "***********"
-        if isinstance(value, dict):
-            obj_dict[key] = __mask(json.dumps(value, cls=EnhancedJSONEncoder), masks)
+        print(key)
+        if key in mask_keys or [x for x in mask_keys if key.startswith(x)]:
+            obj_dict[key] = value[0:5] + "*" * 10
+        # if isinstance(value, dict):
+        #     obj_dict[key] = mask(json.dumps(value, cls=EnhancedJSONEncoder), mask_keys)
 
     return obj_dict
 
 
-def to_json(obj, masks=[]):
+def to_json(obj, mask_keys=[]):
     if not obj:
         return {}
     mask_dict = obj
-    if "__dict__" in mask_dict:
-        mask_dict = __mask(json.dumps(obj.__dict__, cls=EnhancedJSONEncoder), masks)
+    # if isinstance(mask_dict, type):
+    #     mask_dict = mask(json.dumps(obj.__dict__, cls=EnhancedJSONEncoder), mask_keys)
     return json.dumps(mask_dict, indent=4, cls=EnhancedJSONEncoder)