Skip to content

Commit

Permalink
Refactor core (#212)
Browse files Browse the repository at this point in the history
* Renames, regroups and cleanup

* Extract API calls and refactor exposure building

* More unit tests
  • Loading branch information
gouline authored Jan 24, 2024
1 parent a220f0d commit 8be7acb
Show file tree
Hide file tree
Showing 40 changed files with 1,288 additions and 1,373 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -276,19 +276,19 @@ Alternatively, you can invoke dbt-metabase programmatically. Below is the equiva
```python
from dbtmetabase import DbtMetabase
# Initializing client instance
client = DbtMetabase(
# Initializing instance
c = DbtMetabase(
manifest_path="target/manifest.json",
metabase_url="https://metabase.example.com",
metabase_username="[email protected]",
metabase_password="Password123",
)
# Exporting models
client.export_models(metabase_database="business")
c.export_models(metabase_database="business")
# Extracting exposures
client.extract_exposures(output_path=".")
c.extract_exposures(output_path=".")
```

See function header comments for information about other parameters.
Expand Down
2 changes: 1 addition & 1 deletion dbtmetabase/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import logging

from .core import DbtMetabase
from .interface import Filter, MetabaseArgumentError, MetabaseRuntimeError
from .format import Filter

__all__ = [
"DbtMetabase",
Expand Down
67 changes: 52 additions & 15 deletions dbtmetabase/__main__.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,54 @@
import functools
import logging
from pathlib import Path
from typing import Callable, Optional, Sequence
from typing import Any, Callable, List, Mapping, Optional, Sequence, Union

import click
import yaml
from typing_extensions import cast

from ._format import click_list_option_kwargs, setup_logging
from .core import DbtMetabase
from .core_exposures import ExposuresExtractorMixin
from .core_models import ModelsExporterMixin
from .interface import Filter
from .format import Filter, setup_logging


def _click_list_option_kwargs() -> Mapping[str, Any]:
"""Click option that accepts comma-separated values.
Built-in list only allows repeated flags, which is ugly for larger lists.
Returns:
Mapping[str, Any]: Mapping of kwargs (to be unpacked with **).
"""

def callback(
ctx: click.Context,
param: click.Option,
value: Union[str, List[str]],
) -> Optional[List[str]]:
if value is None:
return None

if ctx.get_parameter_source(str(param.name)) in (
click.core.ParameterSource.DEFAULT,
click.core.ParameterSource.DEFAULT_MAP,
) and isinstance(value, list):
# Lists in defaults (config or option) should be lists
return value

elif isinstance(value, str):
str_value = value
if isinstance(value, list):
# When type=list, string value will be a list of chars
str_value = "".join(value)
else:
raise click.BadParameter("must be comma-separated list")

return str_value.split(",")

return {
"type": click.UNPROCESSED,
"callback": callback,
}


@click.group()
Expand Down Expand Up @@ -160,55 +197,55 @@ def wrapper(
metavar="DATABASES",
envvar="INCLUDE_DATABASES",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Only include certain dbt databases.",
)
@click.option(
"--exclude-databases",
metavar="DATABASES",
envvar="EXCLUDE_DATABASES",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Exclude certain dbt databases.",
)
@click.option(
"--include-schemas",
metavar="SCHEMAS",
envvar="INCLUDE_SCHEMAS",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Only include certain dbt schemas.",
)
@click.option(
"--exclude-schemas",
metavar="SCHEMAS",
envvar="EXCLUDE_SCHEMAS",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Exclude certain dbt schemas.",
)
@click.option(
"--include-models",
metavar="MODELS",
envvar="INCLUDE_MODELS",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Only include certain dbt models.",
)
@click.option(
"--exclude-models",
metavar="MODELS",
envvar="EXCLUDE_MODELS",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Exclude certain dbt models.",
)
@click.option(
"--sync-timeout",
metavar="SECS",
envvar="SYNC_TIMEOUT",
show_envvar=True,
default=ModelsExporterMixin.DEFAULT_SYNC_TIMEOUT,
default=DbtMetabase.DEFAULT_MODELS_SYNC_TIMEOUT,
type=click.INT,
help="Number of seconds to wait until Metabase schema matches the dbt project. To skip synchronization, set timeout to 0.",
)
Expand Down Expand Up @@ -267,7 +304,7 @@ def models(
envvar="OUTPUT_PATH",
show_envvar=True,
type=click.Path(exists=True, file_okay=False),
default=ExposuresExtractorMixin.DEFAULT_OUTPUT_PATH,
default=DbtMetabase.DEFAULT_EXPOSURES_OUTPUT_PATH,
show_default=True,
help="Output path for exposure YAML files.",
)
Expand All @@ -283,15 +320,15 @@ def models(
metavar="COLLECTIONS",
envvar="INCLUDE_COLLECTIONS",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Only include certain Metabase collections.",
)
@click.option(
"--exclude-collections",
metavar="COLLECTIONS",
envvar="EXCLUDE_COLLECTIONS",
show_envvar=True,
**click_list_option_kwargs(),
**_click_list_option_kwargs(),
help="Exclude certain Metabase collections.",
)
@click.option(
Expand Down
Loading

0 comments on commit 8be7acb

Please sign in to comment.