Skip to content

Commit

Permalink
deprecate materialization overrides from imported packages (#9971) (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
MichelleArk authored May 1, 2024
1 parent ce483b6 commit 3c82a02
Show file tree
Hide file tree
Showing 39 changed files with 1,737 additions and 1,514 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20231218-195854.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Move flags from UserConfig in profiles.yml to flags in dbt_project.yml
time: 2023-12-18T19:58:54.075811-05:00
custom:
Author: gshank
Issue: "9183"
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240422-173703.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Add require_explicit_package_overrides_for_builtin_materializations to dbt_project.yml flags, which can be used to opt-out of overriding built-in materializations from packages
time: 2024-04-22T17:37:03.892268-04:00
custom:
Author: michelleark
Issue: "10007"
6 changes: 6 additions & 0 deletions .changes/unreleased/Under the Hood-20240418-172528.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Under the Hood
body: Raise deprecation warning if installed package overrides built-in materialization
time: 2024-04-18T17:25:28.37886-04:00
custom:
Author: michelleark
Issue: "9971"
43 changes: 30 additions & 13 deletions core/dbt/cli/flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from dataclasses import dataclass
from importlib import import_module
from multiprocessing import get_context
from pathlib import Path
from pprint import pformat as pf
from typing import Any, Callable, Dict, List, Optional, Set, Union

Expand All @@ -11,8 +12,8 @@
from dbt.cli.exceptions import DbtUsageException
from dbt.cli.resolvers import default_log_path, default_project_dir
from dbt.cli.types import Command as CliCommand
from dbt.config.profile import read_user_config
from dbt.contracts.project import UserConfig
from dbt.config.project import read_project_flags
from dbt.contracts.project import ProjectFlags
from dbt.exceptions import DbtInternalError
from dbt.deprecations import renamed_env_var
from dbt.helper_types import WarnErrorOptions
Expand All @@ -24,7 +25,8 @@
FLAGS_DEFAULTS = {
"INDIRECT_SELECTION": "eager",
"TARGET_PATH": None,
# Cli args without user_config or env var option.
"WARN_ERROR": None,
# Cli args without project_flags or env var option.
"FULL_REFRESH": False,
"STRICT_MODE": False,
"STORE_FAILURES": False,
Expand Down Expand Up @@ -76,7 +78,7 @@ class Flags:
"""Primary configuration artifact for running dbt"""

def __init__(
self, ctx: Optional[Context] = None, user_config: Optional[UserConfig] = None
self, ctx: Optional[Context] = None, project_flags: Optional[ProjectFlags] = None
) -> None:

# Set the default flags.
Expand Down Expand Up @@ -201,27 +203,40 @@ def _assign_params(
invoked_subcommand_ctx, params_assigned_from_default, deprecated_env_vars
)

if not user_config:
if not project_flags:
project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir()))
profiles_dir = getattr(self, "PROFILES_DIR", None)
user_config = read_user_config(profiles_dir) if profiles_dir else None
if profiles_dir and project_dir:
project_flags = read_project_flags(project_dir, profiles_dir)
else:
project_flags = None

# Add entire invocation command to flags
object.__setattr__(self, "INVOCATION_COMMAND", "dbt " + " ".join(sys.argv[1:]))

# Overwrite default assignments with user config if available.
if user_config:
if project_flags:
# Overwrite default assignments with project flags if available.
param_assigned_from_default_copy = params_assigned_from_default.copy()
for param_assigned_from_default in params_assigned_from_default:
user_config_param_value = getattr(user_config, param_assigned_from_default, None)
if user_config_param_value is not None:
project_flags_param_value = getattr(
project_flags, param_assigned_from_default, None
)
if project_flags_param_value is not None:
object.__setattr__(
self,
param_assigned_from_default.upper(),
convert_config(param_assigned_from_default, user_config_param_value),
convert_config(param_assigned_from_default, project_flags_param_value),
)
param_assigned_from_default_copy.remove(param_assigned_from_default)
params_assigned_from_default = param_assigned_from_default_copy

# Add project-level flags that are not available as CLI options / env vars
for (
project_level_flag_name,
project_level_flag_value,
) in project_flags.project_only_flags.items():
object.__setattr__(self, project_level_flag_name.upper(), project_level_flag_value)

# Set hard coded flags.
object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name)
object.__setattr__(self, "MP_CONTEXT", get_context("spawn"))
Expand All @@ -235,9 +250,11 @@ def _assign_params(
# Starting in v1.5, if `log-path` is set in `dbt_project.yml`, it will raise a deprecation warning,
# with the possibility of removing it in a future release.
if getattr(self, "LOG_PATH", None) is None:
project_dir = getattr(self, "PROJECT_DIR", default_project_dir())
project_dir = getattr(self, "PROJECT_DIR", str(default_project_dir()))
version_check = getattr(self, "VERSION_CHECK", True)
object.__setattr__(self, "LOG_PATH", default_log_path(project_dir, version_check))
object.__setattr__(
self, "LOG_PATH", default_log_path(Path(project_dir), version_check)
)

# Support console DO NOT TRACK initiative.
if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"):
Expand Down
2 changes: 1 addition & 1 deletion core/dbt/config/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# all these are just exports, they need "noqa" so flake8 will not complain.
from .profile import Profile, read_user_config # noqa
from .profile import Profile # noqa
from .project import Project, IsFQNResource, PartialProject # noqa
from .runtime import RuntimeConfig # noqa
39 changes: 1 addition & 38 deletions core/dbt/config/profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from dbt.clients.system import load_file_contents
from dbt.clients.yaml_helper import load_yaml_text
from dbt.contracts.connection import Credentials, HasCredentials
from dbt.contracts.project import ProfileConfig, UserConfig
from dbt.contracts.project import ProfileConfig
from dbt.exceptions import (
CompilationError,
DbtProfileError,
Expand All @@ -19,7 +19,6 @@
)
from dbt.events.types import MissingProfileTarget
from dbt.events.functions import fire_event
from dbt.utils import coerce_dict_str

from .renderer import ProfileRenderer

Expand Down Expand Up @@ -51,27 +50,13 @@ def read_profile(profiles_dir: str) -> Dict[str, Any]:
return {}


def read_user_config(directory: str) -> UserConfig:
try:
profile = read_profile(directory)
if profile:
user_config = coerce_dict_str(profile.get("config", {}))
if user_config is not None:
UserConfig.validate(user_config)
return UserConfig.from_dict(user_config)
except (DbtRuntimeError, ValidationError):
pass
return UserConfig()


# The Profile class is included in RuntimeConfig, so any attribute
# additions must also be set where the RuntimeConfig class is created
# `init=False` is a workaround for https://bugs.python.org/issue45081
@dataclass(init=False)
class Profile(HasCredentials):
profile_name: str
target_name: str
user_config: UserConfig
threads: int
credentials: Credentials
profile_env_vars: Dict[str, Any]
Expand All @@ -80,7 +65,6 @@ def __init__(
self,
profile_name: str,
target_name: str,
user_config: UserConfig,
threads: int,
credentials: Credentials,
):
Expand All @@ -89,7 +73,6 @@ def __init__(
"""
self.profile_name = profile_name
self.target_name = target_name
self.user_config = user_config
self.threads = threads
self.credentials = credentials
self.profile_env_vars = {} # never available on init
Expand All @@ -106,12 +89,10 @@ def to_profile_info(self, serialize_credentials: bool = False) -> Dict[str, Any]
result = {
"profile_name": self.profile_name,
"target_name": self.target_name,
"user_config": self.user_config,
"threads": self.threads,
"credentials": self.credentials,
}
if serialize_credentials:
result["user_config"] = self.user_config.to_dict(omit_none=True)
result["credentials"] = self.credentials.to_dict(omit_none=True)
return result

Expand All @@ -124,7 +105,6 @@ def to_target_dict(self) -> Dict[str, Any]:
"name": self.target_name,
"target_name": self.target_name,
"profile_name": self.profile_name,
"config": self.user_config.to_dict(omit_none=True),
}
)
return target
Expand Down Expand Up @@ -246,7 +226,6 @@ def from_credentials(
threads: int,
profile_name: str,
target_name: str,
user_config: Optional[Dict[str, Any]] = None,
) -> "Profile":
"""Create a profile from an existing set of Credentials and the
remaining information.
Expand All @@ -255,20 +234,13 @@ def from_credentials(
:param threads: The number of threads to use for connections.
:param profile_name: The profile name used for this profile.
:param target_name: The target name used for this profile.
:param user_config: The user-level config block from the
raw profiles, if specified.
:raises DbtProfileError: If the profile is invalid.
:returns: The new Profile object.
"""
if user_config is None:
user_config = {}
UserConfig.validate(user_config)
user_config_obj: UserConfig = UserConfig.from_dict(user_config)

profile = cls(
profile_name=profile_name,
target_name=target_name,
user_config=user_config_obj,
threads=threads,
credentials=credentials,
)
Expand Down Expand Up @@ -316,7 +288,6 @@ def from_raw_profile_info(
raw_profile: Dict[str, Any],
profile_name: str,
renderer: ProfileRenderer,
user_config: Optional[Dict[str, Any]] = None,
target_override: Optional[str] = None,
threads_override: Optional[int] = None,
) -> "Profile":
Expand All @@ -328,8 +299,6 @@ def from_raw_profile_info(
disk as yaml and its values rendered with jinja.
:param profile_name: The profile name used.
:param renderer: The config renderer.
:param user_config: The global config for the user, if it
was present.
:param target_override: The target to use, if provided on
the command line.
:param threads_override: The thread count to use, if
Expand All @@ -338,9 +307,6 @@ def from_raw_profile_info(
target could not be found
:returns: The new Profile object.
"""
# user_config is not rendered.
if user_config is None:
user_config = raw_profile.get("config")
# TODO: should it be, and the values coerced to bool?
target_name, profile_data = cls.render_profile(
raw_profile, profile_name, target_override, renderer
Expand All @@ -361,7 +327,6 @@ def from_raw_profile_info(
profile_name=profile_name,
target_name=target_name,
threads=threads,
user_config=user_config,
)

@classmethod
Expand Down Expand Up @@ -396,13 +361,11 @@ def from_raw_profiles(
if not raw_profile:
msg = f"Profile {profile_name} in profiles.yml is empty"
raise DbtProfileError(INVALID_PROFILE_MESSAGE.format(error_string=msg))
user_config = raw_profiles.get("config")

return cls.from_raw_profile_info(
raw_profile=raw_profile,
profile_name=profile_name,
renderer=renderer,
user_config=user_config,
target_override=target_override,
threads_override=threads_override,
)
Expand Down
Loading

0 comments on commit 3c82a02

Please sign in to comment.