From 915b83444b55b28c3702d6f1035d340e52bfc390 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 8 Nov 2023 11:46:00 -0500 Subject: [PATCH] [Backport 1.6.latest] Support new agate Integer data_type in adapter code (#9004) (#9036) Add test for empty seed file with only headers (cherry picked from commit 46b9a1d621400be6204baed2680033ac0d5f1045) --- .../unreleased/Fixes-20231031-144837.yaml | 6 +++++ core/dbt/adapters/base/impl.py | 19 ++++++++++++++- core/dbt/adapters/sql/impl.py | 4 ++++ .../tests/adapter/simple_seed/test_seed.py | 23 +++++++++++++++++++ tests/unit/mock_adapter.py | 3 +++ 5 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231031-144837.yaml diff --git a/.changes/unreleased/Fixes-20231031-144837.yaml b/.changes/unreleased/Fixes-20231031-144837.yaml new file mode 100644 index 00000000000..64b15e29dc9 --- /dev/null +++ b/.changes/unreleased/Fixes-20231031-144837.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix compilation exception running empty seed file and support new Integer agate data_type +time: 2023-10-31T14:48:37.774871-04:00 +custom: + Author: gshank + Issue: "8895" diff --git a/core/dbt/adapters/base/impl.py b/core/dbt/adapters/base/impl.py index d18c9af7f50..f46cee5d017 100644 --- a/core/dbt/adapters/base/impl.py +++ b/core/dbt/adapters/base/impl.py @@ -44,7 +44,12 @@ ) from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol -from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows +from dbt.clients.agate_helper import ( + empty_table, + merge_tables, + table_from_rows, + Integer, +) from dbt.clients.jinja import MacroGenerator from dbt.contracts.graph.manifest import Manifest, MacroManifest from dbt.contracts.graph.nodes import ResultNode @@ -917,6 +922,17 @@ def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: """ raise NotImplementedError("`convert_number_type` is not implemented for this adapter!") + @classmethod + def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str: + """Return the type in the database that best maps to the agate.Number + type for the given agate table and column index. + + :param agate_table: The table + :param col_idx: The index into the agate table for the column. + :return: The name of the type in the database + """ + return "integer" + @classmethod @abc.abstractmethod def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: @@ -974,6 +990,7 @@ def convert_type(cls, agate_table: agate.Table, col_idx: int) -> Optional[str]: def convert_agate_type(cls, agate_table: agate.Table, col_idx: int) -> Optional[str]: agate_type: Type = agate_table.column_types[col_idx] conversions: List[Tuple[Type, Callable[..., str]]] = [ + (Integer, cls.convert_integer_type), (agate.Text, cls.convert_text_type), (agate.Number, cls.convert_number_type), (agate.Boolean, cls.convert_boolean_type), diff --git a/core/dbt/adapters/sql/impl.py b/core/dbt/adapters/sql/impl.py index b74eb02d991..de4c109bb54 100644 --- a/core/dbt/adapters/sql/impl.py +++ b/core/dbt/adapters/sql/impl.py @@ -75,6 +75,10 @@ def convert_number_type(cls, agate_table: agate.Table, col_idx: int) -> str: decimals = agate_table.aggregate(agate.MaxPrecision(col_idx)) # type: ignore[attr-defined] return "float8" if decimals else "integer" + @classmethod + def convert_integer_type(cls, agate_table: agate.Table, col_idx: int) -> str: + return "integer" + @classmethod def convert_boolean_type(cls, agate_table: agate.Table, col_idx: int) -> str: return "boolean" diff --git a/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py b/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py index 40b56e61efb..d4afdc7357e 100644 --- a/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py +++ b/tests/adapter/dbt/tests/adapter/simple_seed/test_seed.py @@ -285,3 +285,26 @@ def seeds(self, test_data_dir): def test_simple_seed(self, project): results = run_dbt(["seed"]) assert len(results) == 3 + + +class BaseTestEmptySeed: + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "quote_columns": False, + }, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"empty_with_header.csv": "a,b,c"} + + def test_empty_seeds(self, project): + # Should create an empty table and not fail + results = run_dbt(["seed"]) + assert len(results) == 1 + + +class TestEmptySeed(BaseTestEmptySeed): + pass diff --git a/tests/unit/mock_adapter.py b/tests/unit/mock_adapter.py index d3bdf87b2e4..8858542619b 100644 --- a/tests/unit/mock_adapter.py +++ b/tests/unit/mock_adapter.py @@ -55,6 +55,9 @@ def convert_text_type(self, *args, **kwargs): def convert_number_type(self, *args, **kwargs): return self.responder.convert_number_type(*args, **kwargs) + def convert_integer_type(self, *args, **kwargs): + return self.responder.convert_integer_type(*args, **kwargs) + def convert_boolean_type(self, *args, **kwargs): return self.responder.convert_boolean_type(*args, **kwargs)