Demote field ordering failures to warning (#290)

* demote field ordering failures to warning * tests for source relationships
gouline · Nov 2, 2024 · 3b8804d · 3b8804d
1 parent e900f51
commit 3b8804d
Show file tree

Hide file tree

Showing 25 changed files with 4,567 additions and 4,051 deletions.
diff --git a/Makefile b/Makefile
@@ -69,7 +69,7 @@ sandbox-models:
 		--metabase-username $$MB_USER \
 		--metabase-password $$MB_PASSWORD \
 		--metabase-database $$POSTGRES_DB \
-		--include-schemas "pub*",other \
+		--include-schemas "pub*",inventory \
 		--http-header x-dummy-key dummy-value \
 		--order-fields \
 		--verbose )

diff --git a/dbtmetabase/_models.py b/dbtmetabase/_models.py
@@ -234,7 +234,7 @@ def __export_model(
             )
 
         if order_fields:
-            success &= self.__export_model_column_order(
+            self.__export_model_column_order(
                 ctx=ctx,
                 model=model,
                 api_table=api_table,
@@ -254,7 +254,7 @@ def __export_model_column_order(
         model: Model,
         api_table: Mapping,
         table_key: str,
-    ) -> bool:
+    ):
         """Exports model column order to Metabase field order."""
 
         api_ord = {}
@@ -295,14 +295,11 @@ def __export_model_column_order(
                 if dbt_only:
                     details.append(f"missing in Metabase {dbt_only}")
 
-                _logger.error(
+                _logger.warning(
                     "Table field order '%s' mismatch: %s",
                     table_key,
                     ", ".join(details),
                 )
-                return False
-
-        return True
 
     def __export_column(
         self,

diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml
@@ -11,6 +11,8 @@ services:
       - ${POSTGRES_PORT}:5432
     networks:
       - common
+    volumes:
+      - ./postgres-initdb:/docker-entrypoint-initdb.d
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U $POSTGRES_USER"]
       interval: 5s

diff --git a/sandbox/metabase.db/metabase.db.mv.db b/sandbox/metabase.db/metabase.db.mv.db
diff --git a/sandbox/models/staging/schema.yml b/sandbox/models/staging/schema.yml
@@ -1,5 +1,10 @@
 version: 2
 
+sources:
+  - name: inventory
+    tables:
+      - name: skus
+
 models:
   - name: stg_customers
     meta:
@@ -21,9 +26,15 @@ models:
       - name: status
         data_tests:
           - accepted_values:
-              values: ['placed', 'shipped', 'completed', 'return_pending', 'returned']
+              values: ["placed", "shipped", "completed", "return_pending", "returned"]
       - name: order_date
       - name: customer_id
+      - name: sku_id
+        data_tests:
+          - not_null
+          - relationships:
+              to: source('inventory', 'skus')
+              field: sku_id
 
   - name: stg_payments
     columns:
@@ -34,6 +45,6 @@ models:
       - name: payment_method
         data_tests:
           - accepted_values:
-              values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card']
+              values: ["credit_card", "coupon", "bank_transfer", "gift_card"]
       - name: order_id
       - name: amount
diff --git a/sandbox/models/staging/stg_orders.sql b/sandbox/models/staging/stg_orders.sql
@@ -14,7 +14,8 @@ renamed as (
         id as order_id,
         user_id as customer_id,
         order_date,
-        status
+        status,
+        1 as sku_id
 
     from source
 

diff --git a/sandbox/postgres-initdb/init.sql b/sandbox/postgres-initdb/init.sql
@@ -0,0 +1,4 @@
+CREATE SCHEMA IF NOT EXISTS inventory;
+DROP TABLE IF EXISTS inventory.skus;
+CREATE TABLE inventory.skus (sku_id INT PRIMARY KEY, product VARCHAR(50) NOT NULL);
+INSERT INTO inventory.skus (sku_id, product) VALUES (1, 'jaffle');
diff --git a/tests/fixtures/api/card/27.json b/tests/fixtures/api/card/27.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": "Orders and customers",
     "archived": false,
-    "view_count": 430,
+    "view_count": 446,
     "collection_position": null,
     "table_id": 10,
     "can_run_adhoc_query": true,
@@ -561,7 +561,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,

diff --git a/tests/fixtures/api/card/28.json b/tests/fixtures/api/card/28.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": null,
     "archived": false,
-    "view_count": 215,
+    "view_count": 223,
     "collection_position": null,
     "table_id": 10,
     "can_run_adhoc_query": true,
@@ -549,7 +549,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,

diff --git a/tests/fixtures/api/card/29.json b/tests/fixtures/api/card/29.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": null,
     "archived": false,
-    "view_count": 214,
+    "view_count": 222,
     "collection_position": null,
     "table_id": null,
     "can_run_adhoc_query": true,
@@ -41,7 +41,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,

diff --git a/tests/fixtures/api/card/30.json b/tests/fixtures/api/card/30.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": "Dummy 1",
     "archived": false,
-    "view_count": 39,
+    "view_count": 43,
     "collection_position": null,
     "table_id": null,
     "can_run_adhoc_query": true,
@@ -41,7 +41,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,

diff --git a/tests/fixtures/api/card/31.json b/tests/fixtures/api/card/31.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": "Dummy 2",
     "archived": false,
-    "view_count": 39,
+    "view_count": 43,
     "collection_position": null,
     "table_id": null,
     "can_run_adhoc_query": true,
@@ -41,7 +41,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,

diff --git a/tests/fixtures/api/card/32.json b/tests/fixtures/api/card/32.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": "CTE SQL",
     "archived": false,
-    "view_count": 30,
+    "view_count": 34,
     "collection_position": null,
     "table_id": null,
     "can_run_adhoc_query": true,
@@ -276,7 +276,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,

diff --git a/tests/fixtures/api/card/33.json b/tests/fixtures/api/card/33.json
@@ -2,7 +2,7 @@
     "cache_invalidated_at": null,
     "description": null,
     "archived": false,
-    "view_count": 11,
+    "view_count": 15,
     "collection_position": null,
     "table_id": 18,
     "can_run_adhoc_query": true,
@@ -146,7 +146,7 @@
     "creator": {
         "email": "[email protected]",
         "first_name": "dbtmetabase",
-        "last_login": "2024-10-12T03:43:50.753203Z",
+        "last_login": "2024-11-02T04:41:42.402168Z",
         "is_qbnewb": false,
         "is_superuser": true,
         "id": 1,
@@ -161,11 +161,11 @@
     "collection_id": null,
     "query_type": "query",
     "name": "Transactions",
-    "last_query_start": "2024-10-12T01:52:30.557471Z",
+    "last_query_start": "2024-10-12T03:53:32.250619Z",
     "dashboard_count": 0,
     "last_used_at": "2024-10-12T03:43:09.869884Z",
     "type": "question",
-    "average_query_time": 58.0,
+    "average_query_time": 114.5,
     "creator_id": 1,
     "moderation_reviews": [],
     "updated_at": "2024-10-12T01:52:30.606845Z",

diff --git a/tests/fixtures/api/dashboard/2.json b/tests/fixtures/api/dashboard/2.json
@@ -1,7 +1,7 @@
 {
     "description": "Dashboard is a dashboard is a dashboard",
     "archived": false,
-    "view_count": 114,
+    "view_count": 118,
     "collection_position": null,
     "dashcards": [
         {
@@ -14,7 +14,7 @@
                 "cache_invalidated_at": null,
                 "description": "Orders and customers",
                 "archived": false,
-                "view_count": 430,
+                "view_count": 446,
                 "collection_position": null,
                 "table_id": 10,
                 "result_metadata": [
@@ -653,7 +653,7 @@
                 "cache_invalidated_at": null,
                 "description": null,
                 "archived": false,
-                "view_count": 215,
+                "view_count": 223,
                 "collection_position": null,
                 "table_id": 10,
                 "result_metadata": [
@@ -1266,7 +1266,7 @@
                 "cache_invalidated_at": null,
                 "description": null,
                 "archived": false,
-                "view_count": 214,
+                "view_count": 222,
                 "collection_position": null,
                 "table_id": null,
                 "result_metadata": [

diff --git a/tests/fixtures/api/database.json b/tests/fixtures/api/database.json
@@ -55,7 +55,7 @@
         "caveats": null,
         "creator_id": 1,
         "is_full_sync": true,
-        "updated_at": "2024-10-12T03:43:24.194904Z",
+        "updated_at": "2024-11-02T04:30:07.348325Z",
         "native_permissions": "write",
         "cache_ttl": null,
         "details": {