API response recorder for test fixtures (#261)

* API response recorder for test fixtures * Tracked H2 app database with idempotent setup * New reference exposures and resulting fixtures and test fixes * Clean up tests and remove stale fixtures
gouline · Jun 20, 2024 · 1e6d0f9 · 1e6d0f9
1 parent adf4dca
commit 1e6d0f9
Show file tree

Hide file tree

Showing 120 changed files with 4,620 additions and 15,252 deletions.
diff --git a/Makefile b/Makefile
@@ -43,6 +43,7 @@ check: check-fmt check-imports check-lint-python check-type
 .PHONY: check
 
 test:
+	rm -rf tests/tmp
 	python3 -m unittest tests
 .PHONY: test
 

diff --git a/dbtmetabase/_exposures.py b/dbtmetabase/_exposures.py
@@ -165,8 +165,6 @@ def extract_exposures(
                 count = counts.get(name, 0)
                 counts[name] = count + 1
 
-                _logger.error(ctx.model_refs)
-
                 exposures.append(
                     {
                         "id": item["id"],
@@ -183,11 +181,13 @@ def extract_exposures(
                             creator_name=creator_name or "",
                             creator_email=creator_email or "",
                             native_query=native_query,
-                            depends_on={
-                                ctx.model_refs[depend.lower()]
-                                for depend in depends
-                                if depend.lower() in ctx.model_refs
-                            },
+                            depends_on=sorted(
+                                [
+                                    ctx.model_refs[depend.lower()]
+                                    for depend in depends
+                                    if depend.lower() in ctx.model_refs
+                                ]
+                            ),
                         ),
                     }
                 )

diff --git a/requirements-test.txt b/requirements-test.txt
@@ -7,5 +7,6 @@ pylint>=3.0.2
 mypy>=1.7.1
 molot~=1.0.0
 dbt-postgres~=1.8.1
+python-dotenv~=1.0.1
 types-requests
 types-PyYAML
diff --git a/sandbox/.gitignore b/sandbox/.gitignore
@@ -0,0 +1 @@
+/metabase/metabase.db.trace.db
diff --git a/sandbox/docker-compose.yml b/sandbox/docker-compose.yml
@@ -24,13 +24,16 @@ services:
     build:
       dockerfile: Dockerfile-metabase
       args:
-        - MB_VERSION=0.50.3
+        - MB_VERSION=0.50.5
     environment:
       - MB_SETUP_TOKEN=${MB_SETUP_TOKEN:-}
+      - MB_DB_FILE=/metabase-data/metabase.db
     ports:
       - ${MB_PORT}:3000
     networks:
       - common
+    volumes:
+      - ./metabase:/metabase-data
     healthcheck:
       test: ["CMD-SHELL", "curl --fail http://localhost:3000/api/health"]
       interval: 10s

diff --git a/sandbox/entrypoint.py b/sandbox/entrypoint.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import time
+import logging
 
 import requests
 from molot import envarg, envarg_int, evaluate, shell, target
@@ -16,10 +16,12 @@
 MB_PASSWORD = envarg("MB_PASSWORD")
 MB_NAME = envarg("MB_NAME", "dbtmetabase")
 
+MB_API_URL = f"http://{MB_HOST}:{MB_PORT}/api"
+
 
 @target(
     description="initial setup",
-    depends=["metabase_setup", "dbt_run", "metabase_content"],
+    depends=["dbt_run", "metabase_setup"],
 )
 def init():
     pass
@@ -31,20 +33,10 @@ def dbt_run():
     shell("dbt run --profiles-dir .")
 
 
-def _session_headers():
-    session_id = requests.post(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/session",
-        json={"username": MB_USER, "password": MB_PASSWORD},
-        timeout=10,
-    ).json()["id"]
-
-    return {"X-Metabase-Session": session_id}
-
-
 @target(description="set up Metabase user and database")
 def metabase_setup():
-    requests.post(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/setup",
+    setup_resp = requests.post(
+        url=f"{MB_API_URL}/setup",
         json={
             "token": MB_SETUP_TOKEN,
             "user": {
@@ -55,189 +47,111 @@ def metabase_setup():
                 "password_confirm": MB_PASSWORD,
                 "password": MB_PASSWORD,
             },
-            "database": {
-                "engine": "postgres",
-                "name": POSTGRES_DB,
-                "details": {
-                    "host": POSTGRES_HOST,
-                    "port": POSTGRES_PORT,
-                    "dbname": POSTGRES_DB,
-                    "user": POSTGRES_USER,
-                    "password": POSTGRES_PASSWORD,
-                    "schema-filters-type": "all",
-                    "ssl": False,
-                    "tunnel-enabled": False,
-                    "advanced-options": False,
-                },
-                "is_on_demand": False,
-                "is_full_sync": True,
-                "is_sample": False,
-                "cache_ttl": None,
-                "refingerprint": False,
-                "auto_run_queries": True,
-                "schedules": {},
-            },
             "prefs": {
                 "site_name": MB_NAME,
                 "site_locale": "en",
                 "allow_tracking": "false",
             },
         },
         timeout=10,
-    ).raise_for_status()
+    )
+    if setup_resp.status_code == 200:
+        logging.info("Metabase setup successful")
+    elif setup_resp.status_code == 403:
+        logging.info("Metabase already set up")
+    else:
+        raise requests.HTTPError(f"Error: {setup_resp.reason}", response=setup_resp)
 
-    requests.post(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/database",
-        headers=_session_headers(),
-        json={
-            "engine": "postgres",
-            "name": POSTGRES_DB,
-            "details": {
-                "host": POSTGRES_HOST,
-                "port": POSTGRES_PORT,
-                "dbname": POSTGRES_DB,
-                "user": POSTGRES_USER,
-                "password": POSTGRES_PASSWORD,
-                "schema-filters-type": "all",
-                "ssl": False,
-                "tunnel-enabled": False,
-                "advanced-options": False,
-            },
-            "is_on_demand": False,
-            "is_full_sync": True,
-            "is_sample": False,
-            "cache_ttl": None,
-            "refingerprint": False,
-            "auto_run_queries": True,
-            "schedules": {},
-        },
+    session_id = requests.post(
+        url=f"{MB_API_URL}/session",
+        json={"username": MB_USER, "password": MB_PASSWORD},
         timeout=10,
-    ).raise_for_status()
-
+    ).json()["id"]
 
-@target(description="add mock content to Metabase")
-def metabase_content():
-    headers = _session_headers()
+    headers = {"X-Metabase-Session": session_id}
 
     database_id = ""
+    sample_database_id = ""
     databases = requests.get(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/database",
+        url=f"{MB_API_URL}/database",
         headers=headers,
-        json={},
         timeout=10,
     ).json()["data"]
     for db in databases:
-        if db["name"] == POSTGRES_DB:
+        if db["name"] == POSTGRES_DB and db["engine"] == "postgres":
             database_id = db["id"]
-            break
-
-    requests.post(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/database/{database_id}/sync_schema",
-        headers=headers,
-        json={},
-        timeout=10,
-    )
-
-    time.sleep(5)
-
-    tables_fields = requests.get(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/database/{database_id}?include=tables.fields",
+        elif db["name"] == "Sample Database" and db["engine"] == "h2":
+            sample_database_id = db["id"]
+
+    if sample_database_id:
+        logging.info("Archiving Metabase sample database %s", sample_database_id)
+        requests.delete(
+            url=f"{MB_API_URL}/database/{sample_database_id}",
+            headers=headers,
+            timeout=10,
+        ).raise_for_status()
+
+    collections = requests.get(
+        url=f"{MB_API_URL}/collection",
         headers=headers,
         timeout=10,
     ).json()
-
-    customers_table_id = ""
-    first_order_field_id = ""
-    for table in tables_fields["tables"]:
-        if table["name"] == "customers":
-            customers_table_id = table["id"]
-            for field in table["fields"]:
-                if field["name"] == "first_order":
-                    first_order_field_id = field["id"]
-                    break
-
+    for collection in collections:
+        if collection.get("is_sample") == True and collection.get("archived") == False:
+            logging.info("Deleting Metabase sample collection %s", collection["id"])
+            requests.put(
+                url=f"{MB_API_URL}/collection/{collection['id']}",
+                headers=headers,
+                json={"archived": True},
+                timeout=10,
+            ).raise_for_status()
+
+    database_body = {
+        "engine": "postgres",
+        "name": POSTGRES_DB,
+        "details": {
+            "host": POSTGRES_HOST,
+            "port": POSTGRES_PORT,
+            "dbname": POSTGRES_DB,
+            "user": POSTGRES_USER,
+            "password": POSTGRES_PASSWORD,
+            "schema-filters-type": "all",
+            "ssl": False,
+            "tunnel-enabled": False,
+            "advanced-options": False,
+        },
+        "is_on_demand": False,
+        "is_full_sync": True,
+        "is_sample": False,
+        "cache_ttl": None,
+        "refingerprint": False,
+        "auto_run_queries": True,
+        "schedules": {},
+    }
+    if not database_id:
+        logging.info("Creating Metabase database")
+        database_id = requests.post(
+            url=f"{MB_API_URL}/database",
+            headers=headers,
+            json=database_body,
+            timeout=10,
+        ).json()["id"]
+    else:
+        logging.info("Updating Metabase database %s", database_id)
+        requests.put(
+            url=f"{MB_API_URL}/database/{database_id}",
+            headers=headers,
+            json=database_body,
+            timeout=10,
+        ).raise_for_status()
+
+    logging.info("Triggering Metabase database sync")
     requests.post(
-        url=f"http://{MB_HOST}:{MB_PORT}/api/card",
+        url=f"{MB_API_URL}/database/{database_id}/sync_schema",
         headers=headers,
-        json={
-            "name": "Customers",
-            "dataset_query": {
-                "database": database_id,
-                "type": "query",
-                "query": {
-                    "source-table": customers_table_id,
-                    "aggregation": [["count"]],
-                    "breakout": [
-                        [
-                            "field",
-                            first_order_field_id,
-                            {"base-type": "type/Date", "temporal-unit": "month"},
-                        ]
-                    ],
-                },
-            },
-            "display": "line",
-            "description": "Customers test",
-            "visualization_settings": {
-                "graph.dimensions": ["first_order"],
-                "graph.metrics": ["count"],
-            },
-            "collection_id": None,
-            "collection_position": None,
-            "result_metadata": [
-                {
-                    "description": None,
-                    "semantic_type": None,
-                    "coercion_strategy": None,
-                    "unit": "month",
-                    "name": "first_order",
-                    "settings": None,
-                    "fk_target_field_id": None,
-                    "field_ref": [
-                        "field",
-                        first_order_field_id,
-                        {"base-type": "type/Date", "temporal-unit": "month"},
-                    ],
-                    "effective_type": "type/DateTimeWithLocalTZ",
-                    "id": first_order_field_id,
-                    "visibility_type": "normal",
-                    "display_name": "First Order",
-                    "fingerprint": {
-                        "global": {"distinct-count": 47, "nil%": 0.38},
-                        "type": {
-                            "type/DateTime": {
-                                "earliest": "2018-01-01",
-                                "latest": "2018-04-07",
-                            }
-                        },
-                    },
-                    "base_type": "type/DateTimeWithLocalTZ",
-                },
-                {
-                    "display_name": "Count",
-                    "semantic_type": "type/Quantity",
-                    "field_ref": ["aggregation", 0],
-                    "name": "count",
-                    "base_type": "type/BigInteger",
-                    "effective_type": "type/BigInteger",
-                    "fingerprint": {
-                        "global": {"distinct-count": 4, "nil%": 0},
-                        "type": {
-                            "type/Number": {
-                                "min": 2,
-                                "q1": 11.298221281347036,
-                                "q3": 27.5,
-                                "max": 38,
-                                "sd": 12.96148139681572,
-                                "avg": 20,
-                            }
-                        },
-                    },
-                },
-            ],
-        },
+        json={},
         timeout=10,
-    )
+    ).raise_for_status()
 
 
 evaluate()
diff --git a/sandbox/metabase/metabase.db.mv.db b/sandbox/metabase/metabase.db.mv.db