test(qe): Run all tests for vitess (only in driver adapters) (#4423)

* Simplify driver adapters test configuration * Add driver adapter versions to connectors * Enable running driver adapter tests for planetscale * Fix actually run of cleaning of node modules, instead of dry-run * Raise connection limit for mysql-planetscale and lower proxy GC of connections * Tag specific versions of driver adapters in excluded tests * Remove exclusion on individual test * Explicit exclusion instead of inheriting from mod * Pin previous version of CRDB 23.1 docker image
prisma · Nov 23, 2023 · 8ddef72 · 8ddef72
1 parent 747c5fa
commit 8ddef72
Show file tree

Hide file tree

Showing 64 changed files with 443 additions and 235 deletions.
diff --git a/.buildkite/engineer b/.buildkite/engineer
@@ -54,7 +54,7 @@ fi
 # Check if the system has engineer installed, if not, use a local copy.
 if ! type "engineer" &> /dev/null; then
     # Setup Prisma engine build & test tool (engineer).
-    curl --fail -sSL "https://prisma-engineer.s3-eu-west-1.amazonaws.com/1.63/latest/$OS/engineer.gz" --output engineer.gz
+    curl --fail -sSL "https://prisma-engineer.s3-eu-west-1.amazonaws.com/1.64/latest/$OS/engineer.gz" --output engineer.gz
     gzip -d engineer.gz
     chmod +x engineer
 

diff --git a/.github/workflows/query-engine-driver-adapters.yml b/.github/workflows/query-engine-driver-adapters.yml
@@ -25,21 +25,23 @@ jobs:
       fail-fast: false
       matrix:
         adapter:
+          - name: 'planetscale'
+            setup_task: 'dev-planetscale-js'
           - name: 'pg (napi)'
-            setup_task: 'dev-pg-postgres13'
+            setup_task: 'dev-pg-js'
           - name: 'neon:ws (napi)'
-            setup_task: 'dev-neon-ws-postgres13'
+            setup_task: 'dev-neon-js'
           - name: 'libsql (napi)'
-            setup_task: 'dev-libsql-sqlite'
+            setup_task: 'dev-libsql-js'
           # TODO: uncomment when WASM engine is functional
           # - name: 'pg (wasm)'
-          #   setup_task: 'dev-pg-postgres13-wasm'
+          #   setup_task: 'dev-pg-wasm'
           #   needs_wasm_pack: true
           # - name: 'neon:ws (wasm)'
-          #   setup_task: 'dev-neon-ws-postgres13-wasm'
+          #   setup_task: 'dev-neon-wasm'
           #   needs_wasm_pack: true
           # - name: 'libsql (wasm)'
-          #   setup_task: 'dev-libsql-sqlite-wasm'
+          #   setup_task: 'dev-libsql-wasm'
           #   needs_wasm_pack: true
         node_version: ['18']
     env:

diff --git a/Makefile b/Makefile
@@ -87,18 +87,18 @@ start-sqlite:
 dev-sqlite:
 	cp $(CONFIG_PATH)/sqlite $(CONFIG_FILE)
 
-dev-libsql-sqlite: build-qe-napi build-connector-kit-js
-	cp $(CONFIG_PATH)/libsql-sqlite $(CONFIG_FILE)
+dev-libsql-js: build-qe-napi build-connector-kit-js
+	cp $(CONFIG_PATH)/libsql-js $(CONFIG_FILE)
 
-test-libsql-sqlite: dev-libsql-sqlite test-qe-st
+test-libsql-js: dev-libsql-js test-qe-st
 
-test-driver-adapter-libsql: test-libsql-sqlite
+test-driver-adapter-libsql: test-libsql-js
 
-dev-libsql-sqlite-wasm: build-qe-wasm build-connector-kit-js
-	cp $(CONFIG_PATH)/libsql-sqlite-wasm $(CONFIG_FILE)
+dev-libsql-wasm: build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/libsql-wasm $(CONFIG_FILE)
 
-test-libsql-sqlite-wasm: dev-libsql-sqlite-wasm test-qe-st
-test-driver-adapter-libsql-sqlite-wasm: test-libsql-sqlite-wasm
+test-libsql-wasm: dev-libsql-wasm test-qe-st
+test-driver-adapter-libsql-wasm: test-libsql-wasm
 
 start-postgres9:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres9
@@ -130,36 +130,36 @@ start-postgres13:
 dev-postgres13: start-postgres13
 	cp $(CONFIG_PATH)/postgres13 $(CONFIG_FILE)
 
-start-pg-postgres13: start-postgres13
+start-pg-js: start-postgres13
 
-dev-pg-postgres13: start-pg-postgres13 build-qe-napi build-connector-kit-js
-	cp $(CONFIG_PATH)/pg-postgres13 $(CONFIG_FILE)
+dev-pg-js: start-pg-js build-qe-napi build-connector-kit-js
+	cp $(CONFIG_PATH)/pg-js $(CONFIG_FILE)
 
-test-pg-postgres13: dev-pg-postgres13 test-qe-st
+test-pg-js: dev-pg-js test-qe-st
 
-dev-pg-postgres13-wasm: start-pg-postgres13 build-qe-wasm build-connector-kit-js
-	cp $(CONFIG_PATH)/pg-postgres13-wasm $(CONFIG_FILE)
+dev-pg-wasm: start-pg-js build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/pg-wasm $(CONFIG_FILE)
 
-test-pg-postgres13-wasm: dev-pg-postgres13-wasm test-qe-st
+test-pg-wasm: dev-pg-wasm test-qe-st
 
-test-driver-adapter-pg: test-pg-postgres13
-test-driver-adapter-pg-wasm: test-pg-postgres13-wasm
+test-driver-adapter-pg: test-pg-js
+test-driver-adapter-pg-wasm: test-pg-wasm
 
-start-neon-postgres13:
-	docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-postgres13
+start-neon-js:
+	docker compose -f docker-compose.yml up --wait -d --remove-orphans neon-proxy
 
-dev-neon-ws-postgres13: start-neon-postgres13 build-qe-napi build-connector-kit-js
-	cp $(CONFIG_PATH)/neon-ws-postgres13 $(CONFIG_FILE)
+dev-neon-js: start-neon-js build-qe-napi build-connector-kit-js
+	cp $(CONFIG_PATH)/neon-js $(CONFIG_FILE)
 
-test-neon-ws-postgres13: dev-neon-ws-postgres13 test-qe-st
+test-neon-js: dev-neon-js test-qe-st
 
-dev-neon-ws-postgres13-wasm: start-neon-postgres13 build-qe-wasm build-connector-kit-js
-	cp $(CONFIG_PATH)/neon-ws-postgres13-wasm $(CONFIG_FILE)
+dev-neon-wasm: start-neon-js build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/neon-wasm $(CONFIG_FILE)
 
-test-neon-ws-postgres13-wasm: dev-neon-ws-postgres13-wasm test-qe-st
+test-neon-wasm: dev-neon-wasm test-qe-st
 
-test-driver-adapter-neon: test-neon-ws-postgres13
-test-driver-adapter-neon-wasm: test-neon-ws-postgres13-wasm
+test-driver-adapter-neon: test-neon-js
+test-driver-adapter-neon-wasm: test-neon-wasm
 
 start-postgres14:
 	docker compose -f docker-compose.yml up --wait -d --remove-orphans postgres14
@@ -283,21 +283,21 @@ start-vitess_8_0:
 dev-vitess_8_0: start-vitess_8_0
 	cp $(CONFIG_PATH)/vitess_8_0 $(CONFIG_FILE)
 
-start-planetscale-vitess8:
-	docker compose -f docker-compose.yml up -d --remove-orphans planetscale-vitess8
+start-planetscale-js:
+	docker compose -f docker-compose.yml up -d --remove-orphans planetscale-proxy
 
-dev-planetscale-vitess8: start-planetscale-vitess8 build-qe-napi build-connector-kit-js
-	cp $(CONFIG_PATH)/planetscale-vitess8 $(CONFIG_FILE)
+dev-planetscale-js: start-planetscale-js build-qe-napi build-connector-kit-js
+	cp $(CONFIG_PATH)/planetscale-js $(CONFIG_FILE)
 
-test-planetscale-vitess8: dev-planetscale-vitess8 test-qe-st
+test-planetscale-js: dev-planetscale-js test-qe-st
 
-dev-planetscale-vitess8-wasm: start-planetscale-vitess8 build-qe-wasm build-connector-kit-js
-	cp $(CONFIG_PATH)/planetscale-vitess8-wasm $(CONFIG_FILE)
+dev-planetscale-wasm: start-planetscale-js build-qe-wasm build-connector-kit-js
+	cp $(CONFIG_PATH)/planetscale-wasm $(CONFIG_FILE)
 
-test-planetscale-vitess8-wasm: dev-planetscale-vitess8-wasm test-qe-st
+test-planetscale-wasm: dev-planetscale-wasm test-qe-st
 
-test-driver-adapter-planetscale: test-planetscale-vitess8
-test-driver-adapter-planetscale-wasm: test-planetscale-vitess8-wasm
+test-driver-adapter-planetscale: test-planetscale-js
+test-driver-adapter-planetscale-wasm: test-planetscale-wasm
 
 ######################
 # Local dev commands #

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3'
 services:
   cockroach_23_1:
-    image: prismagraphql/cockroachdb-custom:23.1
+    image: prismagraphql/cockroachdb-custom:23.1@sha256:c5a97355d56a7692ed34d835dfd8e3663d642219ea90736658a24840ea26862d
     restart: unless-stopped
     command: |
       start-single-node --insecure
@@ -107,7 +107,7 @@ services:
     networks:
       - databases
 
-  neon-postgres13:
+  neon-proxy:
     image: ghcr.io/neondatabase/wsproxy:latest
     restart: unless-stopped
     environment:
@@ -123,22 +123,69 @@ services:
     networks:
       - databases
 
-  planetscale-vitess8:
+  # Tests using a vitess backend were not running properly for query-engine tests, and In
+  # https://github.com/prisma/prisma-engines/pull/4423 work was done to have coverage of the correctness of planetscale
+  # driver adapters.
+  #
+  # Given these tests run against the planetscale proxy, and given a different test suite will exist for vitess with
+  # rust drivers, we opted for the path of least friction when running the driver adapter tests, which is putting a
+  # single mysql box behind the planetscale proxy instead of full vttest cluster.
+  #
+  # The tradeoffs are:
+  #
+  # - we don't exercise vitess but mysql. This is a close approximation, but there might be small differences in
+  # behavior. (ex. vttest can be returning different error messages than mysql)
+  #
+  # - however, we 1) do exercise the planetscale proxy, 2) we use relationMode=prisma and this resembles what actually
+  # happens within the query engine, where vitess does not exist as a provider, and as such there isn't any particular
+  # capability or conditional code making the engine behave differently then when using Mysql.
+  # In the end Vitess is just an abstraction existing in the test kit to a) use the mysql provider, b) run the suite
+  # with relationMode=prisma; c) being able to run or exclude specific tests for that configuration. But the existence
+  # of this testing connector is misleading, and it should probably be just a version of the MySQL testing connector
+  # instead.
+  planetscale-proxy:
     build: ./docker/planetscale_proxy
     environment:
-      MYSQL_HOST: 'vitess-test-8_0'
-      MYSQL_PORT: 33807
-      MYSQL_DATABASE: 'test'
+      MYSQL_HOST: 'mysql-planetscale'
+      MYSQL_PORT: 3306
+      MYSQL_DATABASE: prisma
     ports:
       - '8085:8085'
     depends_on:
-      - vitess-test-8_0
+      mysql-planetscale:
+        condition: service_healthy
     restart: unless-stopped
     healthcheck:
       test: ['CMD', 'nc', '-z', '127.0.0.1', '8085']
       interval: 5s
       timeout: 2s
       retries: 20
+    networks:
+      - databases
+
+  mysql-planetscale:
+    image: mysql:8.0.28
+    command: mysqld
+    restart: unless-stopped
+    platform: linux/x86_64
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: prisma
+    ports:
+      - '3310:3306'
+    networks:
+      - databases
+    tmpfs: /var/lib/planetscale-mysql
+    volumes:
+      - "./docker/planetscale-mysql/my.cnf:/etc/mysql/my.cnf"
+    ulimits:
+      nofile:
+        soft: 20000
+        hard: 40000
+    healthcheck:
+      test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ]
+      timeout: 20s
+      retries: 10
 
   postgres14:
     image: postgres:14
@@ -180,6 +227,10 @@ services:
     networks:
       - databases
     tmpfs: /var/lib/mysql
+    healthcheck:
+      test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ]
+      timeout: 20s
+      retries: 10
 
   mysql-5-7:
     image: mysql:5.7.44
@@ -194,6 +245,10 @@ services:
     networks:
       - databases
     tmpfs: /var/lib/mysql
+    healthcheck:
+      test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ]
+      timeout: 20s
+      retries: 10
 
   mysql-8-0:
     image: mysql:8.0.28
@@ -208,6 +263,10 @@ services:
     networks:
       - databases
     tmpfs: /var/lib/mysql8
+    healthcheck:
+      test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ]
+      timeout: 20s
+      retries: 10
 
   mariadb-10-0:
     image: mariadb:10

diff --git a/docker/planetscale-mysql/my.cnf b/docker/planetscale-mysql/my.cnf
@@ -0,0 +1,6 @@
+[mysqld]
+pid-file        = /var/run/mysqld/mysqld.pid
+socket          = /var/run/mysqld/mysqld.sock
+datadir         = /var/lib/mysql
+secure-file-priv= NULL
+max_connections=1000
diff --git a/docker/planetscale_proxy/Dockerfile b/docker/planetscale_proxy/Dockerfile
@@ -9,7 +9,7 @@ ENTRYPOINT /go/bin/ps-http-sim \
   -http-port=8085 \
   -mysql-addr=$MYSQL_HOST \
   -mysql-port=$MYSQL_PORT \
-  -mysql-idle-timeout=1200s \
+  -mysql-idle-timeout=1s \
   -mysql-no-pass \
   -mysql-max-rows=1000 \
   -mysql-dbname=$MYSQL_DATABASE
diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/assertion_violation_error.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/assertion_violation_error.rs
@@ -2,7 +2,7 @@ use query_engine_tests::*;
 
 #[test_suite(schema(generic))]
 mod raw_params {
-    #[connector_test(only(Postgres), exclude(JS))]
+    #[connector_test(only(Postgres), exclude(Postgres("neon.js"), Postgres("pg.js")))]
     async fn value_too_many_bind_variables(runner: Runner) -> TestResult<()> {
         let n = 32768;
 

diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/interactive_tx.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/interactive_tx.rs
@@ -1,7 +1,7 @@
 use query_engine_tests::test_suite;
 use std::borrow::Cow;
 
-#[test_suite(schema(generic))]
+#[test_suite(schema(generic), exclude(Vitess("planetscale.js")))]
 mod interactive_tx {
     use query_engine_tests::*;
     use tokio::time;
@@ -573,7 +573,7 @@ mod itx_isolation {
     use query_engine_tests::*;
 
     // All (SQL) connectors support serializable.
-    #[connector_test(exclude(MongoDb))]
+    #[connector_test(exclude(MongoDb, Vitess("planetscale.js")))]
     async fn basic_serializable(mut runner: Runner) -> TestResult<()> {
         let tx_id = runner.start_tx(5000, 5000, Some("Serializable".to_owned())).await?;
         runner.set_active_tx(tx_id.clone());
@@ -595,7 +595,7 @@ mod itx_isolation {
         Ok(())
     }
 
-    #[connector_test(exclude(MongoDb))]
+    #[connector_test(exclude(MongoDb, Vitess("planetscale.js")))]
     async fn casing_doesnt_matter(mut runner: Runner) -> TestResult<()> {
         let tx_id = runner.start_tx(5000, 5000, Some("sErIaLiZaBlE".to_owned())).await?;
         runner.set_active_tx(tx_id.clone());

diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/metrics.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/metrics.rs
@@ -1,6 +1,14 @@
 use query_engine_tests::test_suite;
 
-#[test_suite(schema(generic))]
+#[test_suite(
+    schema(generic),
+    exclude(
+        Vitess("planetscale.js"),
+        Postgres("neon.js"),
+        Postgres("pg.js"),
+        Sqlite("libsql.js")
+    )
+)]
 mod metrics {
     use query_engine_metrics::{
         PRISMA_CLIENT_QUERIES_ACTIVE, PRISMA_CLIENT_QUERIES_TOTAL, PRISMA_DATASOURCE_QUERIES_TOTAL,
@@ -9,7 +17,7 @@ mod metrics {
     use query_engine_tests::*;
     use serde_json::Value;
 
-    #[connector_test(exclude(Js))]
+    #[connector_test]
     async fn metrics_are_recorded(runner: Runner) -> TestResult<()> {
         insta::assert_snapshot!(
           run_query!(&runner, r#"mutation { createOneTestModel(data: { id: 1 }) { id }}"#),
@@ -27,7 +35,7 @@ mod metrics {
         let total_operations = get_counter(&json, PRISMA_CLIENT_QUERIES_TOTAL);
 
         match runner.connector_version() {
-            Sqlite => assert_eq!(total_queries, 9),
+            Sqlite(_) => assert_eq!(total_queries, 9),
             SqlServer(_) => assert_eq!(total_queries, 17),
             MongoDb(_) => assert_eq!(total_queries, 5),
             CockroachDb(_) => (), // not deterministic
@@ -40,7 +48,7 @@ mod metrics {
         Ok(())
     }
 
-    #[connector_test(exclude(Js))]
+    #[connector_test]
     async fn metrics_tx_do_not_go_negative(mut runner: Runner) -> TestResult<()> {
         let tx_id = runner.start_tx(5000, 5000, None).await?;
         runner.set_active_tx(tx_id.clone());

diff --git a/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/multi_schema.rs b/query-engine/connector-test-kit-rs/query-engine-tests/tests/new/multi_schema.rs
@@ -1,6 +1,6 @@
 use query_engine_tests::test_suite;
 
-#[test_suite(capabilities(MultiSchema), exclude(Mysql))]
+#[test_suite(capabilities(MultiSchema), exclude(Mysql, Vitess("planetscale.js")))]
 mod multi_schema {
     use query_engine_tests::*;