Strangle nix (#4713)

* Generate the same output as nix when measuring size * Don't use nix in wasm-size.yml, and wasm-benchmarks.yml * Fix wasm-size job * Point base branch to head * Bring back rustup in the build script. Should be a no-op * Get rid of nix in publish-query-engine-wasm * Add build-qe-wasm to Makefile * Also output size of all providers * Do not publish cargo docs * WASM_SIZE_OUTPUT to ENGINE_SIZE_OUTPUT * Try extracting deps for rust-wasm into a workflow * Move rust-wasm-setup * Add check-schema-wasm make target * Make rust wasm setup a composite workflow * Render prisma-schema-wasm version in package.json * add newline * Specify shell following action metadata syntax * Let the build script add targets and components * Pin nightly * Use relative path to scripts because we are always in the repo root * Try to use rustup included in action runners * add wasm-pack and wasm-opt * Driver adapter tests * Add clean wrappers * Remove unnecessary building step * use make target in build-prisma-schema-wasm workflow * update publish-prisma-schema-wasm pipeline * Fix * remove outdated comment * update PENDING.md * fix measure size * Remove things from PENDING.md * Use cargo binistall to accelerate binary installation * Fix previous bug in WASM_BUILD_PROFILE * add wasm target for schema-wasm * Use standard posix bc syntax * deduplicate wasm target * Output sizes in bytes, no KiB * Remove PENDING.md * Ping cargo binstall versions * Remove nix files except the shell * Be intentional about the use of nix * Extract toml config to rust-toolchain.toml cherry-picked from #4699 * Fix shellcheck * Fix publishing prisma-schema-wasm * Address some feedback * Revert rust-toolchain.toml * Remove engine size publishing from pipeline momentarily. Read more Extracted from the scope to: https://github.com/prisma/team-orm/issues/943 * Take back publishing engine sizes * Update flake.nix --------- Co-authored-by: Alexey Orlenko <[email protected]>
prisma · Feb 13, 2024 · e9f802b · e9f802b
1 parent 346196f
commit e9f802b
Show file tree

Hide file tree

Showing 21 changed files with 305 additions and 578 deletions.
diff --git a/.github/workflows/build-prisma-schema-wasm.yml b/.github/workflows/build-prisma-schema-wasm.yml
@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v24
 
-      - run: nix build .#prisma-schema-wasm
-      - run: nix flake check
+      - uses: ./.github/workflows/include/rust-wasm-setup
+
+      - run: make check-schema-wasm-package PROFILE=release
diff --git a/.github/workflows/include/rust-wasm-setup/action.yml b/.github/workflows/include/rust-wasm-setup/action.yml
@@ -0,0 +1,25 @@
+name: Rust + WASM common deps
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set default toolchain
+      shell: bash
+      run: rustup default stable
+
+    - name: Add WASM target
+      shell: bash
+      run: rustup target add wasm32-unknown-unknown
+
+    - uses: cargo-bins/cargo-binstall@main
+
+    - name: Install wasm-bindgen, wasm-opt
+      shell: bash
+      run: |
+        cargo binstall -y \
+            [email protected] \
+            [email protected]
+
+    - name: Install bc
+      shell: bash
+      run: sudo apt update && sudo apt-get install -y bc
diff --git a/.github/workflows/on-push-to-main.yml b/.github/workflows/on-push-to-main.yml
@@ -27,8 +27,5 @@ jobs:
           git config user.email "[email protected]"
           git config user.name "prisma-bot"
 
-      - name: Generate cargo docs for the workspace to gh-pages branch
-        run: nix run .#publish-cargo-docs
-
       - name: Publish engines size to gh-pages branch
         run: nix run .#publish-engine-size
diff --git a/.github/workflows/publish-prisma-schema-wasm.yml b/.github/workflows/publish-prisma-schema-wasm.yml
@@ -32,10 +32,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.inputs.enginesHash }}
-      - uses: cachix/install-nix-action@v24
+
+      - uses: ./.github/workflows/include/rust-wasm-setup
 
       - name: Build
-        run: nix build .#prisma-schema-wasm
+        run: make build-schema-wasm PROFILE=release SCHEMA_WASM_VERSION=${{ github.event.inputs.enginesWrapperVersion }}
 
       - uses: actions/setup-node@v4
         with:
@@ -45,11 +46,9 @@ jobs:
       - name: Set up NPM token for publishing later
         run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > ~/.npmrc
 
-      - name: Update version in package.json & Publish @prisma/prisma-schema-wasm
-        run: |
-          # Update version in package.json and return directory for later usage
-          PACKAGE_DIR=$( nix run .#renderPrismaSchemaWasmPackage ${{ github.event.inputs.enginesWrapperVersion }})
-          npm publish "$PACKAGE_DIR" --access public --tag ${{ github.event.inputs.npmDistTag }}
+      - name: Publish @prisma/prisma-schema-wasm
+        run: npm publish --access public --tag ${{ github.event.inputs.npmDistTag }}
+        working-directory: target/prisma-schema-wasm
       #
       # Failure handlers
       #

diff --git a/.github/workflows/publish-query-engine-wasm.yml b/.github/workflows/publish-query-engine-wasm.yml
@@ -9,14 +9,14 @@ on:
     inputs:
       packageVersion:
         required: true
-        description: 'New @prisma/query-engine-wasm package version'
+        description: "New @prisma/query-engine-wasm package version"
       enginesHash:
         required: true
-        description: 'prisma-engines commit to build'
+        description: "prisma-engines commit to build"
       npmDistTag:
         required: true
-        default: 'latest'
-        description: 'npm dist-tag (e.g. latest or integration)'
+        default: "latest"
+        description: "npm dist-tag (e.g. latest or integration)"
 
 jobs:
   build:
@@ -30,22 +30,24 @@ jobs:
         with:
           ref: ${{ github.event.inputs.enginesHash }}
 
-      - uses: cachix/install-nix-action@v24
+      - uses: ./.github/workflows/include/rust-wasm-setup
 
       - name: Build @prisma/query-engine-wasm
-        run: nix run .#export-query-engine-wasm "${{ github.event.inputs.packageVersion }}" package
+        run: make build-qe-wasm
+        env:
+          QE_WASM_VERSION: ${{ github.event.inputs.packageVersion }}
 
       - name: Install Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '20.x'
+          node-version: "20.x"
 
       - name: Set up NPM token for publishing
         run: echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" > ~/.npmrc
 
       - name: Publish @prisma/query-engine-wasm
         run: npm publish --access public --tag ${{ github.event.inputs.npmDistTag }}
-        working-directory: package
+        working-directory: query-engine/query-engine-wasm/pkg
 
       #
       # Failure handlers
@@ -57,7 +59,7 @@ jobs:
         if: ${{ failure() }}
         uses: rtCamp/[email protected]
         env:
-          SLACK_TITLE: 'Building and publishing @prisma/query-engine-wasm failed :x:'
-          SLACK_COLOR: '#FF0000'
+          SLACK_TITLE: "Building and publishing @prisma/query-engine-wasm failed :x:"
+          SLACK_COLOR: "#FF0000"
           SLACK_CHANNEL: feed-prisma-query-engine-wasm-publish-failures
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_WASM_FAILING }}
diff --git a/.github/workflows/test-query-engine-driver-adapters.yml b/.github/workflows/test-query-engine-driver-adapters.yml
@@ -97,7 +97,7 @@ jobs:
             echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV"
           fi
 
-      - uses: cachix/install-nix-action@v24
+      - uses: ./.github/workflows/include/rust-wasm-setup
       - uses: taiki-e/install-action@nextest
 
       - run: make ${{ matrix.adapter.setup_task }}

diff --git a/.github/workflows/wasm-benchmarks.yml b/.github/workflows/wasm-benchmarks.yml
@@ -24,6 +24,8 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - uses: ./.github/workflows/include/rust-wasm-setup
+
       - name: "Setup Node.js"
         uses: actions/setup-node@v4
 
@@ -32,9 +34,6 @@ jobs:
         with:
           version: 8
 
-      - name: Install bc
-        run: sudo apt update && sudo apt-get install -y bc
-
       - name: "Login to Docker Hub"
         uses: docker/login-action@v3
         continue-on-error: true
@@ -54,7 +53,6 @@ jobs:
             echo "DRIVER_ADAPTERS_BRANCH=$branch" >> "$GITHUB_ENV"
           fi
 
-      - uses: cachix/install-nix-action@v24
       - name: Setup benchmark
         run: make setup-pg-bench
 

diff --git a/.github/workflows/wasm-size.yml b/.github/workflows/wasm-size.yml
@@ -29,22 +29,13 @@ jobs:
       - name: Checkout PR branch
         uses: actions/checkout@v4
 
-      - uses: cachix/install-nix-action@v24
-        with:
-          # we need internet access for the moment
-          extra_nix_config: |
-            sandbox = false
+      - uses: ./.github/workflows/include/rust-wasm-setup
 
       - name: Build and measure PR branch
         id: measure
         run: |
-          nix build -L .#query-engine-wasm-gz
-
-          for provider in "postgresql" "mysql" "sqlite"; do
-            echo "${provider}_size=$(wc --bytes < ./result/query-engine-$provider.wasm)" >> $GITHUB_OUTPUT
-            echo "${provider}_size_gz=$(wc --bytes < ./result/query-engine-$provider.wasm.gz)" >> $GITHUB_OUTPUT
-          done
-
+          export ENGINE_SIZE_OUTPUT=$GITHUB_OUTPUT
+          make measure-qe-wasm
 
   base-wasm-size:
     name: calculate module sizes (base branch)
@@ -59,23 +50,16 @@ jobs:
     steps:
       - name: Checkout base branch
         uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
+        # with:
+        #   ref: ${{ github.event.pull_request.base.sha }}
 
-      - uses: cachix/install-nix-action@v24
-        with:
-          extra_nix_config: |
-            sandbox = false
+      - uses: ./.github/workflows/include/rust-wasm-setup
 
       - name: Build and measure base branch
         id: measure
         run: |
-          nix build -L .#query-engine-wasm-gz
-
-          for provider in "postgresql" "mysql" "sqlite"; do
-            echo "${provider}_size=$(wc --bytes < ./result/query-engine-$provider.wasm)" >> $GITHUB_OUTPUT
-            echo "${provider}_size_gz=$(wc --bytes < ./result/query-engine-$provider.wasm.gz)" >> $GITHUB_OUTPUT
-          done
+          export ENGINE_SIZE_OUTPUT=$GITHUB_OUTPUT
+          make measure-qe-wasm
 
   report-diff:
     name: report module size

diff --git a/Makefile b/Makefile
@@ -1,12 +1,13 @@
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+
 CONFIG_PATH = ./query-engine/connector-test-kit-rs/test-configs
 CONFIG_FILE = .test_config
 SCHEMA_EXAMPLES_PATH = ./query-engine/example_schemas
 DEV_SCHEMA_FILE = dev_datamodel.prisma
 DRIVER_ADAPTERS_BRANCH ?= main
-
-ifndef DISABLE_NIX
-NIX := $(shell type nix 2> /dev/null)
-endif
+ENGINE_SIZE_OUTPUT ?= /dev/stdout
+QE_WASM_VERSION ?= 0.0.0
+SCHEMA_WASM_VERSION ?= 0.0.0
 
 LIBRARY_EXT := $(shell                            \
     case "$$(uname -s)" in                        \
@@ -19,6 +20,20 @@ PROFILE ?= dev
 
 default: build
 
+###############
+# clean tasks #
+###############
+
+clean-qe-wasm:
+	@echo "Cleaning query-engine/query-engine-wasm/pkg" && \
+	cd query-engine/query-engine-wasm/pkg && find . ! -name '.' ! -name '..' ! -name 'README.md' -exec rm -rf {} +
+
+clean-cargo:
+	@echo "Cleaning cargo" && \
+	cargo clean
+
+clean: clean-qe-wasm clean-cargo
+
 ###################
 # script wrappers #
 ###################
@@ -39,6 +54,29 @@ build:
 build-qe:
 	cargo build --package query-engine
 
+build-qe-napi:
+	cargo build --package query-engine-node-api --profile $(PROFILE)
+
+build-qe-wasm:
+	cd query-engine/query-engine-wasm && \
+	./build.sh $(QE_WASM_VERSION) query-engine/query-engine-wasm/pkg
+
+build-qe-wasm-gz: build-qe-wasm
+	@cd query-engine/query-engine-wasm/pkg && \
+    for provider in postgresql mysql sqlite; do \
+        tar -zcvf $$provider.gz $$provider; \
+    done;
+
+build-schema-wasm:
+	@printf '%s\n' "🛠️  Building the Rust crate"
+	cargo build --profile $(PROFILE) --target=wasm32-unknown-unknown -p prisma-schema-build
+
+	@printf '\n%s\n' "📦 Creating the npm package"
+	WASM_BUILD_PROFILE=$(PROFILE) \
+	NPM_PACKAGE_VERSION=$(SCHEMA_WASM_VERSION) \
+	out="$(REPO_ROOT)/target/prisma-schema-wasm" \
+	./prisma-schema-wasm/scripts/install.sh
+
 # Emulate pedantic CI compilation.
 pedantic:
 	RUSTFLAGS="-D warnings" cargo fmt -- --check && RUSTFLAGS="-D warnings" cargo clippy --all-targets
@@ -77,6 +115,11 @@ test-qe-verbose-st:
 test-qe-black-box: build-qe
 	cargo test --package black-box-tests -- --test-threads 1
 
+check-schema-wasm-package: build-schema-wasm
+	PRISMA_SCHEMA_WASM="$(REPO_ROOT)/target/prisma-schema-wasm" \
+	out=$(shell mktemp -d) \
+	NODE=$(shell which node) \
+	./prisma-schema-wasm/scripts/check.sh
 
 ###########################
 # Database setup commands #
@@ -326,21 +369,12 @@ test-driver-adapter-planetscale-wasm: test-planetscale-wasm
 # Local dev commands #
 ######################
 
-build-qe-napi:
-	cargo build --package query-engine-node-api --profile $(PROFILE)
-
-build-qe-wasm:
-ifdef NIX
-	@echo "Building wasm engine on nix"
-	rm -rf query-engine/query-engine-wasm/pkg
-	nix run .#export-query-engine-wasm 0.0.0 query-engine/query-engine-wasm/pkg
-else
-	cd query-engine/query-engine-wasm && ./build.sh 0.0.0 query-engine/query-engine-wasm/pkg
-endif
-
-measure-qe-wasm: build-qe-wasm	
+measure-qe-wasm: build-qe-wasm-gz	
 	@cd query-engine/query-engine-wasm/pkg; \
-	gzip -k -c query_engine_bg.wasm | wc -c | awk '{$$1/=(1024*1024); printf "Current wasm query-engine size compressed: %.3fMB\n", $$1}'
+	for provider in postgresql mysql sqlite; do \
+		echo "$${provider}_size=$$(cat $$provider/* | wc -c | tr -d ' ')" >> $(ENGINE_SIZE_OUTPUT); \
+		echo "$${provider}_size_gz=$$(cat $$provider.gz | wc -c | tr -d ' ')" >> $(ENGINE_SIZE_OUTPUT); \
+	done;
 
 build-driver-adapters-kit: build-driver-adapters
 	cd query-engine/driver-adapters && pnpm i && pnpm build

diff --git a/flake.nix b/flake.nix
@@ -31,16 +31,9 @@
       perSystem = { config, system, pkgs, craneLib, ... }: {
         config._module.args.flakeInputs = inputs;
         imports = [
-          ./nix/all-engines.nix
           ./nix/args.nix
-          ./nix/cargo-doc.nix
-          ./nix/cli-shell.nix
-          ./nix/cli-prisma.nix
-          ./nix/dev-vm.nix
-          ./nix/memory-profiling.nix
-          ./nix/prisma-schema-wasm.nix
-          ./nix/publish-engine-size.nix
           ./nix/shell.nix
+          ./nix/publish-engine-size.nix
         ];
       };
     };

diff --git a/nix/README.md b/nix/README.md
@@ -0,0 +1,5 @@
+This directory contains a nix shell that is convenient to streamline developement, however,
+contributors must not require to depend on nix for any specific workflow.
+
+Instead, automation should be provided in a combination of bash scripts and docker, exposed over
+tasks in the [root's Makefile](/Makefile)