test: use kaocha test-runner for cljs (#90)

* use kaocha test-runner for cljs

* replace faulty kaocha node testrunner with olical testrunner

* read minimal shadow-cljs file for compilation

* update ci

* fix cljs support

* Add cljs code for encryptor salt.

* Concatenate output.

* Handle cljs return value properly.

* Add offsets.

* fix lz4 warning for cljs

* aes fixes

* tweaks

* Fix format and compliance test refer.

* remove unnecessary try-catch block

---------

Co-authored-by: Christian Weilbach <[email protected]>
Co-authored-by: pat <[email protected]>

.circleci/config.yml

@@ -22,10 +22,10 @@ workflows:
           context: docker-deploy
           requires:
             - tools/build
-      #- tools/cljstest:
-      #   context: docker-deploy
-      #   requires:
-      #     - tools/build
+      - tools/cljstest:
+         context: docker-deploy
+         requires:
+           - tools/build
       - tools/deploy:
           context:
             - clojars-deploy

bin/kaocha

@@ -1,4 +1,3 @@
 #!/usr/bin/env bash
 
-clojure -A:test -m kaocha.runner "$@"
-
+clojure -M:test -m kaocha.runner "$@"

bin/run-cljstests

@@ -3,8 +3,15 @@
 set -o errexit
 set -o pipefail
 
-# Compile for browser
-npm run ci-test
+echo "Running tests for node"
+# olical version:
+clojure -M:run-cljs-tests
+# kaocha version:
+#[ -d "node_modules/ws" ] || npm install ws
+#clojure -M:test -m kaocha.runner unit-node
 
-# Compile for node
-npx shadow-cljs compile node-test
+
+echo "Running tests for browser"
+# kaocha version:
+clojure -M:test -m kaocha.runner unit-browser
+# olical version requires karma

bin/run-unittests

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-TIMBRE_LEVEL=':warn' clojure -M:test -m kaocha.runner
+TIMBRE_LEVEL=':warn' clojure -M:test -m kaocha.runner unit

deps.edn

@@ -5,42 +5,45 @@
         fress/fress {:mvn/version "0.4.0"}
         mvxcvi/clj-cbor {:mvn/version "1.1.1"}
         io.replikativ/incognito {:mvn/version "0.3.66"}
-        io.replikativ/hasch {:mvn/version "0.3.7"}
-        io.replikativ/superv.async {:mvn/version "0.3.43"}
+        io.replikativ/hasch {:mvn/version "0.3.94"}
+        io.replikativ/superv.async {:mvn/version "0.3.46"}
         io.replikativ/geheimnis {:mvn/version "0.1.1"}
         org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"}
         com.github.pkpkpk/cljs-node-io {:mvn/version "2.0.332"}
         org.lz4/lz4-java {:mvn/version "1.8.0"}
         com.taoensso/timbre {:mvn/version "6.0.1"}}
- :aliases {:cljs {:extra-deps {thheller/shadow-cljs {:mvn/version "2.20.7"}
+ :aliases {:cljs {:extra-deps {thheller/shadow-cljs {:mvn/version "2.22.0"}
                                binaryage/devtools {:mvn/version "1.0.6"}}
                   :extra-paths ["test"]}
            :dev {:extra-deps {criterium/criterium {:mvn/version "0.4.6"}
                               metasoarous/oz {:mvn/version "2.0.0-alpha5"}
-                              org.clojure/tools.cli {:mvn/version "1.0.206"}}
+                              org.clojure/tools.cli {:mvn/version "1.0.214"}}
                  :extra-paths ["benchmark/src"]}
            :benchmark {:extra-deps {metasoarous/oz {:mvn/version "2.0.0-alpha5"}
                                     org.clojure/tools.cli {:mvn/version "1.0.214"}}
                        :extra-paths ["benchmark/src"]
                        :main-opts ["-m" "benchmark.core"]}
-           :test {:extra-deps {lambdaisland/kaocha {:mvn/version "1.71.1119"}
+           :test {:extra-deps {lambdaisland/kaocha {:mvn/version "1.80.1274"}
+                               lambdaisland/kaocha-cljs {:mvn/version "1.4.130"}
                                org.clojure/test.check {:mvn/version "1.1.1"}}
                   :extra-paths ["test"]
-                  :main-opts ["-e" "(set! *warn-on-reflection* true)"]}
-           :node-test {:extra-paths ["test"]
-                       :main-opts ["-m" "cljs.main" "-t" "node" "-m" "konserve.node-runner"]}
-           :idb-c {:extra-paths ["test"]
-                   :main-opts ["-m" "cljs.main" "-c" "konserve.indexeddb-test"]}
-           :idb-r {:extra-paths ["test"]
-                   :main-opts ["-m" "cljs.main" "-r"]}
+                  :main-opts ["-e" "(set! *warn-on-reflection* true)"]} 
+           :run-cljs-tests {:extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}}
+                            :extra-paths ["test"]
+                            :main-opts ["-m" "cljs-test-runner.main" 
+                                        "-o" "target/cljs" 
+                                        "--exclude" "browser" 
+                                        "--env" "node"]}
            :build  {:deps       {io.github.seancorfield/build-clj       {:git/tag "v0.8.2"
                                                                          :git/sha "0ffdb4c"}
-                                 io.github.borkdude/gh-release-artifact {:git/sha "05f8d8659e6805d513c59447ff41dc8497878462"}
+                                 io.github.borkdude/gh-release-artifact {:git/sha "db5e79559fdbfa009ed4a0921079e20e1e76b269"}
                                  babashka/babashka.curl                 {:mvn/version "0.1.2"}
-                                 babashka/fs                            {:mvn/version "0.1.6"}
-                                 cheshire/cheshire                      {:mvn/version "5.10.2"}}
+                                 babashka/fs                            {:mvn/version "0.3.17"}
+                                 cheshire/cheshire                      {:mvn/version "5.11.0"}}
                     :ns-default build}
-           :format {:extra-deps {cljfmt/cljfmt {:mvn/version "0.8.0"}}
+           :format {:extra-deps {cljfmt/cljfmt {:mvn/version "0.9.2"}}
                     :main-opts ["-m" "cljfmt.main" "check"]}
-           :ffix {:extra-deps {cljfmt/cljfmt {:mvn/version "0.9.0"}}
-                  :main-opts ["-m" "cljfmt.main" "fix"]}}}
+           :ffix {:extra-deps {cljfmt/cljfmt {:mvn/version "0.9.2"}}
+                  :main-opts ["-m" "cljfmt.main" "fix"]}
+           :outdated {:extra-deps {com.github.liquidz/antq {:mvn/version "2.2.983"}}
+                      :main-opts ["-m" "antq.core"]}}}

shadow-cljs.edn

@@ -1,23 +1,7 @@
 {:deps {:aliases [:cljs]}
-
+ :source-paths ["src"]
  :builds
  {:app
   {:target :browser
    :output-dir "public/js"
-   :asset-path "/js"
-   :modules {:main {:entries [konserve.core]}}}
-
-  :browser-test
-  {:target :browser-test
-   :test-dir  "resources/public/js/test"
-   :devtools {:http-port 8021
-              :http-root "resources/public/js/test"}}
-
-  :node-test
-  {:target :node-test
-   :output-to "out/node-tests.js"
-   :autorun true}
-
-  :ci
-  {:target :karma
-   :output-to "target/ci.js"}}}
+   :modules {:main {:entries [konserve.core]}}}}}

src/konserve/compliance_test.cljc

@@ -55,8 +55,8 @@
          (is (= 43 (<!! (k/get-in store [:baz :bar] nil opts))))
          (<!! (k/update-in store [:baz :bar] (fn [x] (+ x 2 3)) opts))
          (is (= 48 (<!! (k/get-in store [:baz :bar] nil opts))))
-         (= true (<!! (k/dissoc store :foo opts)))
-         (= false (<!! (k/dissoc store :not-there opts)))
+         (is (= true (<!! (k/dissoc store :foo opts))))
+         (is (= false (<!! (k/dissoc store :not-there opts))))
          (is (= nil (<!! (k/get-in store [:foo] nil opts))))
          (<!! (k/bassoc store :binbar (byte-array (range 10)) opts))
          (<!! (k/bget store :binbar (fn [{:keys [input-stream]}]
@@ -101,26 +101,23 @@
              (is (exception? (<!! (bget corrupt :bad (fn [_] nil)))))
              (is (exception? (<!! (bassoc corrupt :binbar (byte-array (range 10)))))))))))
 
-#?(:cljs (deftest compliance-test-cljs
-           (async done
-                  (go
-                    (let [store (<! (new-mem-store))]
-                      (is (= (<! (k/get store :foo)) nil))
-                      (<! (k/assoc store :foo :bar))
-                      (is (= :bar (<! (k/get store :foo))))
-                      (<! (k/assoc-in store [:foo] :bar2))
-                      (is (= :bar2 (<! (k/get store :foo))))
-                      (is (= :default
-                             (<! (k/get-in store [:fuu] :default))))
-                      (<! (k/update-in store [:foo] name))
-                      (is (= "bar2" (<! (k/get store :foo))))
-                      (<! (k/assoc-in store [:baz] {:bar 42}))
-                      (is (= (<! (k/get-in store [:baz :bar])) 42))
-                      (<! (k/update-in store [:baz :bar] inc))
-                      (is (= (<! (k/get-in store [:baz :bar])) 43))
-                      (<! (k/update-in store [:baz :bar] #(+ % 2 3)))
-                      (is (= (<! (k/get-in store [:baz :bar])) 48))
-                      (<! (k/dissoc store :foo))
-                      (is (= (<! (k/get-in store [:foo])) nil))
-                      (done))))))
+(defn async-compliance-test [store]
+  (go
+    (and
+     (is (= nil (<! (k/get store :foo))))
+     (is (= [nil :bar] (<! (k/assoc store :foo :bar))))
+     (is (= :bar (<! (k/get store :foo))))
+     (is (= [nil :bar2] (<! (k/assoc-in store [:foo] :bar2))))
+     (is (= :bar2 (<! (k/get store :foo))))
+     (is (= :default (<! (k/get-in store [:fuu] :default))))
+     (<! (k/update-in store [:foo] name))
+     (is (= "bar2" (<! (k/get store :foo))))
+     (<! (k/assoc-in store [:baz] {:bar 42}))
+     (is (= (<! (k/get-in store [:baz :bar])) 42))
+     (<! (k/update-in store [:baz :bar] inc))
+     (is (= (<! (k/get-in store [:baz :bar])) 43))
+     (<! (k/update-in store [:baz :bar] #(+ % 2 3)))
+     (is (= (<! (k/get-in store [:baz :bar])) 48))
+     (<! (k/dissoc store :foo))
+     (is (= (<! (k/get-in store [:foo])) nil)))))
 

src/konserve/compressor.cljc

@@ -48,19 +48,16 @@
 
 (def byte->compressor
   {0 null-compressor
-   1 #?(:clj (try
-              ;; LZ4 requires native code that breaks the native-image executable atm.
-               (if (native-image-build?)
-                 unsupported-lz4-compressor
-                 lz4-compressor)
-               (catch Exception _
-                 lz4-compressor))
+   1 #?(:clj (if (native-image-build?)
+               unsupported-lz4-compressor
+               lz4-compressor)
         :cljs unsupported-lz4-compressor)})
 
 (def compressor->byte
   (invert-map byte->compressor))
 
 (defn get-compressor [type]
   (case type
-    :lz4 lz4-compressor
+    :lz4 #?(:clj lz4-compressor
+            :cljs unsupported-lz4-compressor)
     null-compressor))

src/konserve/encryptor.cljc

@@ -3,7 +3,9 @@
             [konserve.utils :refer [invert-map]]
             [geheimnis.aes :refer [encrypt decrypt]]
             [hasch.core :refer [edn-hash uuid]])
-  (:import [java.io ByteArrayInputStream ByteArrayOutputStream]))
+  #?(:clj (:import [java.io ByteArrayInputStream ByteArrayOutputStream])))
+
+(def ^:const salt-size 64)
 
 (defrecord NullEncryptor [serializer]
   PStoreSerializer
@@ -25,29 +27,39 @@
 (defn get-key [salt key]
   ["key" salt key])
 
-;; TODO cljs support incomplete, prepending of salt is missing
 (defrecord AESEncryptor [serializer key]
   PStoreSerializer
   (-deserialize [_ read-handlers bytes]
-    (let [salt-array (byte-array 64)
-          _ (.read ^ByteArrayInputStream bytes salt-array)
-          salt (map int salt-array)
-          decrypted (decrypt (get-key salt key)
-                             #?(:clj (.readAllBytes ^ByteArrayInputStream bytes) :cljs bytes)
-                             :iv (get-initial-vector salt key))]
-      (-deserialize serializer read-handlers (ByteArrayInputStream. decrypted))))
+    (let [salt #?(:clj (let [salt-array (byte-array salt-size)]
+                         (.read ^ByteArrayInputStream bytes salt-array)
+                         (map int salt-array))
+                  :cljs
+                  (map (fn [b] (if (> b 128) (- b 256) b)) (.slice bytes 0 salt-size)))
+          data #?(:clj (.readAllBytes ^ByteArrayInputStream bytes)
+                  :cljs (.slice bytes salt-size))
+          decrypted (decrypt (get-key salt key) data :iv (get-initial-vector salt key))]
+      (-deserialize serializer read-handlers #?(:clj (ByteArrayInputStream. decrypted)
+                                                :cljs decrypted))))
   (-serialize [_ bytes write-handlers val]
-    #?(:cljs (encrypt key (-serialize serializer bytes write-handlers val))
-       :clj (let [unsigned-byte-offset 128
-                  salt (map #(int (- % unsigned-byte-offset)) (edn-hash (uuid)))
-                  buffer-size (* 16 1024)
-                  bos (ByteArrayOutputStream. buffer-size)
-                  _ (.write ^ByteArrayOutputStream bytes (byte-array salt))
-                  _ (-serialize serializer bos write-handlers val)
-                  ba (.toByteArray bos)
-                  encrypted ^bytes (encrypt (get-key salt key)
-                                            ba :iv (get-initial-vector salt key))]
-              (.write ^ByteArrayOutputStream bytes encrypted)))))
+    (let [unsigned-byte-offset 128
+          salt (map #(int (- (#?(:cljs inc :clj identity) %) unsigned-byte-offset)) (edn-hash (uuid)))]
+      #?(:cljs
+         (let [data (-serialize serializer bytes write-handlers val)
+               iv (get-initial-vector salt key)
+               bytes (encrypt (get-key salt key) (.from js/Array data) :iv iv)
+               output (js/Uint8Array. (+ salt-size (count bytes)))]
+           (.set output (js/Uint8Array.from (into-array salt)) 0)
+           (.set output (js/Uint8Array.from bytes) salt-size)
+           output)
+         :clj
+         (let [buffer-size (* 16 1024)
+               bos (ByteArrayOutputStream. buffer-size)
+               _ (.write ^ByteArrayOutputStream bytes (byte-array salt))
+               _ (-serialize serializer bos write-handlers val)
+               ba (.toByteArray bos)
+               iv (get-initial-vector salt key)
+               encrypted ^bytes (encrypt (get-key salt key) ba :iv iv)]
+           (.write ^ByteArrayOutputStream bytes encrypted))))))
 
 (defn aes-encryptor [config]
   (let [{:keys [key]} config]

src/konserve/impl/defaults.cljc

@@ -60,10 +60,7 @@
           store-key (or store-key (key->store-key key))
           to-array #?(:cljs
                       (fn [value]
-                        (-> serializer
-                            compressor
-                            (encryptor store-key (:encryptor config))
-                            (-serialize nil write-handlers value)))
+                        (-serialize ((encryptor  (:encryptor config)) (compressor serializer)) nil write-handlers value))
                       :clj
                       (fn [value]
                         (let [bos (ByteArrayOutputStream.)]

test/konserve/encryptor_test.cljc

@@ -1,7 +1,8 @@
 (ns konserve.encryptor-test
   (:require [clojure.test :refer [deftest]]
-            [konserve.filestore :refer [connect-fs-store delete-store]]
-            [konserve.compliance-test :refer [compliance-test]]))
+            [clojure.core.async :refer [go <!]]
+            [#?(:clj konserve.filestore :cljs konserve.node-filestore) :refer [connect-fs-store delete-store]]
+            [konserve.compliance-test :refer [#?(:clj compliance-test) async-compliance-test]]))
 
 (deftest encryptor-test
   (let [folder "/tmp/konserve-fs-encryptor-test"
@@ -10,5 +11,13 @@
                                  :config {:encryptor {:type :aes
                                                       :key "s3cr3t"}}
                                  :opts {:sync? true})]
-    (compliance-test store)
-    (delete-store folder)))
+    #?(:clj
+       (do
+         (compliance-test store)
+         (delete-store folder))
+       :cljs
+       (cljs.test/async done
+                        (go
+                          (<! (async-compliance-test store))
+                          (delete-store folder)
+                          (done))))))