redirect transform in http outcall

.github/workflows/backend.yml

@@ -53,29 +53,29 @@ jobs:
       - name: Start dfx
         run: |
           dfx start --background
-      - name: Checkout base branch
-        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes')
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.base_ref }}
-      - name: Deploy main branch
-        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes')
-        run: |
-          dfx deploy backend
+#      - name: Checkout base branch
+#        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes')
+#        uses: actions/checkout@v4
+#        with:
+#          ref: ${{ github.base_ref }}
+#      - name: Deploy main branch
+#        if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes')
+#        run: |
+#          dfx deploy backend
       - uses: actions/checkout@v4
       - name: Deploy current branch
         run: dfx deploy backend
       - name: CanisterPool test
         run: |
           (for f in service/pool/tests/*.test.sh; do
             echo "==== Run test $f ===="
-            ic-repl "$f" || exit
+            ic-repl -v "$f" || exit
           done)
       - name: Actor class test
         run: |
           cd ./service/pool/tests/actor_class
           dfx canister create --all
           dfx build
-          ic-repl ./test.sh
+          ic-repl -v ./test.sh
       - name: Stop dfx
         run: dfx stop

script/deploy_app.sh

@@ -44,7 +44,7 @@ function build_frontend(name) {
 function deploy_frontend(dist) {
   let expired = ite(exist(frontend_info), is_expired(frontend_info, frontend_init?.canister_time_to_live), true);
   let info = ite(exist(frontend_info), opt frontend_info, null);
-  if expired {
+  if expired { // TODO: check it is also not transferred
       "Frontend caniter expired, fetching a new one...";
       let new_info = call Frontend.deployCanister(info, null);
   } else {

script/deploy_testnet.sh

@@ -26,18 +26,18 @@ function install(wasm, args, cycles) {
 
 function start_testnet() {
   "creating a new testnet...";
-  let wasm = file("./pool.wasm");
+  let wasm = file("../target/pool/pool.wasm");
   let backend_init = opt record {
     cycles_per_canister = 105_000_000_000;
-    max_num_canisters = 2;
+    max_num_canisters = 9;
     nonce_time_to_live = 300_000_000_000;
     canister_time_to_live = 1200_000_000_000;
     max_family_tree_size = 5;
     no_uninstall = opt false;
   };
   let frontend_init = opt record {
     cycles_per_canister = 105_000_000_000;
-    max_num_canisters = 2;
+    max_num_canisters = 9;
     nonce_time_to_live = 300_000_000_000;
     canister_time_to_live = 1200_000_000_000;
     max_family_tree_size = 5;
@@ -56,19 +56,33 @@ function start_testnet() {
 
 function populate_asset_canister(Frontend, n) {
   let asset = file("./chunked_map.wasm");
+  let deploy_arg = opt record { arg = encode (); wasm_module = asset; bypass_wasm_transform = opt true };
   while gt(n, 0) {
-    let info = call Frontend.deployCanister(null, opt record { arg = encode (); wasm_module = asset; });
-    assert info[1] == variant { install };
-    stringify("deploying asset canister ", n, " with id ", info[0].id);
-    let n = sub(n, 1);
+      if lt(n, 5) {
+          let info = call Frontend.deployCanister(null, deploy_arg);
+          stringify("deploying asset canister ", n, " with id ", stringify(info));
+          let n = sub(n, 1);
+      } else {
+          let info = par_call [Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg)];
+          stringify("deploying asset canister ", n, " with id ", stringify(info));
+          let n = sub(n, 5);
+      };
   };
   call Frontend.releaseAllCanisters();
 };
 function populate_backend(Backend, n) {
+  let nonce = record { timestamp = 0; nonce = 0 };
+  let origin = record { origin = "admin"; tags = vec {} };
   while gt(n, 0) {
-      let info = call Backend.getCanisterId(record { timestamp = 0; nonce = 0 }, record { origin = "admin"; tags = vec {} });
-      stringify("init backend canister ", n, " with id ", info.id);
-      let n = sub(n, 1);
+      if lt(n, 5) {
+          let info = call Backend.getCanisterId(nonce, origin);
+          stringify("init backend canister ", n, " with id ", stringify(info));
+          let n = sub(n, 1);
+      } else {
+          let info = par_call [Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin)];
+          stringify("init backend canister ", n, " with id ", stringify(info));
+          let n = sub(n, 5);
+      };
   };
   call Backend.releaseAllCanisters();
 };

script/update_testnet.sh

@@ -6,44 +6,59 @@ let testnet_env = env_name("testnet");
 
 function populate_asset_canister(Frontend, n) {
   let asset = file("./chunked_map.wasm");
+  let deploy_arg = opt record { arg = encode (); wasm_module = asset; bypass_wasm_transform = opt true };
   while gt(n, 0) {
-    let info = call Frontend.deployCanister(null, opt record { arg = encode (); wasm_module = asset; });
-    stringify("deploying asset canister ", n, " with id ", info[0].id);
-    let n = sub(n, 1);
+      if lt(n, 5) {
+          let info = call Frontend.deployCanister(null, deploy_arg);
+          stringify("deploying asset canister ", n, " with id ", stringify(info));
+          let n = sub(n, 1);
+      } else {
+          let info = par_call [Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg)];
+          stringify("deploying asset canister ", n, " with id ", stringify(info));
+          let n = sub(n, 5);
+      };
   };
   call Frontend.releaseAllCanisters();
 };
 function populate_backend(Backend, n) {
+  let nonce = record { timestamp = 0; nonce = 0 };
+  let origin = record { origin = "admin"; tags = vec {} };
   while gt(n, 0) {
-      let info = call Backend.getCanisterId(record { timestamp = 0; nonce = 0 }, record { origin = "admin"; tags = vec {} });
-      stringify("init backend canister ", n, " with id ", info.id);
-      let n = sub(n, 1);
+      if lt(n, 5) {
+          let info = call Backend.getCanisterId(nonce, origin);
+          stringify("init backend canister ", n, " with id ", stringify(info));
+          let n = sub(n, 1);
+      } else {
+          let info = par_call [Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin)];
+          stringify("init backend canister ", n, " with id ", stringify(info));
+          let n = sub(n, 5);
+      };
   };
   call Backend.releaseAllCanisters();
 };
 
 load testnet_env;
 let backend_init = opt record {
     cycles_per_canister = 550_000_000_000;
-    max_num_canisters = 50;
+    max_num_canisters = 1000;
     nonce_time_to_live = 300_000_000_000;
     canister_time_to_live = 2700_000_000_000;
     max_family_tree_size = 5;
     no_uninstall = opt false;
 };
 let frontend_init = opt record {
     cycles_per_canister = 550_000_000_000;
-    max_num_canisters = 50;
+    max_num_canisters = 1000;
     nonce_time_to_live = 300_000_000_000;
     canister_time_to_live = 2700_000_000_000;
     max_family_tree_size = 5;
     no_uninstall = opt true;
 };
 let wasm = file("../target/pool/pool.wasm");
-//install(Backend, wasm, backend_init, variant { upgrade });
-//install(Frontend, wasm, frontend_init, variant { upgrade });
-//call Frontend.releaseAllCanisters();
+install(Backend, wasm, backend_init, variant { upgrade });
+install(Frontend, wasm, frontend_init, variant { upgrade });
+call Frontend.releaseAllCanisters();
 populate_asset_canister(Frontend, frontend_init?.max_num_canisters);
-//call Backend.releaseAllCanisters();
+call Backend.releaseAllCanisters();
 populate_backend(Backend, backend_init?.max_num_canisters);
 export(testnet_env, Backend, Frontend, backend_init, frontend_init);

service/pool/IC.mo

@@ -22,16 +22,17 @@ module {
       taken_at_timestamp : Nat64;
   };
   public type http_header = { value : Text; name : Text };
+  public type transform_function = shared query {
+      context : Blob;
+      response : http_request_result;
+  } -> async http_request_result;
   public type http_request_args = {
     url : Text;
     method : { #get; #head; #post };
     max_response_bytes : ?Nat64;
     body : ?Blob;
     transform : ?{
-      function : shared query {
-          context : Blob;
-          response : http_request_result;
-        } -> async http_request_result;
+      function : transform_function;
       context : Blob;
     };
     headers : [http_header];

service/pool/Main.mo

@@ -157,7 +157,8 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             await IC.update_settings { canister_id = info.id; settings };
             statsByOrigin.addCanister({ origin = "external"; tags = [] });
         } else {
-            throw Error.reject "Cannot find canister";
+            stats := Logs.updateStats(stats, #mismatch);
+            throw Error.reject "transferOwnership: Cannot find canister";
         };
     };
     // Install code after transferOwnership. This call can fail if the user removes the playground from its controllers.
@@ -166,6 +167,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             throw Error.reject "Only called by controller";
         };
         if (pool.findId(args.canister_id)) {
+            stats := Logs.updateStats(stats, #mismatch);
             throw Error.reject "Canister is still solely controlled by the playground";
         };
         await IC.install_code args;
@@ -180,10 +182,15 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
         let (info, mode) = switch (opt_info) {
         case null { await* getExpiredCanisterInfo(origin) };
         case (?info) {
-                 if (not pool.find info) {
-                     await* getExpiredCanisterInfo(origin)
-                 } else {
+                 if (pool.find info) {
                      (info, #upgrade)
+                 } else {
+                     if (pool.findId(info.id)) {
+                         await* getExpiredCanisterInfo(origin)
+                     } else {
+                         stats := Logs.updateStats(stats, #mismatch);
+                         throw Error.reject "deployCanister: Cannot find canister";
+                     };
                  };
              };
         };
@@ -216,7 +223,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
                  updateTimer<system>(newInfo);
                  (newInfo, mode);
              };
-        case null { throw Error.reject "Cannot find canister" };
+        case null { throw Error.reject "pool.refresh: Cannot find canister" };
         };
     };
 
@@ -615,16 +622,47 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             throw Error.reject "Only a canister managed by the Motoko Playground can call http_request";
         };
         let cycles = 250_000_000_000;
+        try {
         if (pool.spendCycles(caller, cycles)) {
             Cycles.add<system> cycles;
-            // transform doesn't work at the moment, as it require self query call for transform
-            let res = await IC.http_request(request);
+            let new_request = switch (request.transform) {
+            case null {
+                     { request with transform = null };
+                 };
+            case (?transform) {
+                     if (Option.isSome(pool.getTransform(caller))) {
+                         throw Error.reject "No concurrent http_request allowed";
+                     };
+                     pool.rememberTransform(caller, transform);
+                     let fake_actor: actor { __transform: ICType.transform_function } = actor(Principal.toText(Principal.fromActor this));
+                     let new_transform = ?{ function = fake_actor.__transform; context = Principal.toBlob caller };
+                     { request with transform = new_transform };
+                 };
+            };
+            let res = await IC.http_request(new_request);
             let refunded = -Cycles.refunded();
             assert(pool.spendCycles(caller, refunded) == true);
             res;
         } else {
             throw Error.reject "http_request exceeds cycle spend limit";
         };
+        } finally {
+            pool.removeTransform(caller);
+        };
+    };
+    public shared composite query({ caller }) func __transform({context: Blob; response: ICType.http_request_result}) : async ICType.http_request_result {
+        if (caller != Principal.fromText("aaaaa-aa") and caller != Principal.fromText("2vxsx-fae")) {
+            throw Error.reject "Only the management canister can call __transform";
+        };
+        let id = Principal.fromBlob context;
+        switch (pool.getTransform(id)) {
+            case null {
+                throw Error.reject "No transform found";
+                 };
+            case (?transform) {
+                     await transform.function({ context = transform.context; response });
+                 };
+        };
     };
 
     system func inspect({
@@ -665,6 +703,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             #delete_canister_snapshot : Any;
             #load_canister_snapshot : Any;
             #_ttp_request : Any;
+            #__transform : Any;
         };
     }) : Bool {
         switch msg {
@@ -681,6 +720,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             case (#delete_canister_snapshot _) false;
             case (#load_canister_snapshot _) false;
             case (#_ttp_request _) false;
+            case (#__transform _) false;
             case _ true;
         };
     };

service/pool/Types.mo

@@ -10,6 +10,7 @@ import List "mo:base/List";
 import Option "mo:base/Option";
 import Int "mo:base/Int";
 import Timer "mo:base/Timer";
+import ICType "./IC";
 
 module {
     public type InitParams = {
@@ -80,12 +81,16 @@ module {
         let timers = TrieMap.TrieMap<Principal, Timer.TimerId>(Principal.equal, Principal.hash);
         var snapshots = TrieMap.TrieMap<Principal, Blob>(Principal.equal, Principal.hash);
         // Cycles spent by each canister, not persisted for upgrades
-        var cycles = TrieMap.TrieMap<Principal, Int>(Principal.equal, Principal.hash);
+        let cycles = TrieMap.TrieMap<Principal, Int>(Principal.equal, Principal.hash);
+        type TransformType = { context: Blob; function: ICType.transform_function };
+        let transforms = TrieMap.TrieMap<Principal, TransformType>(Principal.equal, Principal.hash);
 
         public type NewId = { #newId; #reuse:CanisterInfo; #outOfCapacity:Nat };
 
         public func getExpiredCanisterId() : NewId {
             if (len < size) {
+                // increment len here to prevent race condition
+                len += 1;
                 #newId
             } else {
                 switch (tree.entries().next()) {
@@ -124,10 +129,10 @@ module {
         };
 
         public func add(info: CanisterInfo) {
-            if (len >= size) {
+            if (len > size) {
                 assert false;
             };
-            len += 1;
+            // len already incremented in getExpiredCanisterId
             tree.insert info;
             metadata.put(info.id, (info.timestamp, false));
         };
@@ -199,6 +204,15 @@ module {
             cycles.put(cid, new);
             true;
         };
+        public func rememberTransform(cid: Principal, transform: TransformType) {
+            transforms.put(cid, transform);
+        };
+        public func getTransform(cid: Principal) : ?TransformType {
+            transforms.get cid
+        };
+        public func removeTransform(cid: Principal) {
+            transforms.delete cid;
+        };
 
         private func notExpired(info: CanisterInfo, now: Int) : Bool = (info.timestamp > now - ttl);