From 81d7d8119dd1d6d85b458e2092a9e6362c0158bc Mon Sep 17 00:00:00 2001 From: Yan Chen <48968912+chenyan-dfinity@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:17:16 -0700 Subject: [PATCH] redirect transform in http outcall (#247) * test transfer * fix * fix * fix * test * error out deployCanister when it's already transferred * fix * fix * fix newId race condition * redirect http outcall transform * fake transform type * try finally * fix * fix * remove pool.transform * fix --- .github/workflows/backend.yml | 33 +++++++++--------- .github/workflows/frontend.yml | 9 ++--- dfx.json | 21 +++++------ package.json | 3 +- script/deploy_app.sh | 2 +- script/deploy_testnet.sh | 34 ++++++++++++------ script/update_testnet.sh | 39 ++++++++++++++------- service/pool/IC.mo | 9 ++--- service/pool/Main.mo | 46 +++++++++++++++++++++---- service/pool/Types.mo | 9 +++-- service/pool/tests/canisterPool.test.sh | 9 +++++ service/wasm-utils/Cargo.lock | 4 +-- service/wasm-utils/Cargo.toml | 2 +- src/examples.ts | 5 +++ 14 files changed, 150 insertions(+), 75 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 03512fb6..c6d59922 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -14,9 +14,9 @@ jobs: strategy: fail-fast: false env: - DFX_VERSION: 0.20.0 - IC_REPL_VERSION: 0.7.4 - MOC_VERSION: 0.12.0 + DFX_VERSION: 0.22.0 + IC_REPL_VERSION: 0.7.5 + MOPS_VERSION: 0.2.0 steps: - name: Setup Rust uses: actions-rs/toolchain@v1 @@ -48,20 +48,21 @@ jobs: npm i -g ic-mops dfx cache install cd $(dfx cache show) - wget https://github.com/dfinity/motoko/releases/download/$MOC_VERSION/motoko-Linux-x86_64-$MOC_VERSION.tar.gz - tar zxvf motoko-Linux-x86_64-$MOC_VERSION.tar.gz + wget https://github.com/chenyan2002/mops-cli/releases/download/$MOPS_VERSION/mops-cli-linux64 + cp ./mops-cli-linux64 /usr/local/bin/mops-cli + chmod a+x /usr/local/bin/mops-cli - name: Start dfx run: | dfx start --background - - name: Checkout base branch - if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes') - uses: actions/checkout@v4 - with: - ref: ${{ github.base_ref }} - - name: Deploy main branch - if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes') - run: | - dfx deploy backend +# - name: Checkout base branch +# if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes') +# uses: actions/checkout@v4 +# with: +# ref: ${{ github.base_ref }} +# - name: Deploy main branch +# if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'breaking_changes') +# run: | +# dfx deploy backend - uses: actions/checkout@v4 - name: Deploy current branch run: dfx deploy backend @@ -69,13 +70,13 @@ jobs: run: | (for f in service/pool/tests/*.test.sh; do echo "==== Run test $f ====" - ic-repl "$f" || exit + ic-repl -v "$f" || exit done) - name: Actor class test run: | cd ./service/pool/tests/actor_class dfx canister create --all dfx build - ic-repl ./test.sh + ic-repl -v ./test.sh - name: Stop dfx run: dfx stop diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 15f63cd1..36718242 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -21,9 +21,9 @@ jobs: - 18 - 20 env: - DFX_VERSION: 0.20.0 + DFX_VERSION: 0.22.0 SKIP_WASM: true - MOC_VERSION: 0.12.0 + MOPS_VERSION: 0.2.0 steps: - uses: actions/checkout@v4 - name: Setup Node.js ${{ matrix.node }} @@ -41,8 +41,9 @@ jobs: npm i -g ic-mops dfx cache install cd $(dfx cache show) - wget https://github.com/dfinity/motoko/releases/download/$MOC_VERSION/motoko-Linux-x86_64-$MOC_VERSION.tar.gz - tar zxvf motoko-Linux-x86_64-$MOC_VERSION.tar.gz + wget https://github.com/chenyan2002/mops-cli/releases/download/$MOPS_VERSION/mops-cli-linux64 + cp ./mops-cli-linux64 /usr/local/bin/mops-cli + chmod a+x /usr/local/bin/mops-cli - name: Start dfx run: | dfx start --background diff --git a/dfx.json b/dfx.json index 351baad9..dd1eeccb 100644 --- a/dfx.json +++ b/dfx.json @@ -13,32 +13,27 @@ ] }, "backend": { + "type": "custom", "dependencies": ["wasm-utils"], "specified_id": "mwrha-maaaa-aaaab-qabqq-cai", - "main": "service/pool/Main.mo", - "type": "motoko", + "build": ["mops-cli build service/pool/Main.mo"], + "candid": "target/pool/pool.did", + "wasm": "target/pool/pool.wasm", "optimize": "cycles" }, "saved": { + "type": "custom", "specified_id": "vhtho-raaaa-aaaab-qadoq-cai", - "main": "service/saved/Saved.mo", - "type": "motoko", + "build": ["mops-cli build service/saved/Saved.mo"], + "candid": "target/Saved/Saved.did", + "wasm": "target/Saved/Saved.wasm", "optimize": "cycles" }, "react_app": { "dependencies": ["backend", "saved"], "specified_id": "m7sm4-2iaaa-aaaab-qabra-cai", - "frontend": { - "entrypoint": "public/index.html" - }, "source": ["build"], "type": "assets" } - }, - "defaults": { - "build": { - "output": "build", - "packtool": "mops sources" - } } } diff --git a/package.json b/package.json index 3e0d29e6..52ab783f 100644 --- a/package.json +++ b/package.json @@ -51,8 +51,7 @@ "clean": "dfx stop && dfx start --clean --background", "prestart": "dfx start --background; [ -f .dfx/local/canister_ids.json ] || dfx deploy", "predeploy": "npm run prestart", - "prepare": "husky install", - "postinstall": "mops install" + "prepare": "husky install" }, "eslintConfig": { "extends": [ diff --git a/script/deploy_app.sh b/script/deploy_app.sh index 8a2ac4f0..9f709289 100755 --- a/script/deploy_app.sh +++ b/script/deploy_app.sh @@ -44,7 +44,7 @@ function build_frontend(name) { function deploy_frontend(dist) { let expired = ite(exist(frontend_info), is_expired(frontend_info, frontend_init?.canister_time_to_live), true); let info = ite(exist(frontend_info), opt frontend_info, null); - if expired { + if expired { // TODO: check it is also not transferred "Frontend caniter expired, fetching a new one..."; let new_info = call Frontend.deployCanister(info, null); } else { diff --git a/script/deploy_testnet.sh b/script/deploy_testnet.sh index 9fb9e6ac..13e020fd 100755 --- a/script/deploy_testnet.sh +++ b/script/deploy_testnet.sh @@ -26,10 +26,10 @@ function install(wasm, args, cycles) { function start_testnet() { "creating a new testnet..."; - let wasm = file("./pool.wasm"); + let wasm = file("../target/pool/pool.wasm"); let backend_init = opt record { cycles_per_canister = 105_000_000_000; - max_num_canisters = 2; + max_num_canisters = 9; nonce_time_to_live = 300_000_000_000; canister_time_to_live = 1200_000_000_000; max_family_tree_size = 5; @@ -37,7 +37,7 @@ function start_testnet() { }; let frontend_init = opt record { cycles_per_canister = 105_000_000_000; - max_num_canisters = 2; + max_num_canisters = 9; nonce_time_to_live = 300_000_000_000; canister_time_to_live = 1200_000_000_000; max_family_tree_size = 5; @@ -56,19 +56,33 @@ function start_testnet() { function populate_asset_canister(Frontend, n) { let asset = file("./chunked_map.wasm"); + let deploy_arg = opt record { arg = encode (); wasm_module = asset; bypass_wasm_transform = opt true }; while gt(n, 0) { - let info = call Frontend.deployCanister(null, opt record { arg = encode (); wasm_module = asset; }); - assert info[1] == variant { install }; - stringify("deploying asset canister ", n, " with id ", info[0].id); - let n = sub(n, 1); + if lt(n, 5) { + let info = call Frontend.deployCanister(null, deploy_arg); + stringify("deploying asset canister ", n, " with id ", stringify(info)); + let n = sub(n, 1); + } else { + let info = par_call [Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg)]; + stringify("deploying asset canister ", n, " with id ", stringify(info)); + let n = sub(n, 5); + }; }; call Frontend.releaseAllCanisters(); }; function populate_backend(Backend, n) { + let nonce = record { timestamp = 0; nonce = 0 }; + let origin = record { origin = "admin"; tags = vec {} }; while gt(n, 0) { - let info = call Backend.getCanisterId(record { timestamp = 0; nonce = 0 }, record { origin = "admin"; tags = vec {} }); - stringify("init backend canister ", n, " with id ", info.id); - let n = sub(n, 1); + if lt(n, 5) { + let info = call Backend.getCanisterId(nonce, origin); + stringify("init backend canister ", n, " with id ", stringify(info)); + let n = sub(n, 1); + } else { + let info = par_call [Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin)]; + stringify("init backend canister ", n, " with id ", stringify(info)); + let n = sub(n, 5); + }; }; call Backend.releaseAllCanisters(); }; diff --git a/script/update_testnet.sh b/script/update_testnet.sh index a0c1ad9f..b14e6f51 100755 --- a/script/update_testnet.sh +++ b/script/update_testnet.sh @@ -6,18 +6,33 @@ let testnet_env = env_name("testnet"); function populate_asset_canister(Frontend, n) { let asset = file("./chunked_map.wasm"); + let deploy_arg = opt record { arg = encode (); wasm_module = asset; bypass_wasm_transform = opt true }; while gt(n, 0) { - let info = call Frontend.deployCanister(null, opt record { arg = encode (); wasm_module = asset; }); - stringify("deploying asset canister ", n, " with id ", info[0].id); - let n = sub(n, 1); + if lt(n, 5) { + let info = call Frontend.deployCanister(null, deploy_arg); + stringify("deploying asset canister ", n, " with id ", stringify(info)); + let n = sub(n, 1); + } else { + let info = par_call [Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg), Frontend.deployCanister(null, deploy_arg)]; + stringify("deploying asset canister ", n, " with id ", stringify(info)); + let n = sub(n, 5); + }; }; call Frontend.releaseAllCanisters(); }; function populate_backend(Backend, n) { + let nonce = record { timestamp = 0; nonce = 0 }; + let origin = record { origin = "admin"; tags = vec {} }; while gt(n, 0) { - let info = call Backend.getCanisterId(record { timestamp = 0; nonce = 0 }, record { origin = "admin"; tags = vec {} }); - stringify("init backend canister ", n, " with id ", info.id); - let n = sub(n, 1); + if lt(n, 5) { + let info = call Backend.getCanisterId(nonce, origin); + stringify("init backend canister ", n, " with id ", stringify(info)); + let n = sub(n, 1); + } else { + let info = par_call [Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin), Backend.getCanisterId(nonce, origin)]; + stringify("init backend canister ", n, " with id ", stringify(info)); + let n = sub(n, 5); + }; }; call Backend.releaseAllCanisters(); }; @@ -25,7 +40,7 @@ function populate_backend(Backend, n) { load testnet_env; let backend_init = opt record { cycles_per_canister = 550_000_000_000; - max_num_canisters = 50; + max_num_canisters = 1000; nonce_time_to_live = 300_000_000_000; canister_time_to_live = 2700_000_000_000; max_family_tree_size = 5; @@ -33,17 +48,17 @@ let backend_init = opt record { }; let frontend_init = opt record { cycles_per_canister = 550_000_000_000; - max_num_canisters = 50; + max_num_canisters = 1000; nonce_time_to_live = 300_000_000_000; canister_time_to_live = 2700_000_000_000; max_family_tree_size = 5; no_uninstall = opt true; }; let wasm = file("../target/pool/pool.wasm"); -//install(Backend, wasm, backend_init, variant { upgrade }); -//install(Frontend, wasm, frontend_init, variant { upgrade }); -//call Frontend.releaseAllCanisters(); +install(Backend, wasm, backend_init, variant { upgrade }); +install(Frontend, wasm, frontend_init, variant { upgrade }); +call Frontend.releaseAllCanisters(); populate_asset_canister(Frontend, frontend_init?.max_num_canisters); -//call Backend.releaseAllCanisters(); +call Backend.releaseAllCanisters(); populate_backend(Backend, backend_init?.max_num_canisters); export(testnet_env, Backend, Frontend, backend_init, frontend_init); diff --git a/service/pool/IC.mo b/service/pool/IC.mo index 411cbcb1..fd55cd25 100644 --- a/service/pool/IC.mo +++ b/service/pool/IC.mo @@ -22,16 +22,17 @@ module { taken_at_timestamp : Nat64; }; public type http_header = { value : Text; name : Text }; + public type transform_function = shared query { + context : Blob; + response : http_request_result; + } -> async http_request_result; public type http_request_args = { url : Text; method : { #get; #head; #post }; max_response_bytes : ?Nat64; body : ?Blob; transform : ?{ - function : shared query { - context : Blob; - response : http_request_result; - } -> async http_request_result; + function : transform_function; context : Blob; }; headers : [http_header]; diff --git a/service/pool/Main.mo b/service/pool/Main.mo index 24d12f75..dbb21ddf 100644 --- a/service/pool/Main.mo +++ b/service/pool/Main.mo @@ -157,7 +157,8 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { await IC.update_settings { canister_id = info.id; settings }; statsByOrigin.addCanister({ origin = "external"; tags = [] }); } else { - throw Error.reject "Cannot find canister"; + stats := Logs.updateStats(stats, #mismatch); + throw Error.reject "transferOwnership: Cannot find canister"; }; }; // Install code after transferOwnership. This call can fail if the user removes the playground from its controllers. @@ -166,6 +167,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { throw Error.reject "Only called by controller"; }; if (pool.findId(args.canister_id)) { + stats := Logs.updateStats(stats, #mismatch); throw Error.reject "Canister is still solely controlled by the playground"; }; await IC.install_code args; @@ -180,10 +182,15 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { let (info, mode) = switch (opt_info) { case null { await* getExpiredCanisterInfo(origin) }; case (?info) { - if (not pool.find info) { - await* getExpiredCanisterInfo(origin) - } else { + if (pool.find info) { (info, #upgrade) + } else { + if (pool.findId(info.id)) { + await* getExpiredCanisterInfo(origin) + } else { + stats := Logs.updateStats(stats, #mismatch); + throw Error.reject "deployCanister: Cannot find canister"; + }; }; }; }; @@ -216,7 +223,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { updateTimer(newInfo); (newInfo, mode); }; - case null { throw Error.reject "Cannot find canister" }; + case null { throw Error.reject "pool.refresh: Cannot find canister" }; }; }; @@ -617,8 +624,18 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { let cycles = 250_000_000_000; if (pool.spendCycles(caller, cycles)) { Cycles.add cycles; - // transform doesn't work at the moment, as it require self query call for transform - let res = await IC.http_request(request); + let new_request = switch (request.transform) { + case null { + { request with transform = null }; + }; + case (?transform) { + let payload = { caller; transform }; + let fake_actor: actor { __transform: ICType.transform_function } = actor(Principal.toText(Principal.fromActor this)); + let new_transform = ?{ function = fake_actor.__transform; context = to_candid(payload) }; + { request with transform = new_transform }; + }; + }; + let res = await IC.http_request(new_request); let refunded = -Cycles.refunded(); assert(pool.spendCycles(caller, refunded) == true); res; @@ -626,6 +643,19 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { throw Error.reject "http_request exceeds cycle spend limit"; }; }; + public shared composite query({ caller }) func __transform({context: Blob; response: ICType.http_request_result}) : async ICType.http_request_result { + // TODO Remove anonymous identity once https://github.com/dfinity/ic/pull/1337 is released + if (caller != Principal.fromText("aaaaa-aa") and caller != Principal.fromText("2vxsx-fae")) { + throw Error.reject "Only the management canister can call __transform"; + }; + let ?raw : ?{ caller: Principal; transform: {context: Blob; function: ICType.transform_function} } = from_candid context else { + throw Error.reject "__transform: Invalid context"; + }; + if (not pool.findId(raw.caller)) { + throw Error.reject "__transform: Only a canister managed by the Motoko Playground can call __transform"; + }; + await raw.transform.function({ context = raw.transform.context; response }); + }; system func inspect({ msg : { @@ -665,6 +695,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { #delete_canister_snapshot : Any; #load_canister_snapshot : Any; #_ttp_request : Any; + #__transform : Any; }; }) : Bool { switch msg { @@ -681,6 +712,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this { case (#delete_canister_snapshot _) false; case (#load_canister_snapshot _) false; case (#_ttp_request _) false; + case (#__transform _) false; case _ true; }; }; diff --git a/service/pool/Types.mo b/service/pool/Types.mo index 191beb09..6397bf02 100644 --- a/service/pool/Types.mo +++ b/service/pool/Types.mo @@ -10,6 +10,7 @@ import List "mo:base/List"; import Option "mo:base/Option"; import Int "mo:base/Int"; import Timer "mo:base/Timer"; +import ICType "./IC"; module { public type InitParams = { @@ -80,12 +81,14 @@ module { let timers = TrieMap.TrieMap(Principal.equal, Principal.hash); var snapshots = TrieMap.TrieMap(Principal.equal, Principal.hash); // Cycles spent by each canister, not persisted for upgrades - var cycles = TrieMap.TrieMap(Principal.equal, Principal.hash); + let cycles = TrieMap.TrieMap(Principal.equal, Principal.hash); public type NewId = { #newId; #reuse:CanisterInfo; #outOfCapacity:Nat }; public func getExpiredCanisterId() : NewId { if (len < size) { + // increment len here to prevent race condition + len += 1; #newId } else { switch (tree.entries().next()) { @@ -124,10 +127,10 @@ module { }; public func add(info: CanisterInfo) { - if (len >= size) { + if (len > size) { assert false; }; - len += 1; + // len already incremented in getExpiredCanisterId tree.insert info; metadata.put(info.id, (info.timestamp, false)); }; diff --git a/service/pool/tests/canisterPool.test.sh b/service/pool/tests/canisterPool.test.sh index 8db099c9..c92d7ef8 100644 --- a/service/pool/tests/canisterPool.test.sh +++ b/service/pool/tests/canisterPool.test.sh @@ -18,6 +18,11 @@ let nonce = record { timestamp = 1 : int; nonce = 1 : nat }; let CID2 = call S.getCanisterId(nonce, origin); call S.installCode(CID2, record { arg = blob ""; wasm_module = empty_wasm; mode = variant { install }; canister_id = CID2.id }, record { profiling = false; is_whitelisted = false; origin = origin }); read_state("canister", CID2.id, "module_hash"); +let c1 = call S.deployCanister(null, opt record { arg = blob ""; wasm_module = empty_wasm; bypass_wasm_transform = opt true }); +let c1 = c1[0]; +call S.transferOwnership(c1, vec {c1.id; S}); +fail call S.deployCanister(opt c1, opt record { arg = blob ""; wasm_module = empty_wasm; bypass_wasm_transform = opt true }); +assert _ ~= "Cannot find canister"; let init = opt record { cycles_per_canister = 105_000_000_000; @@ -32,6 +37,9 @@ let nonce = record { timestamp = 1 : int; nonce = 1 : nat }; let CID = call S.getCanisterId(nonce, origin); call S.installCode(CID, record { arg = blob ""; wasm_module = empty_wasm; mode = variant { install }; canister_id = CID.id }, record { profiling = false; is_whitelisted = false; origin = origin }); read_state("canister", CID.id, "module_hash"); +let CID3 = call S.deployCanister(null, opt record { arg = blob ""; wasm_module = empty_wasm; bypass_wasm_transform = opt true }); +let CID3 = CID3[0]; +call S.transferOwnership(CID3, vec {CID3.id; S}); // Immediately expire let init = opt record { @@ -98,3 +106,4 @@ call S.getCanisterId(nonce, origin); fail read_state("canister", CID.id, "module_hash"); assert _ ~= "absent"; read_state("canister", CID2.id, "module_hash"); +read_state("canister", CID3.id, "module_hash"); diff --git a/service/wasm-utils/Cargo.lock b/service/wasm-utils/Cargo.lock index 8c900073..d054c81a 100644 --- a/service/wasm-utils/Cargo.lock +++ b/service/wasm-utils/Cargo.lock @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "ic-wasm" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cd6705c8489fba80aa4bbef0f083d9ead20d7c2471b7e3a60ca2790c91344a" +checksum = "5574bf249d201ddd2c27c3fdf178ddacb1be1c705c8a5b4c1339c393758f2bf2" dependencies = [ "candid", "libflate", diff --git a/service/wasm-utils/Cargo.toml b/service/wasm-utils/Cargo.toml index 940bab3b..d26d1b78 100644 --- a/service/wasm-utils/Cargo.toml +++ b/service/wasm-utils/Cargo.toml @@ -13,7 +13,7 @@ ic-cdk = "0.16" serde = "1.0" serde_bytes = "0.11" candid = "0.10" -ic-wasm = { version = "0.8.3", default-features = false } +ic-wasm = { version = "0.8.5", default-features = false } sha2 = "0.10.6" [profile.release] diff --git a/src/examples.ts b/src/examples.ts index cd68bc4d..4c8a2882 100644 --- a/src/examples.ts +++ b/src/examples.ts @@ -69,6 +69,11 @@ export const exampleProjects: ExampleProject[] = [ repo: { dir: "motoko/basic_dao/src", ...example }, readme: `${readmeURL}/basic_dao/README.md`, }, + { + name: "Http Outcall", + repo: { dir: "motoko/send_http_get/src/send_http_get_backend", ...example }, + readme: `${readmeURL}/send_http_get/README.md`, + }, ]; export async function fetchExample(