refactor stats

package.json

@@ -16,6 +16,7 @@
     "prettier-plugin-motoko": "^0.8.1",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",
+    "react-google-charts": "^4.0.1",
     "react-markdown": "^6.0.2",
     "react-modal": "^3.14.3",
     "styled-components": "^5.3.0",

service/pool/Logs.mo

@@ -7,71 +7,73 @@ import {get} "mo:base/Option";
 
 module {
     public type Origin = { origin: Text; tags: [Text] };
-    public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>, Map.Tree<Text, Nat>);
+    public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>);
     public class StatsByOrigin() {
         var canisters = Map.RBTree<Text, Nat>(compare);
         var installs = Map.RBTree<Text, Nat>(compare);
-        var tags = Map.RBTree<Text, Nat>(compare);
-        public func share() : SharedStatsByOrigin = (canisters.share(), installs.share(), tags.share());
+        public func share() : SharedStatsByOrigin = (canisters.share(), installs.share());
         public func unshare(x : SharedStatsByOrigin) {
             canisters.unshare(x.0);
             installs.unshare(x.1);
-            tags.unshare(x.2);
         };
-        func addTags(list: [Text]) {
+        func addTags(map: Map.RBTree<Text,Nat>, list: [Text]) {
             for (tag in list.vals()) {
-                switch (tags.get(tag)) {
-                case null { tags.put(tag, 1) };
-                case (?n) { tags.put(tag, n + 1) };
+                switch (map.get(tag)) {
+                case null { map.put(tag, 1) };
+                case (?n) { map.put(tag, n + 1) };
                 };
             };
         };
-        public func addCanister(origin: Origin) {
-            switch (canisters.get(origin.origin)) {
-            case null { canisters.put(origin.origin, 1) };
-            case (?n) { canisters.put(origin.origin, n + 1) };
+        // if to is null, delete the from tag
+        func merge_tag_(map: Map.RBTree<Text,Nat>, from: Text, opt_to: ?Text) {
+            ignore do ? {
+                let n1 = map.remove(from)!;
+                let to = opt_to!;
+                switch (map.get(to)) {
+                case null { map.put(to, n1) };
+                case (?n2) { map.put(to, n1 + n2) };
+                };
             };
-            // Not storing tags for create canister to avoid duplicate counting of tags
-            // addTags(origin.tags);
+        };
+        public func merge_tag(from: Text, to: ?Text) {
+            merge_tag_(canisters, from, to);
+            merge_tag_(installs, from, to);
+        };
+        public func addCanister(origin: Origin) {
+            addTags(canisters, ["origin:" # origin.origin]);
+            addTags(canisters, origin.tags);
         };
         public func addInstall(origin: Origin) {
-            switch (installs.get(origin.origin)) {
-            case null { installs.put(origin.origin, 1) };
-            case (?n) { installs.put(origin.origin, n + 1) };
-            };
-            // Only record tags for canister install
-            addTags(origin.tags);
+            addTags(installs, ["origin:" # origin.origin]);
+            addTags(installs, origin.tags);
         };
-        public func dump() : ([(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
+        public func dump() : ([(Text, Nat)], [(Text, Nat)]) {
             (toArray<(Text, Nat)>(canisters.entries()),
              toArray<(Text, Nat)>(installs.entries()),
-             toArray<(Text, Nat)>(tags.entries())
             )
         };
         public func metrics() : Text {
             var result = "";
             let now = timeNow() / 1_000_000;
-            for ((origin, count) in canisters.entries()) {
-                let name = "canisters_" # origin;
-                let desc = "Number of canisters requested from " # origin;
-                result := result # encode_single_value("counter", name, count, desc, now);
-            };
-            for ((origin, count) in installs.entries()) {
-                let name = "installs_" # origin;
-                let desc = "Number of Wasm installed from " # origin;
-                result := result # encode_single_value("counter", name, count, desc, now);
-            };
-            let profiling = get(tags.get("profiling"), 0);
-            let asset = get(tags.get("asset"), 0);
-            let install = get(tags.get("install"), 0);
-            let reinstall = get(tags.get("reinstall"), 0);
-            let upgrade = get(tags.get("upgrade"), 0);
+            let canister_playground = get(canisters.get("origin:playground"), 0);
+            let canister_dfx = get(canisters.get("origin:dfx"), 0);
+            let install_playground = get(installs.get("origin:playground"), 0);
+            let install_dfx = get(installs.get("origin:dfx"), 0);
+            let profiling = get(installs.get("wasm:profiling"), 0);
+            let asset = get(installs.get("wasm:asset"), 0);
+            let install = get(installs.get("mode:install"), 0);
+            let reinstall = get(installs.get("mode:reinstall"), 0);
+            let upgrade = get(installs.get("mode:upgrade"), 0);
             result := result
-            # encode_single_value("counter", "profiling", profiling, "Number of Wasm profiled", now)
-            # encode_single_value("counter", "asset", asset, "Number of asset Wasm canister installed", now)
-            # encode_single_value("counter", "install", install, "Number of Wasm with install mode", now)
-            # encode_single_value("counter", "reinstall", reinstall, "Number of Wasm with reinstall mode", now)
-            # encode_single_value("counter", "upgrade", upgrade, "Number of Wasm with upgrad mode", now);
+            # encode_single_value("counter", "create_from_playground", canister_playground, "Number of canisters created from playground", now)
+            # encode_single_value("counter", "install_from_playground", install_playground, "Number of Wasms installed from playground", now)
+            # encode_single_value("counter", "create_from_dfx", canister_dfx, "Number of canisters created from dfx", now)
+            # encode_single_value("counter", "install_from_dfx", install_dfx, "Number of Wasms installed from dfx", now)
+            # encode_single_value("counter", "profiling", profiling, "Number of Wasms profiled", now)
+            # encode_single_value("counter", "asset", asset, "Number of asset Wasms canister installed", now)
+            # encode_single_value("counter", "install", install, "Number of Wasms with install mode", now)
+            # encode_single_value("counter", "reinstall", reinstall, "Number of Wasms with reinstall mode", now)
+            # encode_single_value("counter", "upgrade", upgrade, "Number of Wasms with upgrad mode", now);
             result;
         };
     };

service/pool/Main.mo

@@ -33,7 +33,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
     stable var stableChildren : [(Principal, [Principal])] = [];
     stable var stableTimers : [Types.CanisterInfo] = [];
     stable var previousParam : ?Types.InitParams = null;
-    stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf, #leaf);
+    stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf);
 
     system func preupgrade() {
         let (tree, metadata, children, timers) = pool.share();
@@ -62,9 +62,9 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
         params;
     };
 
-    public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
-        let (canister, install, tags) = statsByOrigin.dump();
-        (stats, canister, install, tags);
+    public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)]) {
+        let (canister, install) = statsByOrigin.dump();
+        (stats, canister, install);
     };
 
     public query func balance() : async Nat {
@@ -174,15 +174,15 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             // Build tags from install arguments
             let tags = Buffer.fromArray<Text>(install_config.origin.tags);
             if (install_config.profiling) {
-                tags.add("profiling");
+                tags.add("wasm:profiling");
             };
             if (install_config.is_whitelisted) {
-                tags.add("asset");
+                tags.add("wasm:asset");
             };
             switch (args.mode) {
-            case (#install) { tags.add("install") };
-            case (#upgrade) { tags.add("upgrade") };
-            case (#reinstall) { tags.add("reinstall") };
+            case (#install) { tags.add("mode:install") };
+            case (#upgrade) { tags.add("mode:upgrade") };
+            case (#reinstall) { tags.add("mode:reinstall") };
             };
             let origin = { origin = install_config.origin.origin; tags = Buffer.toArray(tags) };
             statsByOrigin.addInstall(origin);
@@ -274,6 +274,12 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
         stats := Logs.defaultStats;
         statsByOrigin := Logs.StatsByOrigin();
     };
+    public shared ({ caller }) func mergeTags(from: Text, to: ?Text) : async () {
+        if (caller != controller) {
+            throw Error.reject "Only called by controller";
+        };
+        statsByOrigin.merge_tag(from, to);
+    };
 
     // Metrics
     public query func http_request(req : Metrics.HttpRequest) : async Metrics.HttpResponse {
@@ -427,6 +433,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             #installCode : Any;
             #removeCode : Any;
             #resetStats : Any;
+            #mergeTags : Any;
             #wallet_receive : Any;
 
             #create_canister : Any;

src/App.tsx

@@ -82,7 +82,7 @@ async function fetchFromUrlParams(
       const { origin, files } = result;
       await dispatch({
         type: "setOrigin",
-        payload: { origin: `playground:post:${origin}`, tags: [] },
+        payload: { origin: "playground", tags: [`post:${origin}`] },
       });
       return files;
     }
@@ -95,7 +95,7 @@ async function fetchFromUrlParams(
     };
     await dispatch({
       type: "setOrigin",
-      payload: { origin: "playground:git", tags: [`git:${git}`] },
+      payload: { origin: "playground", tags: [`git:${git}`] },
     });
     return await worker.fetchGithub(repo);
   }
@@ -147,7 +147,7 @@ async function fetchFromUrlParams(
       }
       await dispatch({
         type: "setOrigin",
-        payload: { origin: "playground:tag", tags: [`tag:${tag}`] },
+        payload: { origin: "playground", tags: [`tag:${tag}`] },
       });
       return files;
     }
@@ -296,6 +296,7 @@ export function App() {
             isFirstOpen={isFirstVisit}
           />
           <DeployModal
+            state={workplaceState}
             isOpen={showDeployModal}
             close={() => setShowDeployModal(false)}
             onDeploy={deployWorkplace}

src/Stats.tsx

@@ -0,0 +1,102 @@
+import { useState, useEffect } from "react";
+import { backend } from "./config/actor";
+import { Chart, Pie } from "./components/Chart";
+
+function extract_slice(raw, cond) {
+  const res = raw
+    .filter(([name, _]) => cond(name))
+    .map(([name, n]) => [name, Number(n)]);
+  res.sort((a, b) => b[1] - a[1]);
+  return res;
+}
+function join_slice(left, right) {
+  const l = Object.fromEntries(left);
+  const r = Object.fromEntries(right);
+  const full_key = [
+    ...new Set(
+      left.map(([name, _]) => name).concat(right.map(([name, _]) => name))
+    ),
+  ];
+  const res = full_key.map((name) => [name, l[name] || 0, r[name] || 0]);
+  res.sort((a, b) => b[2] - a[2]);
+  return res;
+}
+function two_metric(canisters, installs, title, cond) {
+  const new_canister = extract_slice(canisters, cond);
+  const wasm = extract_slice(installs, cond);
+  return [[title, "Canister", "Wasm"]].concat(join_slice(new_canister, wasm));
+}
+function one_metric(map, title, cond) {
+  const slice = extract_slice(map, cond);
+  return [[title, "Wasm"]].concat(slice);
+}
+
+export function Stats() {
+  const [example, setExample] = useState([]);
+  const [moc, setMoc] = useState([]);
+  const [ref, setRef] = useState([]);
+  const [origin, setOrigin] = useState([]);
+  const [imports, setImports] = useState([]);
+  const [wasm, setWasm] = useState([]);
+  const [mode, setMode] = useState([]);
+  const [packages, setPackages] = useState([]);
+
+  useEffect(() => {
+    async function doit() {
+      // eslint-disable-next-line
+      const [_, canisters, installs] = await backend.getStats();
+      setExample(
+        two_metric(
+          canisters,
+          installs,
+          "Example",
+          (name) => name.startsWith("example:") || name.startsWith("file:new")
+        )
+      );
+      setRef(
+        two_metric(canisters, installs, "Reference link", (name) =>
+          name.startsWith("ref:")
+        )
+      );
+      setOrigin(
+        two_metric(canisters, installs, "Origin", (name) =>
+          name.startsWith("origin:")
+        )
+      );
+      setImports(
+        two_metric(
+          canisters,
+          installs,
+          "Playground imports",
+          (name) =>
+            name.startsWith("post:") ||
+            name.startsWith("git:") ||
+            name.startsWith("tag:") ||
+            name.startsWith("upload:wasm")
+        )
+      );
+      setMoc(one_metric(installs, "Moc", (name) => name.startsWith("moc:")));
+      setWasm(one_metric(installs, "Wasm", (name) => name.startsWith("wasm:")));
+      setMode(
+        one_metric(installs, "Install mode", (name) => name.startsWith("mode:"))
+      );
+      setPackages(
+        one_metric(installs, "Imports", (name) => name.startsWith("import:"))
+      );
+    }
+    doit();
+  }, []);
+
+  return (
+    <>
+      <Pie title="Install mode" data={mode} />
+      <Chart title="Example" data={example} />
+      <Chart title="Reference link" data={ref} />
+      <Chart title="Origin" data={origin} />
+      <Chart title="Playground code imports" data={imports} />
+      <Chart title="External package/canister imports" data={packages} />
+      <Chart title="Moc" data={moc} />
+      <Chart title="Wasm" data={wasm} />
+    </>
+  );
+}

src/build.ts

@@ -154,16 +154,22 @@ export async function deploy(
   }
 }
 
+function mkOrigin(origin: Origin, is_install: boolean) {
+  const tags =
+    origin.session_tags && is_install
+      ? origin.tags.concat(origin.session_tags)
+      : origin.tags;
+  return { origin: origin.origin, tags: [...new Set(tags)] };
+}
+
 async function createCanister(
   worker,
   logger: ILoggingStore,
   origin: Origin
 ): Promise<CanisterInfo> {
   const timestamp = BigInt(Date.now()) * BigInt(1_000_000);
   const nonce = await worker.pow(timestamp);
-  // remove tags for create canister to avoid duplicate counting
-  const no_tags = { origin: origin.origin, tags: [] };
-  const info = await backend.getCanisterId(nonce, no_tags);
+  const info = await backend.getCanisterId(nonce, mkOrigin(origin, false));
   logger.log(`Got canister id ${info.id}`);
   return {
     id: info.id,
@@ -198,7 +204,7 @@ async function install(
   const installConfig = {
     profiling,
     is_whitelisted: false,
-    origin,
+    origin: mkOrigin(origin, true),
   };
   const new_info = await backend.installCode(
     canisterInfo,

src/components/CanisterModal.tsx

@@ -110,7 +110,7 @@ export function CanisterModal({ isOpen, close, deploySetter }) {
     await deploySetter.setShowDeployModal(true);
     await dispatch({
       type: "setOrigin",
-      payload: { origin: "playground:wasm", tags: [] },
+      payload: { origin: "playground", tags: [`upload:wasm`] },
     });
   }
   async function addCanister() {