Skip to content

Commit

Permalink
refactor stats (#195)
Browse files Browse the repository at this point in the history 
* logs contains only canister and install level key-value pairs

* fix

* mergeTags

* babystep for stats endpoint

* fix

* fix

* fix

* track packages

* Update src/Stats.tsx

Co-authored-by: Ryan Vandersmith <[email protected]>

* validate origin

* checkpoint

* refactor

* remove origin slice from grafana, since we have stats page

---------

Co-authored-by: Ryan Vandersmith <[email protected]>
  • Loading branch information
@chenyan-dfinity @rvanasa
chenyan-dfinity and rvanasa authored Sep 12, 2023
1 parent 856f44f commit 4793ef1
Show file tree
Hide file tree
Showing 17 changed files with 559 additions and 225 deletions.
288 changes: 152 additions & 136 deletions package-lock.json
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"prettier-plugin-motoko": "^0.8.1",
"react": "^17.0.2",
"react-dom": "^17.0.2",
"react-google-charts": "^4.0.1",
"react-markdown": "^6.0.2",
"react-modal": "^3.14.3",
"styled-components": "^5.3.0",
Expand Down
86 changes: 44 additions & 42 deletions service/pool/Logs.mo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,71 +7,73 @@ import {get} "mo:base/Option";

module {
public type Origin = { origin: Text; tags: [Text] };
public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>, Map.Tree<Text, Nat>);
public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>);
public class StatsByOrigin() {
var canisters = Map.RBTree<Text, Nat>(compare);
var installs = Map.RBTree<Text, Nat>(compare);
var tags = Map.RBTree<Text, Nat>(compare);
public func share() : SharedStatsByOrigin = (canisters.share(), installs.share(), tags.share());
public func share() : SharedStatsByOrigin = (canisters.share(), installs.share());
public func unshare(x : SharedStatsByOrigin) {
canisters.unshare(x.0);
installs.unshare(x.1);
tags.unshare(x.2);
};
func addTags(list: [Text]) {
func addTags(map: Map.RBTree<Text,Nat>, list: [Text]) {
for (tag in list.vals()) {
switch (tags.get(tag)) {
case null { tags.put(tag, 1) };
case (?n) { tags.put(tag, n + 1) };
switch (map.get(tag)) {
case null { map.put(tag, 1) };
case (?n) { map.put(tag, n + 1) };
};
};
};
public func addCanister(origin: Origin) {
switch (canisters.get(origin.origin)) {
case null { canisters.put(origin.origin, 1) };
case (?n) { canisters.put(origin.origin, n + 1) };
// if to is null, delete the from tag
func merge_tag_(map: Map.RBTree<Text,Nat>, from: Text, opt_to: ?Text) {
ignore do ? {
let n1 = map.remove(from)!;
let to = opt_to!;
switch (map.get(to)) {
case null { map.put(to, n1) };
case (?n2) { map.put(to, n1 + n2) };
};
};
// Not storing tags for create canister to avoid duplicate counting of tags
// addTags(origin.tags);
};
public func merge_tag(from: Text, to: ?Text) {
merge_tag_(canisters, from, to);
merge_tag_(installs, from, to);
};
public func addCanister(origin: Origin) {
addTags(canisters, ["origin:" # origin.origin]);
addTags(canisters, origin.tags);
};
public func addInstall(origin: Origin) {
switch (installs.get(origin.origin)) {
case null { installs.put(origin.origin, 1) };
case (?n) { installs.put(origin.origin, n + 1) };
};
// Only record tags for canister install
addTags(origin.tags);
addTags(installs, ["origin:" # origin.origin]);
addTags(installs, origin.tags);
};
public func dump() : ([(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
public func dump() : ([(Text, Nat)], [(Text, Nat)]) {
(toArray<(Text, Nat)>(canisters.entries()),
toArray<(Text, Nat)>(installs.entries()),
toArray<(Text, Nat)>(tags.entries())
)
};
public func metrics() : Text {
var result = "";
let now = timeNow() / 1_000_000;
for ((origin, count) in canisters.entries()) {
let name = "canisters_" # origin;
let desc = "Number of canisters requested from " # origin;
result := result # encode_single_value("counter", name, count, desc, now);
};
for ((origin, count) in installs.entries()) {
let name = "installs_" # origin;
let desc = "Number of Wasm installed from " # origin;
result := result # encode_single_value("counter", name, count, desc, now);
};
let profiling = get(tags.get("profiling"), 0);
let asset = get(tags.get("asset"), 0);
let install = get(tags.get("install"), 0);
let reinstall = get(tags.get("reinstall"), 0);
let upgrade = get(tags.get("upgrade"), 0);
let canister_playground = get(canisters.get("origin:playground"), 0);
let canister_dfx = get(canisters.get("origin:dfx"), 0);
let install_playground = get(installs.get("origin:playground"), 0);
let install_dfx = get(installs.get("origin:dfx"), 0);
let profiling = get(installs.get("wasm:profiling"), 0);
let asset = get(installs.get("wasm:asset"), 0);
let install = get(installs.get("mode:install"), 0);
let reinstall = get(installs.get("mode:reinstall"), 0);
let upgrade = get(installs.get("mode:upgrade"), 0);
result := result
# encode_single_value("counter", "profiling", profiling, "Number of Wasm profiled", now)
# encode_single_value("counter", "asset", asset, "Number of asset Wasm canister installed", now)
# encode_single_value("counter", "install", install, "Number of Wasm with install mode", now)
# encode_single_value("counter", "reinstall", reinstall, "Number of Wasm with reinstall mode", now)
# encode_single_value("counter", "upgrade", upgrade, "Number of Wasm with upgrad mode", now);
# encode_single_value("counter", "create_from_playground", canister_playground, "Number of canisters created from playground", now)
# encode_single_value("counter", "install_from_playground", install_playground, "Number of Wasms installed from playground", now)
# encode_single_value("counter", "create_from_dfx", canister_dfx, "Number of canisters created from dfx", now)
# encode_single_value("counter", "install_from_dfx", install_dfx, "Number of Wasms installed from dfx", now)
# encode_single_value("counter", "profiling", profiling, "Number of Wasms profiled", now)
# encode_single_value("counter", "asset", asset, "Number of asset Wasms canister installed", now)
# encode_single_value("counter", "install", install, "Number of Wasms with install mode", now)
# encode_single_value("counter", "reinstall", reinstall, "Number of Wasms with reinstall mode", now)
# encode_single_value("counter", "upgrade", upgrade, "Number of Wasms with upgrad mode", now);
result;
};
};
Expand Down
47 changes: 33 additions & 14 deletions service/pool/Main.mo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
stable var stableChildren : [(Principal, [Principal])] = [];
stable var stableTimers : [Types.CanisterInfo] = [];
stable var previousParam : ?Types.InitParams = null;
stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf, #leaf);
stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf);

system func preupgrade() {
let (tree, metadata, children, timers) = pool.share();
Expand Down Expand Up @@ -62,9 +62,9 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
params;
};

public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
let (canister, install, tags) = statsByOrigin.dump();
(stats, canister, install, tags);
public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)]) {
let (canister, install) = statsByOrigin.dump();
(stats, canister, install);
};

public query func balance() : async Nat {
Expand Down Expand Up @@ -118,10 +118,22 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
};
};
func validateOrigin(origin: Logs.Origin) : Bool {
if (origin.origin == "") {
return false;
};
for (tag in origin.tags.vals()) {
// reject server side tags
if (tag == "mode:install" or tag == "mode:reinstall" or tag == "mode:upgrade" or tag == "wasm:profiling" or tag == "wasm:asset") {
return false;
}
};
return true;
};

public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce, origin : Logs.Origin) : async Types.CanisterInfo {
if (origin.origin == "") {
throw Error.reject "Please specify an origin";
if (not validateOrigin(origin)) {
throw Error.reject "Please specify a valid origin";
};
if (caller != controller and not nonceCache.checkProofOfWork(nonce)) {
stats := Logs.updateStats(stats, #mismatch);
Expand All @@ -138,8 +150,8 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {

type InstallConfig = { profiling: Bool; is_whitelisted: Bool; origin: Logs.Origin };
public shared ({ caller }) func installCode(info : Types.CanisterInfo, args : Types.InstallArgs, install_config : InstallConfig) : async Types.CanisterInfo {
if (install_config.origin.origin == "") {
throw Error.reject "Please specify an origin";
if (not validateOrigin(install_config.origin)) {
throw Error.reject "Please specify a valid origin";
};
if (info.timestamp == 0) {
stats := Logs.updateStats(stats, #mismatch);
Expand Down Expand Up @@ -174,15 +186,15 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
// Build tags from install arguments
let tags = Buffer.fromArray<Text>(install_config.origin.tags);
if (install_config.profiling) {
tags.add("profiling");
tags.add("wasm:profiling");
};
if (install_config.is_whitelisted) {
tags.add("asset");
tags.add("wasm:asset");
};
switch (args.mode) {
case (#install) { tags.add("install") };
case (#upgrade) { tags.add("upgrade") };
case (#reinstall) { tags.add("reinstall") };
case (#install) { tags.add("mode:install") };
case (#upgrade) { tags.add("mode:upgrade") };
case (#reinstall) { tags.add("mode:reinstall") };
};
let origin = { origin = install_config.origin.origin; tags = Buffer.toArray(tags) };
statsByOrigin.addInstall(origin);
Expand Down Expand Up @@ -274,11 +286,17 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
stats := Logs.defaultStats;
statsByOrigin := Logs.StatsByOrigin();
};
public shared ({ caller }) func mergeTags(from: Text, to: ?Text) : async () {
if (caller != controller) {
throw Error.reject "Only called by controller";
};
statsByOrigin.merge_tag(from, to);
};

// Metrics
public query func http_request(req : Metrics.HttpRequest) : async Metrics.HttpResponse {
if (req.url == "/metrics") {
let body = Metrics.metrics(stats, statsByOrigin);
let body = Metrics.metrics(stats);
{
status_code = 200;
headers = [("Content-Type", "text/plain; version=0.0.4"), ("Content-Length", Nat.toText(body.size()))];
Expand Down Expand Up @@ -427,6 +445,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
#installCode : Any;
#removeCode : Any;
#resetStats : Any;
#mergeTags : Any;
#wallet_receive : Any;

#create_canister : Any;
Expand Down
3 changes: 1 addition & 2 deletions service/pool/Metrics.mo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module {
body: Blob;
};
let encode_single_value = Logs.encode_single_value;
public func metrics(stats: Logs.Stats, origin: Logs.StatsByOrigin) : Blob {
public func metrics(stats: Logs.Stats) : Blob {
let now = Time.now() / 1_000_000;
var result = "";
result := result # encode_single_value("counter", "canister_count", stats.num_of_canisters, "Number of canisters deployed", now);
Expand All @@ -24,7 +24,6 @@ module {
result := result # encode_single_value("counter", "out_of_capacity", stats.error_out_of_capacity, "Number of out of capacity requests", now);
result := result # encode_single_value("counter", "total_wait_time", stats.error_total_wait_time, "Number of seconds waiting for out of capacity requests", now);
result := result # encode_single_value("counter", "mismatch", stats.error_mismatch, "Number of mismatch requests including wrong nounce and timestamp", now);
result := result # origin.metrics();
Text.encodeUtf8(result)
};
}
7 changes: 4 additions & 3 deletions src/App.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ async function fetchFromUrlParams(
const { origin, files } = result;
await dispatch({
type: "setOrigin",
payload: { origin: `playground:post:${origin}`, tags: [] },
payload: { origin: "playground", tags: [`post:${origin}`] },
});
return files;
}
Expand All @@ -95,7 +95,7 @@ async function fetchFromUrlParams(
};
await dispatch({
type: "setOrigin",
payload: { origin: "playground:git", tags: [`git:${git}`] },
payload: { origin: "playground", tags: [`git:${git}`] },
});
return await worker.fetchGithub(repo);
}
Expand Down Expand Up @@ -147,7 +147,7 @@ async function fetchFromUrlParams(
}
await dispatch({
type: "setOrigin",
payload: { origin: "playground:tag", tags: [`tag:${tag}`] },
payload: { origin: "playground", tags: [`tag:${tag}`] },
});
return files;
}
Expand Down Expand Up @@ -296,6 +296,7 @@ export function App() {
isFirstOpen={isFirstVisit}
/>
<DeployModal
state={workplaceState}
isOpen={showDeployModal}
close={() => setShowDeployModal(false)}
onDeploy={deployWorkplace}
Expand Down
Loading

0 comments on commit 4793ef1

Please sign in to comment.