Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor stats #195

Merged
merged 13 commits into from
Sep 12, 2023
Merged

refactor stats #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6065800
logs contains only canister and install level key-value pairs
chenyan-dfinity Sep 10, 2023
b0b959f
fix
chenyan-dfinity Sep 10, 2023
83f29fe
mergeTags
chenyan-dfinity Sep 10, 2023
0570f01
babystep for stats endpoint
chenyan-dfinity Sep 11, 2023
aff3603
fix
chenyan-dfinity Sep 11, 2023
adb69b1
fix
chenyan-dfinity Sep 11, 2023
dfe69e2
fix
chenyan-dfinity Sep 11, 2023
cf5766b
track packages
chenyan-dfinity Sep 11, 2023
cfeecef
Update src/Stats.tsx
chenyan-dfinity Sep 11, 2023
ca13003
validate origin
chenyan-dfinity Sep 11, 2023
fb38b31
checkpoint
chenyan-dfinity Sep 12, 2023
faffb7b
refactor
chenyan-dfinity Sep 12, 2023
0261240
remove origin slice from grafana, since we have stats page
chenyan-dfinity Sep 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
288 changes: 152 additions & 136 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"prettier-plugin-motoko": "^0.8.1",
"react": "^17.0.2",
"react-dom": "^17.0.2",
"react-google-charts": "^4.0.1",
"react-markdown": "^6.0.2",
"react-modal": "^3.14.3",
"styled-components": "^5.3.0",
Expand Down
86 changes: 44 additions & 42 deletions service/pool/Logs.mo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,71 +7,73 @@ import {get} "mo:base/Option";

module {
public type Origin = { origin: Text; tags: [Text] };
public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>, Map.Tree<Text, Nat>);
public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>);
public class StatsByOrigin() {
var canisters = Map.RBTree<Text, Nat>(compare);
var installs = Map.RBTree<Text, Nat>(compare);
var tags = Map.RBTree<Text, Nat>(compare);
public func share() : SharedStatsByOrigin = (canisters.share(), installs.share(), tags.share());
public func share() : SharedStatsByOrigin = (canisters.share(), installs.share());
public func unshare(x : SharedStatsByOrigin) {
canisters.unshare(x.0);
installs.unshare(x.1);
tags.unshare(x.2);
};
func addTags(list: [Text]) {
func addTags(map: Map.RBTree<Text,Nat>, list: [Text]) {
for (tag in list.vals()) {
switch (tags.get(tag)) {
case null { tags.put(tag, 1) };
case (?n) { tags.put(tag, n + 1) };
switch (map.get(tag)) {
case null { map.put(tag, 1) };
case (?n) { map.put(tag, n + 1) };
};
};
};
public func addCanister(origin: Origin) {
switch (canisters.get(origin.origin)) {
case null { canisters.put(origin.origin, 1) };
case (?n) { canisters.put(origin.origin, n + 1) };
// if to is null, delete the from tag
func merge_tag_(map: Map.RBTree<Text,Nat>, from: Text, opt_to: ?Text) {
ignore do ? {
let n1 = map.remove(from)!;
let to = opt_to!;
switch (map.get(to)) {
case null { map.put(to, n1) };
case (?n2) { map.put(to, n1 + n2) };
};
};
// Not storing tags for create canister to avoid duplicate counting of tags
// addTags(origin.tags);
};
public func merge_tag(from: Text, to: ?Text) {
merge_tag_(canisters, from, to);
merge_tag_(installs, from, to);
};
public func addCanister(origin: Origin) {
addTags(canisters, ["origin:" # origin.origin]);
addTags(canisters, origin.tags);
};
public func addInstall(origin: Origin) {
switch (installs.get(origin.origin)) {
case null { installs.put(origin.origin, 1) };
case (?n) { installs.put(origin.origin, n + 1) };
};
// Only record tags for canister install
addTags(origin.tags);
addTags(installs, ["origin:" # origin.origin]);
addTags(installs, origin.tags);
};
public func dump() : ([(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
public func dump() : ([(Text, Nat)], [(Text, Nat)]) {
(toArray<(Text, Nat)>(canisters.entries()),
toArray<(Text, Nat)>(installs.entries()),
toArray<(Text, Nat)>(tags.entries())
)
};
public func metrics() : Text {
var result = "";
let now = timeNow() / 1_000_000;
for ((origin, count) in canisters.entries()) {
let name = "canisters_" # origin;
let desc = "Number of canisters requested from " # origin;
result := result # encode_single_value("counter", name, count, desc, now);
};
for ((origin, count) in installs.entries()) {
let name = "installs_" # origin;
let desc = "Number of Wasm installed from " # origin;
result := result # encode_single_value("counter", name, count, desc, now);
};
let profiling = get(tags.get("profiling"), 0);
let asset = get(tags.get("asset"), 0);
let install = get(tags.get("install"), 0);
let reinstall = get(tags.get("reinstall"), 0);
let upgrade = get(tags.get("upgrade"), 0);
let canister_playground = get(canisters.get("origin:playground"), 0);
let canister_dfx = get(canisters.get("origin:dfx"), 0);
let install_playground = get(installs.get("origin:playground"), 0);
let install_dfx = get(installs.get("origin:dfx"), 0);
let profiling = get(installs.get("wasm:profiling"), 0);
let asset = get(installs.get("wasm:asset"), 0);
let install = get(installs.get("mode:install"), 0);
let reinstall = get(installs.get("mode:reinstall"), 0);
let upgrade = get(installs.get("mode:upgrade"), 0);
result := result
# encode_single_value("counter", "profiling", profiling, "Number of Wasm profiled", now)
# encode_single_value("counter", "asset", asset, "Number of asset Wasm canister installed", now)
# encode_single_value("counter", "install", install, "Number of Wasm with install mode", now)
# encode_single_value("counter", "reinstall", reinstall, "Number of Wasm with reinstall mode", now)
# encode_single_value("counter", "upgrade", upgrade, "Number of Wasm with upgrad mode", now);
# encode_single_value("counter", "create_from_playground", canister_playground, "Number of canisters created from playground", now)
# encode_single_value("counter", "install_from_playground", install_playground, "Number of Wasms installed from playground", now)
# encode_single_value("counter", "create_from_dfx", canister_dfx, "Number of canisters created from dfx", now)
# encode_single_value("counter", "install_from_dfx", install_dfx, "Number of Wasms installed from dfx", now)
# encode_single_value("counter", "profiling", profiling, "Number of Wasms profiled", now)
# encode_single_value("counter", "asset", asset, "Number of asset Wasms canister installed", now)
# encode_single_value("counter", "install", install, "Number of Wasms with install mode", now)
# encode_single_value("counter", "reinstall", reinstall, "Number of Wasms with reinstall mode", now)
# encode_single_value("counter", "upgrade", upgrade, "Number of Wasms with upgrad mode", now);
result;
};
};
Expand Down
47 changes: 33 additions & 14 deletions service/pool/Main.mo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
stable var stableChildren : [(Principal, [Principal])] = [];
stable var stableTimers : [Types.CanisterInfo] = [];
stable var previousParam : ?Types.InitParams = null;
stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf, #leaf);
stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf);

system func preupgrade() {
let (tree, metadata, children, timers) = pool.share();
Expand Down Expand Up @@ -62,9 +62,9 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
params;
};

public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
let (canister, install, tags) = statsByOrigin.dump();
(stats, canister, install, tags);
public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)]) {
let (canister, install) = statsByOrigin.dump();
(stats, canister, install);
};

public query func balance() : async Nat {
Expand Down Expand Up @@ -118,10 +118,22 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
};
};
func validateOrigin(origin: Logs.Origin) : Bool {
if (origin.origin == "") {
return false;
};
for (tag in origin.tags.vals()) {
// reject server side tags
if (tag == "mode:install" or tag == "mode:reinstall" or tag == "mode:upgrade" or tag == "wasm:profiling" or tag == "wasm:asset") {
return false;
}
};
return true;
};

public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce, origin : Logs.Origin) : async Types.CanisterInfo {
if (origin.origin == "") {
throw Error.reject "Please specify an origin";
if (not validateOrigin(origin)) {
throw Error.reject "Please specify a valid origin";
};
if (caller != controller and not nonceCache.checkProofOfWork(nonce)) {
stats := Logs.updateStats(stats, #mismatch);
Expand All @@ -138,8 +150,8 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {

type InstallConfig = { profiling: Bool; is_whitelisted: Bool; origin: Logs.Origin };
public shared ({ caller }) func installCode(info : Types.CanisterInfo, args : Types.InstallArgs, install_config : InstallConfig) : async Types.CanisterInfo {
if (install_config.origin.origin == "") {
throw Error.reject "Please specify an origin";
if (not validateOrigin(install_config.origin)) {
throw Error.reject "Please specify a valid origin";
};
if (info.timestamp == 0) {
stats := Logs.updateStats(stats, #mismatch);
Expand Down Expand Up @@ -174,15 +186,15 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
// Build tags from install arguments
let tags = Buffer.fromArray<Text>(install_config.origin.tags);
if (install_config.profiling) {
tags.add("profiling");
tags.add("wasm:profiling");
};
if (install_config.is_whitelisted) {
tags.add("asset");
tags.add("wasm:asset");
};
switch (args.mode) {
case (#install) { tags.add("install") };
case (#upgrade) { tags.add("upgrade") };
case (#reinstall) { tags.add("reinstall") };
case (#install) { tags.add("mode:install") };
case (#upgrade) { tags.add("mode:upgrade") };
case (#reinstall) { tags.add("mode:reinstall") };
};
let origin = { origin = install_config.origin.origin; tags = Buffer.toArray(tags) };
statsByOrigin.addInstall(origin);
Expand Down Expand Up @@ -274,11 +286,17 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
stats := Logs.defaultStats;
statsByOrigin := Logs.StatsByOrigin();
};
public shared ({ caller }) func mergeTags(from: Text, to: ?Text) : async () {
if (caller != controller) {
throw Error.reject "Only called by controller";
};
statsByOrigin.merge_tag(from, to);
};

// Metrics
public query func http_request(req : Metrics.HttpRequest) : async Metrics.HttpResponse {
if (req.url == "/metrics") {
let body = Metrics.metrics(stats, statsByOrigin);
let body = Metrics.metrics(stats);
{
status_code = 200;
headers = [("Content-Type", "text/plain; version=0.0.4"), ("Content-Length", Nat.toText(body.size()))];
Expand Down Expand Up @@ -427,6 +445,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
#installCode : Any;
#removeCode : Any;
#resetStats : Any;
#mergeTags : Any;
#wallet_receive : Any;

#create_canister : Any;
Expand Down
3 changes: 1 addition & 2 deletions service/pool/Metrics.mo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module {
body: Blob;
};
let encode_single_value = Logs.encode_single_value;
public func metrics(stats: Logs.Stats, origin: Logs.StatsByOrigin) : Blob {
public func metrics(stats: Logs.Stats) : Blob {
let now = Time.now() / 1_000_000;
var result = "";
result := result # encode_single_value("counter", "canister_count", stats.num_of_canisters, "Number of canisters deployed", now);
Expand All @@ -24,7 +24,6 @@ module {
result := result # encode_single_value("counter", "out_of_capacity", stats.error_out_of_capacity, "Number of out of capacity requests", now);
result := result # encode_single_value("counter", "total_wait_time", stats.error_total_wait_time, "Number of seconds waiting for out of capacity requests", now);
result := result # encode_single_value("counter", "mismatch", stats.error_mismatch, "Number of mismatch requests including wrong nounce and timestamp", now);
result := result # origin.metrics();
Text.encodeUtf8(result)
};
}
7 changes: 4 additions & 3 deletions src/App.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ async function fetchFromUrlParams(
const { origin, files } = result;
await dispatch({
type: "setOrigin",
payload: { origin: `playground:post:${origin}`, tags: [] },
payload: { origin: "playground", tags: [`post:${origin}`] },
});
return files;
}
Expand All @@ -95,7 +95,7 @@ async function fetchFromUrlParams(
};
await dispatch({
type: "setOrigin",
payload: { origin: "playground:git", tags: [`git:${git}`] },
payload: { origin: "playground", tags: [`git:${git}`] },
});
return await worker.fetchGithub(repo);
}
Expand Down Expand Up @@ -147,7 +147,7 @@ async function fetchFromUrlParams(
}
await dispatch({
type: "setOrigin",
payload: { origin: "playground:tag", tags: [`tag:${tag}`] },
payload: { origin: "playground", tags: [`tag:${tag}`] },
});
return files;
}
Expand Down Expand Up @@ -296,6 +296,7 @@ export function App() {
isFirstOpen={isFirstVisit}
/>
<DeployModal
state={workplaceState}
isOpen={showDeployModal}
close={() => setShowDeployModal(false)}
onDeploy={deployWorkplace}
Expand Down
Loading
Loading