Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unit tests for ddc-validator-pallet #61

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pallets/ddc-staking/src/mock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ parameter_types! {
pub const DefaultEdgeChillDelay: EraIndex = 1;
pub const DefaultStorageBondSize: Balance = 100;
pub const DefaultStorageChillDelay: EraIndex = 1;
pub const DdcAccountsPalletId: PalletId = PalletId(*b"accounts");
}

impl crate::pallet::Config for Test {
Expand All @@ -107,6 +108,7 @@ impl crate::pallet::Config for Test {
type Event = Event;
type UnixTime = Timestamp;
type WeightInfo = ();
type StakersPayoutSource = DdcAccountsPalletId;
}

pub(crate) type DdcStakingCall = crate::Call<Test>;
Expand Down
10 changes: 7 additions & 3 deletions pallets/ddc-validator/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,6 @@ pub mod pallet {

let era = Self::get_current_era();
log::info!("current era: {:?}", era);

// Produce an assignment for the next era if it's not produced yet.
match Self::last_managed_era() {
Some(last_managed_era) if era < last_managed_era => (),
Expand Down Expand Up @@ -495,6 +494,10 @@ pub mod pallet {
}

fn is_valid(bytes_sent: u64, bytes_received: u64) -> bool {
if bytes_sent == bytes_received {
return true;
}

let percentage_difference = 1f32 - (bytes_received as f32 / bytes_sent as f32);

return if percentage_difference > 0.0 &&
Expand Down Expand Up @@ -661,9 +664,10 @@ pub mod pallet {

// form url for each node
let edge_url = format!(
"{}{}{}",
"{}{}{}/$.{}",
mock_data_url,
"ddc:dac:aggregation:nodes:132855/$.",
"ddc:dac:aggregation:nodes:",
current_era - 1,
utils::account_to_string::<T>(assigned_edge.clone())
);
info!("edge url: {:?}", edge_url);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JSON.GET":"[{\"totalBytesSent\":600,\"totalQueries\":0,\"totalReads\":3,\"totalReadsAcked\":3,\"totalQueriesAcked\":0,\"averageResponseTimeMs\":1.8852459016393444,\"totalBytesReceived\":600,\"requestIds\":[\"84640a53-fc1f-4ac5-921c-6695056840bc\",\"d0a55c8b-fcb9-41b5-aa9a-8b40e9c4edf7\",\"80a62530-fd76-40b5-bc53-dd82365e89ce\"],\"totalWritesAcked\":3,\"averageResponseTimeMsSamples\":61,\"totalWrites\":3}]"}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JSON.GET":"{\"fileRequestId\":\"84640a53-fc1f-4ac5-921c-6695056840bc\",\"fileInfo\":{\"chunkCids\":[\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcb2\"],\"requestedChunkCids\":[\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcb2\"]},\"bucketId\":5,\"timestamp\":1688473909701,\"chunks\":{\"84640a53-fc1f-4ac5-921c-6695056840bc:bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcb2\":{\"log\":{\"type\":1,\"signature\":null,\"aggregated\":1,\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\",\"era\":3,\"bucketId\":5,\"userAddress\":\"172.18.0.1:49690\",\"bytesSent\":100,\"timestamp\":1688473909701,\"nodePublicKey\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"sessionId\":\"6afL1/kz+7NInOaHjwfUWZuoBwSZ2qX9JDOuSULCENk=\"},\"cid\":\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcb2\",\"ack\":{\"userTimestamp\":1688473909709,\"nonce\":\"Gt7AFrznCPzgI/5q+tynLy2BKUooSGAkUCaGsF6AOMs=\",\"signature\":\"Gll6xIftbtP4JPB6miy3DLKMBDrKz05QSIBNJ3RqYi0Fg1wNJRFoSIqhOJDXDUB4NlrLQdl+HWrYRL20Hsx6iA==\",\"aggregated\":1,\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\",\"bytesReceived\":100,\"requestedChunkCids\":[\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcb2\"],\"nodePublicKey\":\"0101010101010101010101010101010101010101010101010101010101010101\"}}},\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\"}"}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JSON.GET":"{\"fileRequestId\":\"d0a55c8b-fcb9-41b5-aa9a-8b40e9c4edf7\",\"fileInfo\":{\"chunkCids\":[\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcd2\"],\"requestedChunkCids\":[\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcd2\"]},\"bucketId\":5,\"timestamp\":1688473909723,\"chunks\":{\"d0a55c8b-fcb9-41b5-aa9a-8b40e9c4edf7:bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcd2\":{\"log\":{\"type\":2,\"signature\":null,\"aggregated\":1,\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\",\"era\":3,\"bucketId\":5,\"userAddress\":\"172.18.0.1:49714\",\"bytesSent\":200,\"timestamp\":1688473909723,\"nodePublicKey\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"sessionId\":\"6afL1/kz+7NInOaHjwfUWZuoBwSZ2qX9JDOuSULCENk=\"},\"cid\":\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcd2\",\"ack\":{\"userTimestamp\":1688473909726,\"nonce\":\"2mdA3QVq02ME77vnqGjBNQqTRhM5gHjVAWcroNd5IZA=\",\"signature\":\"6PIWJXyEyHfWZrX6HKFNj3fRm5LbBsBX54zmVZpo+nMqqTV26xaQbEsRTfuhm4k+XGxci3VSuofPWFEpRuOmhg==\",\"aggregated\":1,\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\",\"bytesReceived\":200,\"requestedChunkCids\":[\"bafk2bzaceds3kr47j7imur2krbwhydosbocto73kx3erhmknjelpk6doltcd2\"],\"nodePublicKey\":\"0101010101010101010101010101010101010101010101010101010101010101\"}}},\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\"}"}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JSON.GET":"{\"fileRequestId\":\"80a62530-fd76-40b5-bc53-dd82365e89ce\",\"fileInfo\":{\"chunkCids\":[\"bafk2bzaced573lhaxdjlztd5xawkeakiz7j7dayrqbzrx5shjobordrrrlio4\"],\"requestedChunkCids\":[\"bafk2bzaced573lhaxdjlztd5xawkeakiz7j7dayrqbzrx5shjobordrrrlio4\"]},\"bucketId\":5,\"timestamp\":1688473910041,\"chunks\":{\"80a62530-fd76-40b5-bc53-dd82365e89ce:bafk2bzaced573lhaxdjlztd5xawkeakiz7j7dayrqbzrx5shjobordrrrlio4\":{\"log\":{\"type\":1,\"signature\":null,\"aggregated\":1,\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\",\"era\":3,\"bucketId\":5,\"userAddress\":\"172.18.0.1:49736\",\"bytesSent\":300,\"timestamp\":1688473910041,\"nodePublicKey\":\"0101010101010101010101010101010101010101010101010101010101010101\",\"sessionId\":\"6afL1/kz+7NInOaHjwfUWZuoBwSZ2qX9JDOuSULCENk=\"},\"cid\":\"bafk2bzaced573lhaxdjlztd5xawkeakiz7j7dayrqbzrx5shjobordrrrlio4\",\"ack\":{\"userTimestamp\":1688473910043,\"nonce\":\"VU1XCPjQxA4y6TnrOAy44QabS7nmmxAU8vyduqoLt9U=\",\"signature\":\"Us7//t0+y00oHhthPSjqphHJYwE1qgPQtBSdy5hZxUInKdQXTBX58VLQekoKMHtptQQggR1gPf9Pyy1enmXziQ==\",\"aggregated\":1,\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\",\"bytesReceived\":300,\"requestedChunkCids\":[\"bafk2bzaced573lhaxdjlztd5xawkeakiz7j7dayrqbzrx5shjobordrrrlio4\"],\"nodePublicKey\":\"0101010101010101010101010101010101010101010101010101010101010101\"}}},\"userPublicKey\":\"knW4sRsf5iyoE+bP+duWD8KEYiRddtPDQ7Hpx1KLyEM=\"}"}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JSON.GET":null}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JSON.GET":"{\"0101010101010101010101010101010101010101010101010101010101010101\":{\"d2bf4b844dfefd6772a8843e669f943408966a977e3ae2af1dd78e0f55f4df67\":{\"result\":true,\"data\":\"eyJlZGdlIjoiMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMTAxMDEwMSIsInJlc3VsdCI6dHJ1ZSwicGF5bG9hZCI6WzI0MiwgMjIxLCA1MSwgMzQsIDI5LCAyNDIsIDE1MCwgMTE5LCA2LCAxMzEsIDQxLCAxNSwgMTYxLCAxNzMsIDEyMSwgNDMsIDIyMSwgMjIsIDE1MCwgMTczLCAxODAsIDIyNCwgMTc5LCA1OCwgNzIsIDQsIDk2LCAxODgsIDE3LCAxNjQsIDE3NywgMTA5XSwidG90YWxzIjp7InJlY2VpdmVkIjo2MDAsInNlbnQiOjYwMCwiZmFpbGVkX2J5X2NsaWVudCI6MCwiZmFpbHVyZV9yYXRlIjowfX0=\"}},\"1c4a1b081af8dd09096ebb6e7ad61dd549ac2931cdb2b1216589094ad71ca90b\":{\"d2bf4b844dfefd6772a8843e669f943408966a977e3ae2af1dd78e0f55f4df67\":{\"result\":false,\"data\":\"eyJlZGdlIjoiMWM0YTFiMDgxYWY4ZGQwOTA5NmViYjZlN2FkNjFkZDU0OWFjMjkzMWNkYjJiMTIxNjU4OTA5NGFkNzFjYTkwYiIsInJlc3VsdCI6ZmFsc2UsInBheWxvYWQiOlsxMDQsMjM4LDczLDczLDIwNCwxNzIsMjU0LDE4MSw3NywxMTYsMTM2LDE4OSwyNDYsOTAsMTY0LDE1NCwxNjcsMywxNTUsMjUzLDgwLDMzLDI1MSwxNCw3OCwyMzYsMTY3LDEzNCwxNjEsNjQsMjIyLDE4MV0sInRvdGFscyI6eyJyZWNlaXZlZCI6OTAwLCJzZW50Ijo1NDQsImZhaWxlZF9ieV9jbGllbnQiOjAsImZhaWx1cmVfcmF0ZSI6MH19\"}}}"}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"edge":"0101010101010101010101010101010101010101010101010101010101010101","result":true,"payload":[242, 221, 51, 34, 29, 242, 150, 119, 6, 131, 41, 15, 161, 173, 121, 43, 221, 22, 150, 173, 180, 224, 179, 58, 72, 4, 96, 188, 17, 164, 177, 109],"totals":{"received":600,"sent":600,"failed_by_client":0,"failure_rate":0}}
69 changes: 67 additions & 2 deletions pallets/ddc-validator/src/mock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ use frame_support::{
weights::Weight,
PalletId,
};
use frame_system::{offchain::SendTransactionTypes, EnsureRoot};
use frame_system::{offchain::SendTransactionTypes};
use pallet_contracts as contracts;
use pallet_session::ShouldEndSession;
use sp_core::H256;
use sp_io::TestExternalities;
use sp_runtime::{
curve,
curve::PiecewiseLinear,
Expand Down Expand Up @@ -317,7 +318,71 @@ impl pallet_balances::Config for Test {

// Build genesis storage according to the mock runtime.
pub fn new_test_ext() -> sp_io::TestExternalities {
frame_system::GenesisConfig::default().build_storage::<Test>().unwrap().into()
let mut storage = frame_system::GenesisConfig::default().build_storage::<Test>().unwrap();

let balances = vec![
// edge stash
(AccountId::from([0x1; 32]), 1000),
// edge controller
(AccountId::from([0x11; 32]), 1000),

// validator1 stash; has to be equal to the OCW key in the current implementation
(AccountId::from([0xd2, 0xbf, 0x4b, 0x84, 0x4d, 0xfe, 0xfd, 0x67, 0x72, 0xa8, 0x84, 0x3e, 0x66, 0x9f, 0x94, 0x34, 0x08, 0x96, 0x6a, 0x97, 0x7e, 0x3a, 0xe2, 0xaf, 0x1d, 0xd7, 0x8e, 0x0f, 0x55, 0xf4, 0xdf, 0x67]), 10000),
// validator1 controller
(AccountId::from([0xaa; 32]), 10000),

// validator2 stash
(AccountId::from([0xb; 32]), 10000),
// validator2 controller
(AccountId::from([0xbb; 32]), 10000),

// validator3 stash
(AccountId::from([0xc; 32]), 10000),
// validator3 controller
(AccountId::from([0xcc; 32]), 10000),
];
let _ = pallet_balances::GenesisConfig::<Test> { balances }
.assimilate_storage(&mut storage);

let stakers = vec![
(
AccountId::from([0xd2, 0xbf, 0x4b, 0x84, 0x4d, 0xfe, 0xfd, 0x67, 0x72, 0xa8, 0x84, 0x3e, 0x66, 0x9f, 0x94, 0x34, 0x08, 0x96, 0x6a, 0x97, 0x7e, 0x3a, 0xe2, 0xaf, 0x1d, 0xd7, 0x8e, 0x0f, 0x55, 0xf4, 0xdf, 0x67]),
AccountId::from([0xaa; 32]),
1000,
pallet_staking::StakerStatus::Validator
),

(
AccountId::from([0xb; 32]),
AccountId::from([0xbb; 32]),
1000,
pallet_staking::StakerStatus::Validator
),

(
AccountId::from([0xc; 32]),
AccountId::from([0xcc; 32]),
1000,
pallet_staking::StakerStatus::Validator
)
];
let _ = pallet_staking::GenesisConfig::<Test> { stakers, ..Default::default() }
.assimilate_storage(&mut storage);

let edges = vec![
(
AccountId::from([0x1; 32]),
AccountId::from([0x11; 32]),
100,
1
)
];
let storages = vec![];
let _ = pallet_ddc_staking::GenesisConfig::<Test> { edges, storages, ..Default::default() }
.assimilate_storage(&mut storage);

TestExternalities::new(storage)

}

pub type Extrinsic = TestXt<Call, ()>;
Expand Down
177 changes: 170 additions & 7 deletions pallets/ddc-validator/src/tests.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,174 @@
use crate::mock::*;
use frame_support::assert_ok;
use sp_core::crypto::AccountId32;
use sp_runtime::DispatchResult;
use crate::mock::Timestamp;
use crate::shm;
use crate::utils;
use crate::{EraIndex, ValidationDecision, DacTotalAggregates, KEY_TYPE, DEFAULT_DATA_PROVIDER_URL, TIME_START_MS, ERA_DURATION_MS, ERA_IN_BLOCKS};
use pallet_ddc_accounts::BucketsDetails;
use frame_support::{assert_ok, traits::{OffchainWorker, OnInitialize}};
use sp_core::{
offchain::{testing, OffchainWorkerExt, OffchainDbExt, TransactionPoolExt},
};
use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStore};
use std::sync::Arc;
use codec::Decode;


const OCW_PUB_KEY_STR: &str = "d2bf4b844dfefd6772a8843e669f943408966a977e3ae2af1dd78e0f55f4df67";
const OCW_SEED: &str = "news slush supreme milk chapter athlete soap sausage put clutch what kitten";


#[test]
fn save_validated_data_works() {
new_test_ext().execute_with(|| {
assert_ok!(DispatchResult::Ok(()));
});
fn it_sets_validation_decision_with_one_validator_in_quorum() {

let mut t = new_test_ext();

let (offchain, offchain_state) = testing::TestOffchainExt::new();
t.register_extension(OffchainDbExt::new(offchain.clone()));
t.register_extension(OffchainWorkerExt::new(offchain));

let keystore = KeyStore::new();
keystore.sr25519_generate_new(KEY_TYPE, Some(OCW_SEED)).unwrap();
t.register_extension(KeystoreExt(Arc::new(keystore)));

let (pool, pool_state) = testing::TestTransactionPoolExt::new();
t.register_extension(TransactionPoolExt::new(pool));

let era_to_validate: EraIndex = 3;
let cdn_node_to_validate = AccountId::from([0x1; 32]);
let cdn_node_to_validate_str = utils::account_to_string::<Test>(cdn_node_to_validate.clone());

{
let mut state = offchain_state.write();

let mut expect_request = |url: &str, response: &[u8]| {
state.expect_request(testing::PendingRequest {
method: "GET".into(),
uri: url.to_string(),
response: Some(response.to_vec()),
sent: true,
..Default::default()
});
};

expect_request(
&format!("{}/JSON.GET/ddc:dac:aggregation:nodes:{}/$.{}", DEFAULT_DATA_PROVIDER_URL, era_to_validate, cdn_node_to_validate_str),
include_bytes!("./mock-data/set-1/aggregated-node-data-for-era.json")
);

expect_request(
&format!("{}/JSON.GET/ddc:dac:data:file:84640a53-fc1f-4ac5-921c-6695056840bc", DEFAULT_DATA_PROVIDER_URL),
include_bytes!("./mock-data/set-1/file-request1.json")
);

expect_request(
&format!("{}/JSON.GET/ddc:dac:data:file:d0a55c8b-fcb9-41b5-aa9a-8b40e9c4edf7", DEFAULT_DATA_PROVIDER_URL),
include_bytes!("./mock-data/set-1/file-request2.json")
);

expect_request(
&format!("{}/JSON.GET/ddc:dac:data:file:80a62530-fd76-40b5-bc53-dd82365e89ce", DEFAULT_DATA_PROVIDER_URL),
include_bytes!("./mock-data/set-1/file-request3.json")
);

let decision: ValidationDecision = serde_json::from_slice(include_bytes!("./mock-data/set-1/validation-decision.json")).unwrap();
let serialized_decision = serde_json::to_string(&decision).unwrap();
let encoded_decision_vec = shm::base64_encode(&serialized_decision.as_bytes().to_vec()).unwrap();
let encoded_decision_str = encoded_decision_vec.iter().cloned().collect::<String>();
let result_json = serde_json::json!({
"result": decision.result,
"data": encoded_decision_str,
});
let result_json_str = serde_json::to_string(&result_json).unwrap();
let unescaped_result_json = utils::unescape(&result_json_str);
let url_encoded_result_json = utils::url_encode(&unescaped_result_json);

expect_request(
&format!("{}/FCALL/save_validation_result_by_node/1/{}:{}:{}/{}",
DEFAULT_DATA_PROVIDER_URL,
OCW_PUB_KEY_STR,
cdn_node_to_validate_str,
era_to_validate,
url_encoded_result_json,
),
include_bytes!("./mock-data/set-1/save-validation-decision-result.json")
);

expect_request(
&format!("{}/JSON.GET/ddc:dac:shared:nodes:{}", DEFAULT_DATA_PROVIDER_URL, era_to_validate),
include_bytes!("./mock-data/set-1/shared-validation-decisions-for-era.json")
);

}

t.execute_with(|| {

let era_block_number = ERA_IN_BLOCKS as u32 * era_to_validate;
System::set_block_number(era_block_number); // required for randomness

Timestamp::set_timestamp((TIME_START_MS + ERA_DURATION_MS * (era_to_validate as u128 - 1)) as u64);
DdcValidator::on_initialize(era_block_number - 1); // make assignments

Timestamp::set_timestamp((TIME_START_MS + ERA_DURATION_MS * (era_to_validate as u128 + 1)) as u64);
DdcValidator::offchain_worker(era_block_number + 1); // execute assignments

let mut transactions = pool_state.read().transactions.clone();
transactions.reverse();
assert_eq!(transactions.len(), 3);

let tx = transactions.pop().unwrap();
let tx = Extrinsic::decode(&mut &*tx).unwrap();
assert!(tx.signature.is_some());

let bucket_info1 = BucketsDetails { bucket_id: 5, amount: 100u128 };
let bucket_info2 = BucketsDetails { bucket_id: 5, amount: 200u128 };
let bucket_info3 = BucketsDetails { bucket_id: 5, amount: 300u128 };

assert_eq!(tx.call, crate::mock::Call::DdcValidator(
crate::Call::charge_payments_content_owners {
paying_accounts: vec![
bucket_info3,
bucket_info1,
bucket_info2,
]
}
));

let tx = transactions.pop().unwrap();
let tx = Extrinsic::decode(&mut &*tx).unwrap();
assert!(tx.signature.is_some());
assert_eq!(tx.call, crate::mock::Call::DdcValidator(
crate::Call::payout_cdn_owners {
era: era_to_validate + 1
}
));

let tx = transactions.pop().unwrap();
let tx = Extrinsic::decode(&mut &*tx).unwrap();
assert!(tx.signature.is_some());

let common_decision: ValidationDecision = serde_json::from_slice(include_bytes!("./mock-data/set-1/validation-decision.json")).unwrap();
let common_decisions = vec![common_decision.clone()];
let serialized_decisions = serde_json::to_string(&common_decisions).unwrap();

assert_eq!(tx.call, crate::mock::Call::DdcValidator(
crate::Call::set_validation_decision {
era: era_to_validate + 1,
cdn_node: cdn_node_to_validate,
validation_decision: ValidationDecision {
edge: cdn_node_to_validate_str,
result: true,
payload: utils::hash(&serialized_decisions),
totals: DacTotalAggregates {
received: common_decision.totals.received,
sent: common_decision.totals.sent,
failed_by_client: common_decision.totals.failed_by_client,
failure_rate: common_decision.totals.failure_rate,
}
}
}
));

})
}


Loading