Skip to content

Commit

Permalink
add account balances for enclave on all parentchains and shard vault
Browse files Browse the repository at this point in the history
  • Loading branch information
@brenzi
brenzi committed Oct 14, 2024
1 parent cdf2659 commit 7ca88dd
Show file tree
Hide file tree
Showing 4 changed files with 175 additions and 75 deletions.
3 changes: 2 additions & 1 deletion core-primitives/types/src/parentchain.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ pub type BlockHash = sp_core::H256;
/// Alias to 512-bit hash when used in the context of a transaction signature on the chain.
pub type Signature = MultiSignature;

#[derive(Encode, Decode, Copy, Clone, Debug, PartialEq, Eq)]
#[derive(Encode, Decode, Copy, Clone, Debug, Default, PartialEq, Eq)]
#[cfg_attr(feature = "std", derive(Serialize, Deserialize))]
pub enum ParentchainId {
/// The Integritee Parentchain, the trust root of the enclave and serving finality to sidechains.
#[default]
Integritee,
/// A target chain containing custom business logic.
TargetA,
Expand Down
66 changes: 56 additions & 10 deletions service/src/account_funding.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use sp_core::{
};
use sp_keyring::AccountKeyring;
use sp_runtime::{MultiAddress, Saturating};
use std::{thread, time::Duration};
use std::{fmt::Display, thread, time::Duration};
use substrate_api_client::{
ac_compose_macros::compose_extrinsic, ac_primitives::Bytes, extrinsic::BalancesExtrinsics,
GetBalance, GetStorage, GetTransactionPayment, SubmitAndWatch, XtStatus,
Expand All @@ -39,25 +39,71 @@ use teerex_primitives::SgxAttestationMethod;

const SGX_RA_PROOF_MAX_LEN: usize = 5000;
const MAX_URL_LEN: usize = 256;
/// Information about the enclave on-chain account.
pub trait EnclaveAccountInfo {

#[derive(Clone)]
pub enum AccountAndRole {
EnclaveSigner(AccountId),
ShardVault(AccountId),
}

impl Display for AccountAndRole {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
AccountAndRole::EnclaveSigner(account_id) => {
write!(f, "EnclaveSigner({})", account_id.to_ss58check())
},
AccountAndRole::ShardVault(account_id) => {
write!(f, "ShardVault({})", account_id.to_ss58check())
},
}
}
}

impl AccountAndRole {
pub fn account_id(&self) -> AccountId {
match self {
AccountAndRole::EnclaveSigner(account_id) => account_id.clone(),
AccountAndRole::ShardVault(account_id) => account_id.clone(),
}
}
}

/// Information about an account on a specified parentchain.
pub trait ParentchainAccountInfo {
fn free_balance(&self) -> ServiceResult<Balance>;
fn parentchain_id(&self) -> ServiceResult<ParentchainId>;
fn account_and_role(&self) -> ServiceResult<AccountAndRole>;
}

pub struct EnclaveAccountInfoProvider {
pub struct ParentchainAccountInfoProvider {
parentchain_id: ParentchainId,
node_api: ParentchainApi,
account_id: AccountId32,
account_and_role: AccountAndRole,
}

impl EnclaveAccountInfo for EnclaveAccountInfoProvider {
impl ParentchainAccountInfo for ParentchainAccountInfoProvider {
fn free_balance(&self) -> ServiceResult<Balance> {
self.node_api.get_free_balance(&self.account_id).map_err(|e| e.into())
self.node_api
.get_free_balance(&self.account_and_role.account_id())
.map_err(|e| e.into())
}

fn parentchain_id(&self) -> ServiceResult<ParentchainId> {
Ok(self.parentchain_id)
}

fn account_and_role(&self) -> ServiceResult<AccountAndRole> {
Ok(self.account_and_role.clone())
}
}

impl EnclaveAccountInfoProvider {
pub fn new(node_api: ParentchainApi, account_id: AccountId32) -> Self {
EnclaveAccountInfoProvider { node_api, account_id }
impl ParentchainAccountInfoProvider {
pub fn new(
parentchain_id: ParentchainId,
node_api: ParentchainApi,
account_and_role: AccountAndRole,
) -> Self {
ParentchainAccountInfoProvider { parentchain_id, node_api, account_and_role }
}
}

Expand Down
134 changes: 89 additions & 45 deletions service/src/main_impl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use crate::teeracle::{schedule_periodic_reregistration_thread, start_periodic_ma
#[cfg(not(feature = "dcap"))]
use crate::utils::check_files;
use crate::{
account_funding::{setup_reasonable_account_funding, EnclaveAccountInfoProvider},
account_funding::{setup_reasonable_account_funding, ParentchainAccountInfoProvider},
config::Config,
enclave::{
api::enclave_init,
Expand Down Expand Up @@ -52,7 +52,7 @@ use its_storage::{interface::FetchBlocks, BlockPruner, SidechainStorageLock};
use log::*;
use regex::Regex;
use sgx_types::*;
use sp_runtime::traits::Header as HeaderT;
use sp_runtime::traits::{Header as HeaderT, IdentifyAccount};
use substrate_api_client::{
api::XtStatus, rpc::HandleSubscription, GetAccountInformation, GetBalance, GetChainInfo,
SubmitAndWatch, SubscribeChain, SubscribeEvents,
Expand All @@ -65,7 +65,10 @@ use sgx_verify::extract_tcb_info_from_raw_dcap_quote;

use itp_enclave_api::Enclave;

use crate::{account_funding::shard_vault_initial_funds, error::ServiceResult};
use crate::{
account_funding::{shard_vault_initial_funds, AccountAndRole},
error::ServiceResult,
};
use enclave_bridge_primitives::ShardIdentifier;
use itc_parentchain::primitives::ParentchainId;
use itp_types::parentchain::{AccountId, Balance};
Expand Down Expand Up @@ -368,24 +371,6 @@ fn start_worker<E, T, D, InitializationHandler, WorkerModeProvider>(
}
});

// ------------------------------------------------------------------------
// Start prometheus metrics server.
if config.enable_metrics_server() {
let enclave_wallet = Arc::new(EnclaveAccountInfoProvider::new(
integritee_rpc_api.clone(),
tee_accountid.clone(),
));
let metrics_handler = Arc::new(MetricsHandler::new(enclave_wallet));
let metrics_server_port = config
.try_parse_metrics_server_port()
.expect("metrics server port to be a valid port number");
tokio_handle.spawn(async move {
if let Err(e) = start_metrics_server(metrics_handler, metrics_server_port).await {
error!("Unexpected error in Prometheus metrics server: {:?}", e);
}
});
}

// ------------------------------------------------------------------------
// Start trusted worker rpc server
if WorkerModeProvider::worker_mode() == WorkerMode::Sidechain
Expand Down Expand Up @@ -502,9 +487,12 @@ fn start_worker<E, T, D, InitializationHandler, WorkerModeProvider>(
.expect("our enclave should be registered at this point");
trace!("verified that our enclave is registered: {:?}", my_enclave);

let (we_are_primary_validateer, re_init_parentchain_needed) =
match integritee_rpc_api.primary_worker_for_shard(shard, None).unwrap() {
Some(primary_enclave) => match primary_enclave.instance_signer() {
let (we_are_primary_validateer, re_init_parentchain_needed) = match integritee_rpc_api
.primary_worker_for_shard(shard, None)
.unwrap()
{
Some(primary_enclave) =>
match primary_enclave.instance_signer() {
AnySigner::Known(MultiSigner::Ed25519(primary)) =>
if primary.encode() == tee_accountid.encode() {
println!("We are primary worker on this shard and we have been previously running.");
Expand Down Expand Up @@ -539,24 +527,24 @@ fn start_worker<E, T, D, InitializationHandler, WorkerModeProvider>(
);
},
},
None => {
println!("We are the primary worker on this shard and the shard is untouched. Will initialize it");
enclave.init_shard(shard.encode()).unwrap();
if WorkerModeProvider::worker_mode() != WorkerMode::Teeracle {
enclave
.init_shard_creation_parentchain_header(
shard,
&ParentchainId::Integritee,
&register_enclave_xt_header,
)
.unwrap();
debug!("shard config should be initialized on integritee network now");
(true, true)
} else {
(true, false)
}
},
};
None => {
println!("We are the primary worker on this shard and the shard is untouched. Will initialize it");
enclave.init_shard(shard.encode()).unwrap();
if WorkerModeProvider::worker_mode() != WorkerMode::Teeracle {
enclave
.init_shard_creation_parentchain_header(
shard,
&ParentchainId::Integritee,
&register_enclave_xt_header,
)
.unwrap();
debug!("shard config should be initialized on integritee network now");
(true, true)
} else {
(true, false)
}
},
};
debug!("getting shard creation: {:?}", enclave.get_shard_creation_info(shard));
initialization_handler.registered_on_parentchain();

Expand Down Expand Up @@ -672,12 +660,68 @@ fn start_worker<E, T, D, InitializationHandler, WorkerModeProvider>(
shard,
&enclave,
integritee_rpc_api.clone(),
maybe_target_a_rpc_api,
maybe_target_b_rpc_api,
maybe_target_a_rpc_api.clone(),
maybe_target_b_rpc_api.clone(),
run_config.shielding_target,
we_are_primary_validateer,
);

// ------------------------------------------------------------------------
// Start prometheus metrics server.
if config.enable_metrics_server() {
let mut account_info_providers: Vec<Arc<ParentchainAccountInfoProvider>> = vec![];
account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new(
ParentchainId::Integritee,
integritee_rpc_api.clone(),
AccountAndRole::EnclaveSigner(tee_accountid.clone()),
)));
let shielding_target = run_config.shielding_target.unwrap_or_default();
let shard_vault =
enclave.get_ecc_vault_pubkey(shard).expect("shard vault must be defined by now");
account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new(
shielding_target,
match shielding_target {
ParentchainId::Integritee => integritee_rpc_api.clone(),
ParentchainId::TargetA => maybe_target_a_rpc_api
.clone()
.expect("target A must be initialized to be used as shielding target"),
ParentchainId::TargetB => maybe_target_b_rpc_api
.clone()
.expect("target B must be initialized to be used as shielding target"),
},
AccountAndRole::ShardVault(
enclave
.get_ecc_vault_pubkey(shard)
.expect("shard vault must be defined by now")
.into(),
),
)));
maybe_target_a_rpc_api.map(|api| {
account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new(
ParentchainId::TargetA,
api.clone(),
AccountAndRole::EnclaveSigner(tee_accountid.clone()),
)))
});
maybe_target_b_rpc_api.map(|api| {
account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new(
ParentchainId::TargetB,
api.clone(),
AccountAndRole::EnclaveSigner(tee_accountid.clone()),
)))
});

let metrics_handler = Arc::new(MetricsHandler::new(account_info_providers));
let metrics_server_port = config
.try_parse_metrics_server_port()
.expect("metrics server port to be a valid port number");
tokio_handle.spawn(async move {
if let Err(e) = start_metrics_server(metrics_handler, metrics_server_port).await {
error!("Unexpected error in Prometheus metrics server: {:?}", e);
}
});
}

if WorkerModeProvider::worker_mode() == WorkerMode::Sidechain {
println!("[Integritee:SCV] starting block production");
let last_synced_header =
Expand All @@ -699,7 +743,7 @@ fn init_provided_shard_vault<E: EnclaveBase>(
shielding_target: Option<ParentchainId>,
we_are_primary_validateer: bool,
) {
let shielding_target = shielding_target.unwrap_or(ParentchainId::Integritee);
let shielding_target = shielding_target.unwrap_or_default();
let rpc_api = match shielding_target {
ParentchainId::Integritee => integritee_rpc_api,
ParentchainId::TargetA => maybe_target_a_rpc_api
Expand Down
47 changes: 28 additions & 19 deletions service/src/prometheus_metrics.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
use crate::teeracle::teeracle_metrics::update_teeracle_metrics;

use crate::{
account_funding::EnclaveAccountInfo,
account_funding::ParentchainAccountInfo,
error::{Error, ServiceResult},
};
use async_trait::async_trait;
Expand All @@ -40,9 +40,9 @@ use itp_enclave_metrics::EnclaveMetric;
use lazy_static::lazy_static;
use log::*;
use prometheus::{
proto::MetricFamily, register_gauge, register_histogram, register_histogram_vec,
register_int_counter, register_int_gauge, register_int_gauge_vec, Gauge, Histogram,
HistogramOpts, HistogramVec, IntCounter, IntGauge, IntGaugeVec,
proto::MetricFamily, register_gauge, register_gauge_vec, register_histogram,
register_histogram_vec, register_int_counter, register_int_gauge, register_int_gauge_vec,
Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, IntCounter, IntGauge, IntGaugeVec,
};
use serde::{Deserialize, Serialize};
use std::{fmt::Debug, net::SocketAddr, sync::Arc};
Expand All @@ -57,8 +57,8 @@ const COUNT_HISTOGRAM_BUCKETS: [f64; 12] =
lazy_static! {
/// Register all the prometheus metrics we want to monitor (aside from the default process ones).

static ref ENCLAVE_ACCOUNT_FREE_BALANCE: IntGauge =
register_int_gauge!("integritee_worker_enclave_account_free_balance", "Free balance of the enclave account")
static ref ACCOUNT_FREE_BALANCE: GaugeVec =
register_gauge_vec!("integritee_worker_account_free_balance", "Free balance of an account on a parentchain with a role (lossy f64)", &["parentchain","role"])
.unwrap();
static ref ENCLAVE_SIDECHAIN_BLOCK_HEIGHT: IntGauge =
register_int_gauge!("integritee_worker_enclave_sidechain_block_height", "Enclave sidechain block height")
Expand Down Expand Up @@ -135,15 +135,15 @@ pub trait HandleMetrics {
async fn handle_metrics(&self) -> Result<Self::ReplyType, Rejection>;
}

/// Metrics handler implementation.
/// Metrics handler implementation. This is for untrusted sources of metrics (non-enclave)
pub struct MetricsHandler<Wallet> {
enclave_wallet: Arc<Wallet>,
wallets: Vec<Arc<Wallet>>,
}

#[async_trait]
impl<Wallet> HandleMetrics for MetricsHandler<Wallet>
where
Wallet: EnclaveAccountInfo + Send + Sync,
Wallet: ParentchainAccountInfo + Send + Sync,
{
type ReplyType = String;

Expand All @@ -164,20 +164,29 @@ where

impl<Wallet> MetricsHandler<Wallet>
where
Wallet: EnclaveAccountInfo + Send + Sync,
Wallet: ParentchainAccountInfo + Send + Sync,
{
pub fn new(enclave_wallet: Arc<Wallet>) -> Self {
MetricsHandler { enclave_wallet }
pub fn new(wallets: Vec<Arc<Wallet>>) -> Self {
MetricsHandler { wallets }
}

async fn update_metrics(&self) {
match self.enclave_wallet.free_balance() {
Ok(b) => {
ENCLAVE_ACCOUNT_FREE_BALANCE.set(b as i64);
},
Err(e) => {
error!("Failed to fetch free balance metric, value will not be updated: {:?}", e);
},
for wallet in &self.wallets {
if let (Ok(balance), Ok(parentchain_id), Ok(account_and_role)) =
(wallet.free_balance(), wallet.parentchain_id(), wallet.account_and_role())
{
ACCOUNT_FREE_BALANCE
.with_label_values(
[
format!("{}", parentchain_id).as_str(),
format!("{}", account_and_role).as_str(),
]
.as_slice(),
)
.set(balance as f64)
} else {
error!("failed to update prometheus metric for a wallet");
}
}
}
}
Expand Down

0 comments on commit 7ca88dd

Please sign in to comment.