diff --git a/core-primitives/types/src/parentchain.rs b/core-primitives/types/src/parentchain.rs index 76976757a..686205051 100644 --- a/core-primitives/types/src/parentchain.rs +++ b/core-primitives/types/src/parentchain.rs @@ -59,10 +59,11 @@ pub type BlockHash = sp_core::H256; /// Alias to 512-bit hash when used in the context of a transaction signature on the chain. pub type Signature = MultiSignature; -#[derive(Encode, Decode, Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Encode, Decode, Copy, Clone, Debug, Default, PartialEq, Eq)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] pub enum ParentchainId { /// The Integritee Parentchain, the trust root of the enclave and serving finality to sidechains. + #[default] Integritee, /// A target chain containing custom business logic. TargetA, diff --git a/service/src/account_funding.rs b/service/src/account_funding.rs index 074a7e731..a5f1d56d3 100644 --- a/service/src/account_funding.rs +++ b/service/src/account_funding.rs @@ -30,7 +30,7 @@ use sp_core::{ }; use sp_keyring::AccountKeyring; use sp_runtime::{MultiAddress, Saturating}; -use std::{thread, time::Duration}; +use std::{fmt::Display, thread, time::Duration}; use substrate_api_client::{ ac_compose_macros::compose_extrinsic, ac_primitives::Bytes, extrinsic::BalancesExtrinsics, GetBalance, GetStorage, GetTransactionPayment, SubmitAndWatch, XtStatus, @@ -39,25 +39,71 @@ use teerex_primitives::SgxAttestationMethod; const SGX_RA_PROOF_MAX_LEN: usize = 5000; const MAX_URL_LEN: usize = 256; -/// Information about the enclave on-chain account. -pub trait EnclaveAccountInfo { + +#[derive(Clone)] +pub enum AccountAndRole { + EnclaveSigner(AccountId), + ShardVault(AccountId), +} + +impl Display for AccountAndRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AccountAndRole::EnclaveSigner(account_id) => { + write!(f, "EnclaveSigner({})", account_id.to_ss58check()) + }, + AccountAndRole::ShardVault(account_id) => { + write!(f, "ShardVault({})", account_id.to_ss58check()) + }, + } + } +} + +impl AccountAndRole { + pub fn account_id(&self) -> AccountId { + match self { + AccountAndRole::EnclaveSigner(account_id) => account_id.clone(), + AccountAndRole::ShardVault(account_id) => account_id.clone(), + } + } +} + +/// Information about an account on a specified parentchain. +pub trait ParentchainAccountInfo { fn free_balance(&self) -> ServiceResult; + fn parentchain_id(&self) -> ServiceResult; + fn account_and_role(&self) -> ServiceResult; } -pub struct EnclaveAccountInfoProvider { +pub struct ParentchainAccountInfoProvider { + parentchain_id: ParentchainId, node_api: ParentchainApi, - account_id: AccountId32, + account_and_role: AccountAndRole, } -impl EnclaveAccountInfo for EnclaveAccountInfoProvider { +impl ParentchainAccountInfo for ParentchainAccountInfoProvider { fn free_balance(&self) -> ServiceResult { - self.node_api.get_free_balance(&self.account_id).map_err(|e| e.into()) + self.node_api + .get_free_balance(&self.account_and_role.account_id()) + .map_err(|e| e.into()) + } + + fn parentchain_id(&self) -> ServiceResult { + Ok(self.parentchain_id) + } + + fn account_and_role(&self) -> ServiceResult { + Ok(self.account_and_role.clone()) } } -impl EnclaveAccountInfoProvider { - pub fn new(node_api: ParentchainApi, account_id: AccountId32) -> Self { - EnclaveAccountInfoProvider { node_api, account_id } +impl ParentchainAccountInfoProvider { + pub fn new( + parentchain_id: ParentchainId, + node_api: ParentchainApi, + account_and_role: AccountAndRole, + ) -> Self { + ParentchainAccountInfoProvider { parentchain_id, node_api, account_and_role } } } diff --git a/service/src/main_impl.rs b/service/src/main_impl.rs index cb5adc062..f79e92c57 100644 --- a/service/src/main_impl.rs +++ b/service/src/main_impl.rs @@ -4,7 +4,7 @@ use crate::teeracle::{schedule_periodic_reregistration_thread, start_periodic_ma #[cfg(not(feature = "dcap"))] use crate::utils::check_files; use crate::{ - account_funding::{setup_reasonable_account_funding, EnclaveAccountInfoProvider}, + account_funding::{setup_reasonable_account_funding, ParentchainAccountInfoProvider}, config::Config, enclave::{ api::enclave_init, @@ -52,7 +52,7 @@ use its_storage::{interface::FetchBlocks, BlockPruner, SidechainStorageLock}; use log::*; use regex::Regex; use sgx_types::*; -use sp_runtime::traits::Header as HeaderT; +use sp_runtime::traits::{Header as HeaderT, IdentifyAccount}; use substrate_api_client::{ api::XtStatus, rpc::HandleSubscription, GetAccountInformation, GetBalance, GetChainInfo, SubmitAndWatch, SubscribeChain, SubscribeEvents, @@ -65,7 +65,10 @@ use sgx_verify::extract_tcb_info_from_raw_dcap_quote; use itp_enclave_api::Enclave; -use crate::{account_funding::shard_vault_initial_funds, error::ServiceResult}; +use crate::{ + account_funding::{shard_vault_initial_funds, AccountAndRole}, + error::ServiceResult, +}; use enclave_bridge_primitives::ShardIdentifier; use itc_parentchain::primitives::ParentchainId; use itp_types::parentchain::{AccountId, Balance}; @@ -368,24 +371,6 @@ fn start_worker( } }); - // ------------------------------------------------------------------------ - // Start prometheus metrics server. - if config.enable_metrics_server() { - let enclave_wallet = Arc::new(EnclaveAccountInfoProvider::new( - integritee_rpc_api.clone(), - tee_accountid.clone(), - )); - let metrics_handler = Arc::new(MetricsHandler::new(enclave_wallet)); - let metrics_server_port = config - .try_parse_metrics_server_port() - .expect("metrics server port to be a valid port number"); - tokio_handle.spawn(async move { - if let Err(e) = start_metrics_server(metrics_handler, metrics_server_port).await { - error!("Unexpected error in Prometheus metrics server: {:?}", e); - } - }); - } - // ------------------------------------------------------------------------ // Start trusted worker rpc server if WorkerModeProvider::worker_mode() == WorkerMode::Sidechain @@ -502,9 +487,12 @@ fn start_worker( .expect("our enclave should be registered at this point"); trace!("verified that our enclave is registered: {:?}", my_enclave); - let (we_are_primary_validateer, re_init_parentchain_needed) = - match integritee_rpc_api.primary_worker_for_shard(shard, None).unwrap() { - Some(primary_enclave) => match primary_enclave.instance_signer() { + let (we_are_primary_validateer, re_init_parentchain_needed) = match integritee_rpc_api + .primary_worker_for_shard(shard, None) + .unwrap() + { + Some(primary_enclave) => + match primary_enclave.instance_signer() { AnySigner::Known(MultiSigner::Ed25519(primary)) => if primary.encode() == tee_accountid.encode() { println!("We are primary worker on this shard and we have been previously running."); @@ -539,24 +527,24 @@ fn start_worker( ); }, }, - None => { - println!("We are the primary worker on this shard and the shard is untouched. Will initialize it"); - enclave.init_shard(shard.encode()).unwrap(); - if WorkerModeProvider::worker_mode() != WorkerMode::Teeracle { - enclave - .init_shard_creation_parentchain_header( - shard, - &ParentchainId::Integritee, - ®ister_enclave_xt_header, - ) - .unwrap(); - debug!("shard config should be initialized on integritee network now"); - (true, true) - } else { - (true, false) - } - }, - }; + None => { + println!("We are the primary worker on this shard and the shard is untouched. Will initialize it"); + enclave.init_shard(shard.encode()).unwrap(); + if WorkerModeProvider::worker_mode() != WorkerMode::Teeracle { + enclave + .init_shard_creation_parentchain_header( + shard, + &ParentchainId::Integritee, + ®ister_enclave_xt_header, + ) + .unwrap(); + debug!("shard config should be initialized on integritee network now"); + (true, true) + } else { + (true, false) + } + }, + }; debug!("getting shard creation: {:?}", enclave.get_shard_creation_info(shard)); initialization_handler.registered_on_parentchain(); @@ -672,12 +660,68 @@ fn start_worker( shard, &enclave, integritee_rpc_api.clone(), - maybe_target_a_rpc_api, - maybe_target_b_rpc_api, + maybe_target_a_rpc_api.clone(), + maybe_target_b_rpc_api.clone(), run_config.shielding_target, we_are_primary_validateer, ); + // ------------------------------------------------------------------------ + // Start prometheus metrics server. + if config.enable_metrics_server() { + let mut account_info_providers: Vec> = vec![]; + account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new( + ParentchainId::Integritee, + integritee_rpc_api.clone(), + AccountAndRole::EnclaveSigner(tee_accountid.clone()), + ))); + let shielding_target = run_config.shielding_target.unwrap_or_default(); + let shard_vault = + enclave.get_ecc_vault_pubkey(shard).expect("shard vault must be defined by now"); + account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new( + shielding_target, + match shielding_target { + ParentchainId::Integritee => integritee_rpc_api.clone(), + ParentchainId::TargetA => maybe_target_a_rpc_api + .clone() + .expect("target A must be initialized to be used as shielding target"), + ParentchainId::TargetB => maybe_target_b_rpc_api + .clone() + .expect("target B must be initialized to be used as shielding target"), + }, + AccountAndRole::ShardVault( + enclave + .get_ecc_vault_pubkey(shard) + .expect("shard vault must be defined by now") + .into(), + ), + ))); + maybe_target_a_rpc_api.map(|api| { + account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new( + ParentchainId::TargetA, + api.clone(), + AccountAndRole::EnclaveSigner(tee_accountid.clone()), + ))) + }); + maybe_target_b_rpc_api.map(|api| { + account_info_providers.push(Arc::new(ParentchainAccountInfoProvider::new( + ParentchainId::TargetB, + api.clone(), + AccountAndRole::EnclaveSigner(tee_accountid.clone()), + ))) + }); + + let metrics_handler = Arc::new(MetricsHandler::new(account_info_providers)); + let metrics_server_port = config + .try_parse_metrics_server_port() + .expect("metrics server port to be a valid port number"); + tokio_handle.spawn(async move { + if let Err(e) = start_metrics_server(metrics_handler, metrics_server_port).await { + error!("Unexpected error in Prometheus metrics server: {:?}", e); + } + }); + } + if WorkerModeProvider::worker_mode() == WorkerMode::Sidechain { println!("[Integritee:SCV] starting block production"); let last_synced_header = @@ -699,7 +743,7 @@ fn init_provided_shard_vault( shielding_target: Option, we_are_primary_validateer: bool, ) { - let shielding_target = shielding_target.unwrap_or(ParentchainId::Integritee); + let shielding_target = shielding_target.unwrap_or_default(); let rpc_api = match shielding_target { ParentchainId::Integritee => integritee_rpc_api, ParentchainId::TargetA => maybe_target_a_rpc_api diff --git a/service/src/prometheus_metrics.rs b/service/src/prometheus_metrics.rs index 6de708a5a..14953ec5a 100644 --- a/service/src/prometheus_metrics.rs +++ b/service/src/prometheus_metrics.rs @@ -21,7 +21,7 @@ use crate::teeracle::teeracle_metrics::update_teeracle_metrics; use crate::{ - account_funding::EnclaveAccountInfo, + account_funding::ParentchainAccountInfo, error::{Error, ServiceResult}, }; use async_trait::async_trait; @@ -40,9 +40,9 @@ use itp_enclave_metrics::EnclaveMetric; use lazy_static::lazy_static; use log::*; use prometheus::{ - proto::MetricFamily, register_gauge, register_histogram, register_histogram_vec, - register_int_counter, register_int_gauge, register_int_gauge_vec, Gauge, Histogram, - HistogramOpts, HistogramVec, IntCounter, IntGauge, IntGaugeVec, + proto::MetricFamily, register_gauge, register_gauge_vec, register_histogram, + register_histogram_vec, register_int_counter, register_int_gauge, register_int_gauge_vec, + Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, IntCounter, IntGauge, IntGaugeVec, }; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, net::SocketAddr, sync::Arc}; @@ -57,8 +57,8 @@ const COUNT_HISTOGRAM_BUCKETS: [f64; 12] = lazy_static! { /// Register all the prometheus metrics we want to monitor (aside from the default process ones). - static ref ENCLAVE_ACCOUNT_FREE_BALANCE: IntGauge = - register_int_gauge!("integritee_worker_enclave_account_free_balance", "Free balance of the enclave account") + static ref ACCOUNT_FREE_BALANCE: GaugeVec = + register_gauge_vec!("integritee_worker_account_free_balance", "Free balance of an account on a parentchain with a role (lossy f64)", &["parentchain","role"]) .unwrap(); static ref ENCLAVE_SIDECHAIN_BLOCK_HEIGHT: IntGauge = register_int_gauge!("integritee_worker_enclave_sidechain_block_height", "Enclave sidechain block height") @@ -135,15 +135,15 @@ pub trait HandleMetrics { async fn handle_metrics(&self) -> Result; } -/// Metrics handler implementation. +/// Metrics handler implementation. This is for untrusted sources of metrics (non-enclave) pub struct MetricsHandler { - enclave_wallet: Arc, + wallets: Vec>, } #[async_trait] impl HandleMetrics for MetricsHandler where - Wallet: EnclaveAccountInfo + Send + Sync, + Wallet: ParentchainAccountInfo + Send + Sync, { type ReplyType = String; @@ -164,20 +164,29 @@ where impl MetricsHandler where - Wallet: EnclaveAccountInfo + Send + Sync, + Wallet: ParentchainAccountInfo + Send + Sync, { - pub fn new(enclave_wallet: Arc) -> Self { - MetricsHandler { enclave_wallet } + pub fn new(wallets: Vec>) -> Self { + MetricsHandler { wallets } } async fn update_metrics(&self) { - match self.enclave_wallet.free_balance() { - Ok(b) => { - ENCLAVE_ACCOUNT_FREE_BALANCE.set(b as i64); - }, - Err(e) => { - error!("Failed to fetch free balance metric, value will not be updated: {:?}", e); - }, + for wallet in &self.wallets { + if let (Ok(balance), Ok(parentchain_id), Ok(account_and_role)) = + (wallet.free_balance(), wallet.parentchain_id(), wallet.account_and_role()) + { + ACCOUNT_FREE_BALANCE + .with_label_values( + [ + format!("{}", parentchain_id).as_str(), + format!("{}", account_and_role).as_str(), + ] + .as_slice(), + ) + .set(balance as f64) + } else { + error!("failed to update prometheus metric for a wallet"); + } } } }