Switch from ias to dcap (#1375)

* Switch from ias to dcap * Mount sgx qcnl * Mount sgx qcnl * Start aesmd in the background * Fix apt install * Fix apt install * Fix entrypoint * Fix lib path * Remove starting aesmd * Try with entrypoint * Move WORKDIR down * only ignore yaml files * Move back WORKDIR * Addedd qcnl sgx conf * change entrypoint to command * change entrypoint to command * Don't run aesmd service in cli * Try to remove IAS_SPID envs * Do not use ias version of the node * Do not use ias version of the node * Add back ias node * attestation-service: enable SGX DCAP QPL logging * docker: add missing DCAP packages * attestation: rename function used in MURA to distingiuish between DCAP and EPID attestation * attestation: preparing for DCAP MURA * attestation: prepare for DCAP * move qe_get_* to the worker * backup, linker error * testing * dcap-mura: fix DCAP API calls order * enclave-runtime: add dcap feature flag to it as well * fixup! enclave-runtime: add dcap feature flag to it as well * attestation-handler: return private_key as well from generate_dcap_ra_cert * add logging * fix logging * mura-dcap: add target to provisioning as well * MURA: use the same target * add printing for cert_der * to be dropped * mura: base64 encode the payload * fixup! mura: base64 encode the payload * fixup! mura: base64 encode the payload * mura: basically disable it with dcap * gha: add back clippy * attestation-handler: remove build.rs * attestation-handler: remove commented out code * attestation-handler: add debug log * fix typo * enclave-runtime: fix clippy * build: revert dcap lib linking * build: revert cargo verbosity * docker: remove epid leftover files * attestation-handler: return error if base64 decoding fails * docker: fix typo in network name * local-setup: print worker command as well * dcap with sw mode: fix * gha/build: remove worker_features_arg * dcap with sw mode: fix integration tests * cli: add a note regarding dcap * clippy: fix * mura: fix dcap related parameter references * fixup! mura: fix dcap related parameter references * docker: enable debug logging * add more logs * fixup! add more logs * mura: make sure enclave gets the correct reference * Revert "docker: enable debug logging" This reverts commit 1681a81. * Revert "add more logs" This reverts commit f141e03. * service: remove potentially unnecessary clone() * attestation-handler: remove enum variant import * attestation-handler: add TODO for MURA removal * enclave-runtime/attestation: remove obsolete todo --------- Co-authored-by: Szilard Parrag <[email protected]>
integritee-network · Jul 17, 2023 · 3443f15 · 3443f15
1 parent 7f3bcb9
commit 3443f15
Show file tree

Hide file tree

Showing 33 changed files with 517 additions and 182 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -3,7 +3,7 @@
 .github
 .idea
 ci/
-docker/
+docker/*yml
 docs/
 local-setup/
 scripts/

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -37,17 +37,20 @@ jobs:
             mode: sidechain
             host: integritee-builder-sgx
             sgx_mode: HW
+            additional_features: dcap
           - flavor_id: offchain-worker
             mode: offchain-worker
             host: integritee-builder-sgx
             sgx_mode: HW
+            additional_features: dcap
           - flavor_id: teeracle
             mode: teeracle
             host: integritee-builder-sgx
             sgx_mode: HW
+            additional_features: dcap
           - flavor_id: sidechain-evm
             mode: sidechain
-            additional_features: evm
+            additional_features: evm,dcap
             host: integritee-builder-sgx
             sgx_mode: HW
 
@@ -60,7 +63,7 @@ jobs:
           echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV
           if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
              echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV
-             echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd" >> $GITHUB_ENV
+             echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
           else
              echo "DOCKER_DEVICES=" >> $GITHUB_ENV
              echo "DOCKER_VOLUMES=" >> $GITHUB_ENV
@@ -189,8 +192,8 @@ jobs:
       WORKER_IMAGE_TAG: integritee-worker:dev
       CLIENT_IMAGE_TAG: integritee-cli:dev
       COINMARKETCAP_KEY: ${{ secrets.COINMARKETCAP_KEY }}
-      IAS_EPID_SPID: ${{ secrets.IAS_SPID }}
-      IAS_EPID_KEY: ${{ secrets.IAS_PRIMARY_KEY }}
+      # IAS_EPID_SPID: ${{ secrets.IAS_SPID }}
+      # IAS_EPID_KEY: ${{ secrets.IAS_PRIMARY_KEY }}
       TEERACLE_INTERVAL_SECONDS: 10
 
     strategy:
@@ -254,6 +257,7 @@ jobs:
              echo "SGX_PROVISION=/dev/sgx/provision" >> $GITHUB_ENV
              echo "SGX_ENCLAVE=/dev/sgx/enclave" >> $GITHUB_ENV
              echo "AESMD=/var/run/aesmd" >> $GITHUB_ENV
+             echo "SGX_QCNL=/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
           fi
 
           echo "LOG_DIR=./logs-$version" >> $GITHUB_ENV
@@ -367,7 +371,7 @@ jobs:
           - flavor_id: teeracle
             mode: teeracle
             sgx_mode: HW
-            worker_features: dcap
+            additional_features: dcap
 
     steps:
       - uses: actions/checkout@v3
@@ -384,7 +388,7 @@ jobs:
           echo "FINGERPRINT=$fingerprint" >> $GITHUB_ENV
           if [[ ${{ matrix.sgx_mode }} == 'HW' ]]; then
               echo "DOCKER_DEVICES=--device=/dev/sgx/enclave --device=/dev/sgx/provision" >> $GITHUB_ENV
-              echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd" >> $GITHUB_ENV
+              echo "DOCKER_VOLUMES=--volume /var/run/aesmd:/var/run/aesmd --volume /etc/sgx_default_qcnl.conf:/etc/sgx_default_qcnl.conf" >> $GITHUB_ENV
           else
               echo "DOCKER_DEVICES=" >> $GITHUB_ENV
               echo "DOCKER_VOLUMES=" >> $GITHUB_ENV
@@ -423,7 +427,7 @@ jobs:
         run: >
           docker build -t integritee/${{ matrix.flavor_id }}:${{ github.ref_name }}
           --target deployed-worker
-          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg SGX_COMMERCIAL_KEY=enclave-runtime/intel_sgx.pem --build-arg SGX_PASSFILE=enclave-runtime/passfile.txt --build-arg SGX_PRODUCTION=1 --build-arg WORKER_FEATURES_ARG=${{ matrix.worker_features }} --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }}
+          --build-arg WORKER_MODE_ARG=${{ matrix.mode }} --build-arg SGX_COMMERCIAL_KEY=enclave-runtime/intel_sgx.pem --build-arg SGX_PASSFILE=enclave-runtime/passfile.txt --build-arg SGX_PRODUCTION=1 --build-arg ADDITIONAL_FEATURES_ARG=${{ matrix.additional_features }} --build-arg SGX_MODE=${{ matrix.sgx_mode }}
           -f build.Dockerfile .
 
       - name: Save released teeracle

diff --git a/build.Dockerfile b/build.Dockerfile
@@ -38,8 +38,6 @@ ENV SGX_MODE=$SGX_MODE
 ARG SGX_PRODUCTION=0
 ENV SGX_PRODUCTION=$SGX_PRODUCTION
 
-ARG WORKER_FEATURES_ARG
-ENV WORKER_FEATURES=$WORKER_FEATURES_ARG
 
 ENV WORKHOME=/home/ubuntu/work
 ENV HOME=/home/ubuntu
@@ -80,6 +78,19 @@ RUN --mount=type=cache,id=cargo-registry-cache,target=/opt/rust/registry/cache,s
 FROM oasisprotocol/aesmd:master AS runner
 ENV SGX_SDK /opt/sgxsdk
 ENV LD_LIBRARY_PATH "${SGX_SDK}/sdk_libs"
+RUN apt-get install -y \
+	libsgx-aesm-ecdsa-plugin \
+	libsgx-ae-qve \
+	libsgx-aesm-quote-ex-plugin \
+	libsgx-dcap-default-qpl \
+	libsgx-dcap-ql \
+	libsgx-dcap-quote-verify \ 
+	libsgx-epid \ 
+	libsgx-headers \ 
+	libsgx-quote-ex \ 
+	libsgx-ra-network \ 
+	libsgx-ra-uefi \ 
+	libsgx-uae-service
 
 ### Deployed CLI client
 ##################################################
@@ -116,14 +127,19 @@ COPY --from=builder /home/ubuntu/work/worker/bin/* ./
 COPY --from=builder /lib/x86_64-linux-gnu/libsgx* /lib/x86_64-linux-gnu/
 COPY --from=builder /lib/x86_64-linux-gnu/libdcap* /lib/x86_64-linux-gnu/
 
-RUN touch spid.txt key.txt
 RUN chmod +x /usr/local/bin/integritee-service
 RUN ls -al /usr/local/bin
 
 # checks
 ENV SGX_SDK /opt/sgxsdk
-ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$SGX_SDK/sdk_libs
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/sgx-aesm-service/aesm:$SGX_SDK/sdk_libs
+ENV AESM_PATH=/opt/intel/sgx-aesm-service/aesm
+
+COPY ./docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+
 RUN ldd /usr/local/bin/integritee-service && \
 	/usr/local/bin/integritee-service --version
 
-ENTRYPOINT ["/usr/local/bin/integritee-service"]
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -60,3 +60,5 @@ teeracle = []
 sidechain = []
 offchain-worker = []
 production = []
+# dcap feature flag is not used in this crate, but for easier build purposes only it present here as well
+dcap = []
diff --git a/core-primitives/attestation-handler/Cargo.toml b/core-primitives/attestation-handler/Cargo.toml
@@ -37,7 +37,7 @@ sgx_rand = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sd
 sgx_tcrypto = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sdk.git", optional = true }
 sgx_tse = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sdk.git", optional = true }
 sgx_tstd = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sdk.git", features = ["untrusted_fs", "net", "backtrace"], optional = true }
-sgx_types = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sdk.git" }
+sgx_types = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sdk.git", features = ["extra_traits"] }
 
 # local deps
 itp-ocall-api = { path = "../ocall-api", default-features = false }

diff --git a/core-primitives/attestation-handler/src/attestation_handler.rs b/core-primitives/attestation-handler/src/attestation_handler.rs
@@ -48,7 +48,7 @@ use sgx_tcrypto::{rsgx_sha256_slice, SgxEccHandle};
 use sgx_tse::{rsgx_create_report, rsgx_verify_report};
 use sgx_types::{
 	c_int, sgx_epid_group_id_t, sgx_quote_nonce_t, sgx_quote_sign_type_t, sgx_report_data_t,
-	sgx_spid_t, sgx_status_t, sgx_target_info_t, SgxResult, *,
+	sgx_spid_t, sgx_status_t, sgx_status_t::SGX_ERROR_UNEXPECTED, sgx_target_info_t, SgxResult, *,
 };
 use sp_core::{ed25519, Pair};
 use std::{
@@ -82,15 +82,15 @@ pub trait AttestationHandler {
 	/// but instead generate a mock certificate.
 	fn generate_ias_ra_cert(&self, skip_ra: bool) -> EnclaveResult<Vec<u8>>;
 
-	/// Returns the DER encoded certificate and the raw DCAP quote.
+	/// Returns the DER encoded private_key, DER encoded certificate and the raw DCAP quote.
 	/// If skip_ra is set, it will not perform a remote attestation via IAS
 	/// but instead generate a mock certificate.
 	fn generate_dcap_ra_cert(
 		&self,
-		quoting_enclave_target_info: &sgx_target_info_t,
-		quote_size: u32,
+		quoting_enclave_target_info: Option<&sgx_target_info_t>,
+		quote_size: Option<&u32>,
 		skip_ra: bool,
-	) -> EnclaveResult<(Vec<u8>, Vec<u8>)>;
+	) -> EnclaveResult<(Vec<u8>, Vec<u8>, Vec<u8>)>;
 
 	/// Get the measurement register value of the enclave
 	fn get_mrenclave(&self) -> EnclaveResult<[u8; MR_ENCLAVE_SIZE]>;
@@ -107,7 +107,7 @@ pub trait AttestationHandler {
 
 	/// Create the remote attestation report and encapsulate it in a DER certificate
 	/// Returns a pair consisting of (private key DER, certificate DER)
-	fn create_ra_report_and_signature(
+	fn create_epid_ra_report_and_signature(
 		&self,
 		sign_type: sgx_quote_sign_type_t,
 		skip_ra: bool,
@@ -119,6 +119,35 @@ pub struct IntelAttestationHandler<OCallApi, SigningKeyRepo> {
 	pub(crate) signing_key_repo: Arc<SigningKeyRepo>,
 }
 
+impl<OCallApi, AccessSigningKey> IntelAttestationHandler<OCallApi, AccessSigningKey>
+where
+	OCallApi: EnclaveAttestationOCallApi,
+	AccessSigningKey: AccessKey<KeyType = ed25519::Pair>,
+{
+	fn create_payload_epid(
+		&self,
+		pub_k: &[u8; 32],
+		sign_type: sgx_quote_sign_type_t,
+	) -> EnclaveResult<String> {
+		info!("    [Enclave] Create attestation report");
+		let (attn_report, sig, cert) = match self.create_epid_attestation_report(&pub_k, sign_type)
+		{
+			Ok(r) => r,
+			Err(e) => {
+				error!("    [Enclave] Error in create_attestation_report: {:?}", e);
+				return Err(e.into())
+			},
+		};
+		println!("    [Enclave] Create attestation report successful");
+		debug!("              attn_report = {:?}", attn_report);
+		debug!("              sig         = {:?}", sig);
+		debug!("              cert        = {:?}", cert);
+
+		// concat the information
+		Ok(attn_report + "|" + &sig + "|" + &cert)
+	}
+}
+
 impl<OCallApi, AccessSigningKey> AttestationHandler
 	for IntelAttestationHandler<OCallApi, AccessSigningKey>
 where
@@ -132,7 +161,7 @@ where
 		// FIXME: should call `create_ra_report_and_signature` in skip_ra mode as well:
 		// https://github.com/integritee-network/worker/issues/321.
 		let cert_der = if !skip_ra {
-			match self.create_ra_report_and_signature(sign_type, skip_ra) {
+			match self.create_epid_ra_report_and_signature(sign_type, skip_ra) {
 				Ok((_key_der, cert_der)) => cert_der,
 				Err(e) => return Err(e),
 			}
@@ -154,7 +183,8 @@ where
 		// our certificate is unlinkable
 		let sign_type = sgx_quote_sign_type_t::SGX_UNLINKABLE_SIGNATURE;
 
-		let (_key_der, cert_der) = match self.create_ra_report_and_signature(sign_type, false) {
+		let (_key_der, cert_der) = match self.create_epid_ra_report_and_signature(sign_type, false)
+		{
 			Ok(r) => r,
 			Err(e) => return Err(e),
 		};
@@ -175,11 +205,14 @@ where
 		quoting_enclave_target_info: &sgx_target_info_t,
 		quote_size: u32,
 	) -> EnclaveResult<()> {
-		let (_cert_der, dcap_quote) =
-			match self.generate_dcap_ra_cert(quoting_enclave_target_info, quote_size, false) {
-				Ok(r) => r,
-				Err(e) => return Err(e),
-			};
+		let (_priv_key_der, _cert_der, dcap_quote) = match self.generate_dcap_ra_cert(
+			Some(quoting_enclave_target_info),
+			Some(&quote_size),
+			false,
+		) {
+			Ok(r) => r,
+			Err(e) => return Err(e),
+		};
 
 		if let Err(err) = io::write(&dcap_quote, RA_DUMP_CERT_DER_FILE) {
 			error!(
@@ -192,7 +225,7 @@ where
 		Ok(())
 	}
 
-	fn create_ra_report_and_signature(
+	fn create_epid_ra_report_and_signature(
 		&self,
 		sign_type: sgx_quote_sign_type_t,
 		skip_ra: bool,
@@ -209,36 +242,20 @@ where
 		debug!("     pubkey Y is {:02x}", pub_k.gy.iter().format(""));
 
 		let payload = if !skip_ra {
-			info!("    [Enclave] Create attestation report");
-			let (attn_report, sig, cert) =
-				match self.create_attestation_report(&chain_signer.public().0, sign_type) {
-					Ok(r) => r,
-					Err(e) => {
-						error!("    [Enclave] Error in create_attestation_report: {:?}", e);
-						return Err(e.into())
-					},
-				};
-			println!("    [Enclave] Create attestation report successful");
-			debug!("              attn_report = {:?}", attn_report);
-			debug!("              sig         = {:?}", sig);
-			debug!("              cert        = {:?}", cert);
-
-			// concat the information
-			attn_report + "|" + &sig + "|" + &cert
+			self.create_payload_epid(&chain_signer.public().0, sign_type)?
 		} else {
 			Default::default()
 		};
 
 		// generate an ECC certificate
 		info!("    [Enclave] Generate ECC Certificate");
-		let (key_der, cert_der) =
-			match cert::gen_ecc_cert(&payload.into_bytes(), &prv_k, &pub_k, &ecc_handle) {
-				Ok(r) => r,
-				Err(e) => {
-					error!("    [Enclave] gen_ecc_cert failed: {:?}", e);
-					return Err(e.into())
-				},
-			};
+		let (key_der, cert_der) = match cert::gen_ecc_cert(&payload, &prv_k, &pub_k, &ecc_handle) {
+			Ok(r) => r,
+			Err(e) => {
+				error!("    [Enclave] gen_ecc_cert failed: {:?}", e);
+				return Err(e.into())
+			},
+		};
 
 		let _ = ecc_handle.close();
 		info!("    [Enclave] Generate ECC Certificate successful");
@@ -247,10 +264,14 @@ where
 
 	fn generate_dcap_ra_cert(
 		&self,
-		quoting_enclave_target_info: &sgx_target_info_t,
-		quote_size: u32,
+		quoting_enclave_target_info: Option<&sgx_target_info_t>,
+		quote_size: Option<&u32>,
 		skip_ra: bool,
-	) -> EnclaveResult<(Vec<u8>, Vec<u8>)> {
+	) -> EnclaveResult<(Vec<u8>, Vec<u8>, Vec<u8>)> {
+		if !skip_ra && quoting_enclave_target_info.is_none() && quote_size.is_none() {
+			error!("Enclave Attestation] remote attestation not skipped, but Quoting Enclave (QE) data is not available");
+			return Err(EnclaveError::Sgx(sgx_status_t::SGX_ERROR_UNEXPECTED))
+		}
 		let chain_signer = self.signing_key_repo.retrieve_key()?;
 		info!("[Enclave Attestation] Ed25519 signer pub key: {:?}", chain_signer.public().0);
 
@@ -262,8 +283,8 @@ where
 		let qe_quote = if !skip_ra {
 			let qe_quote = match self.retrieve_qe_dcap_quote(
 				&chain_signer.public().0,
-				quoting_enclave_target_info,
-				quote_size,
+				quoting_enclave_target_info.unwrap(),
+				*quote_size.unwrap(),
 			) {
 				Ok(quote) => quote,
 				Err(e) => {
@@ -276,20 +297,25 @@ where
 			Default::default()
 		};
 
+		let qe_quote_base_64 = base64::encode(&qe_quote[..]);
 		// generate an ECC certificate
 		debug!("[Enclave] Generate ECC Certificate");
-		let (_key_der, cert_der) = match cert::gen_ecc_cert(&qe_quote, &prv_k, &pub_k, &ecc_handle)
-		{
-			Ok(r) => r,
-			Err(e) => {
-				error!("[Enclave] gen_ecc_cert failed: {:?}", e);
-				return Err(e.into())
-			},
-		};
+		let (key_der, cert_der) =
+			match cert::gen_ecc_cert(&qe_quote_base_64, &prv_k, &pub_k, &ecc_handle) {
+				Ok(r) => r,
+				Err(e) => {
+					error!("[Enclave] gen_ecc_cert failed: {:?}", e);
+					return Err(e.into())
+				},
+			};
 
 		let _ = ecc_handle.close();
 
-		Ok((cert_der, qe_quote))
+		debug!("[Enclave] Generated ECC cert info:");
+		trace!("[Enclave] Generated ECC cert info: key_der={:#?}", &key_der);
+		trace!("[Enclave] Generated ECC cert info: cert_der={:#?}", &cert_der);
+		trace!("[Enclave] Generated ECC cert info: qe_quote={:#?}", &qe_quote);
+		Ok((key_der, cert_der, qe_quote))
 	}
 }
 
@@ -511,7 +537,7 @@ where
 			+ (u32::from(array[3]) << 24)
 	}
 
-	fn create_attestation_report(
+	fn create_epid_attestation_report(
 		&self,
 		pub_k: &[u8; 32],
 		sign_type: sgx_quote_sign_type_t,