feat: update for rational post (#23)

* feat: updates for rational post

* feat(update to consume new PoSt API + sector id)

* refactor(move symbols around in FFI test to make it easier to read)

* refactor(point crates to head of master)

* refactor(remove unused symbols)

* bug(bump cache key to consume new Groth parameters)

* bug(install filecoin-proofs from head of master, which mimics Cargo dependency)

.circleci/config.yml

@@ -14,7 +14,7 @@ jobs:
           command: cargo generate-lockfile
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
       - run: rustup component add rustfmt-preview
@@ -27,7 +27,7 @@ jobs:
           name: Ensure cache is hydrated with PoRep and PoSt Groth parameters (for test)
           command: |
             # install paramfetch
-            cargo install filecoin-proofs --force --version=0.6.4
+            cargo install filecoin-proofs --force --git=https://github.com/filecoin-project/rust-fil-proofs.git --branch=master
             which paramcache || { printf '%s\n' "missing paramcache binary" >&2; exit 1; }
 
             # compute and cache Groth parameters for 1KiB sectors
@@ -37,7 +37,7 @@ jobs:
           paths:
             - Cargo.lock
       - save_cache:
-          key: cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          key: cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
           paths:
             - /root/.cargo
             - /root/.rustup
@@ -54,7 +54,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Test (stable)
           command: cargo +stable test --verbose --all
@@ -82,7 +82,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Test (stable) in release profile
           command: |
@@ -101,7 +101,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Build the static lib for linking into the example
           command: ./scripts/build-release.sh $(cat ./rust-toolchain) --verbose --package sector-builder-ffi
@@ -121,7 +121,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Test (stable) in release profile
           command: cargo +stable test --verbose --release --all -- --ignored
@@ -138,7 +138,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Test (nightly)
           command: cargo +$(cat rust-toolchain) test --verbose --all
@@ -156,7 +156,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Benchmarks (nightly)
           command: cargo +$(cat rust-toolchain) build --benches --verbose --all
@@ -174,7 +174,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Run cargo fmt
           command: cargo fmt --all -- --check
@@ -191,7 +191,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Run cargo clippy
           command: cargo clippy --all
@@ -208,7 +208,7 @@ jobs:
           at: "."
       - restore_cache:
           keys:
-            - cargo-v2b-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v4-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run:
           name: Install jq
           command: apt-get install jq -yqq

sector-builder-ffi/examples/simple.rs

@@ -6,8 +6,6 @@
 #[macro_use(defer)]
 extern crate scopeguard;
 
-include!(concat!(env!("OUT_DIR"), "/libsector_builder_ffi.rs"));
-
 use std::error::Error;
 use std::io::Write;
 use std::ptr;
@@ -25,6 +23,8 @@ use filecoin_proofs::error::ExpectWithBacktrace;
 use rand::{thread_rng, Rng};
 use tempfile::{NamedTempFile, TempDir};
 
+include!(concat!(env!("OUT_DIR"), "/libsector_builder_ffi.rs"));
+
 ///////////////////////////////////////////////////////////////////////////////
 // SectorBuilder lifecycle test
 ///////////////////////////////
@@ -144,7 +144,6 @@ unsafe fn sector_builder_lifecycle(use_live_store: bool) -> Result<(), Box<dyn E
             sector_class: sector_builder_ffi_FFISectorClass {
                 sector_size: LIVE_SECTOR_SIZE,
                 porep_proof_partitions: 2,
-                post_proof_partitions: 1,
             },
             max_bytes: 1016 * 1024 * 256,
             first_piece_bytes: 400 * 1024 * 256,
@@ -157,7 +156,6 @@ unsafe fn sector_builder_lifecycle(use_live_store: bool) -> Result<(), Box<dyn E
             sector_class: sector_builder_ffi_FFISectorClass {
                 sector_size: TEST_SECTOR_SIZE,
                 porep_proof_partitions: 2,
-                post_proof_partitions: 1,
             },
             max_bytes: 1016,
             first_piece_bytes: 400,
@@ -356,7 +354,7 @@ unsafe fn sector_builder_lifecycle(use_live_store: bool) -> Result<(), Box<dyn E
                 &mut (*resp).comm_d,
                 &mut (*resp).comm_r_star,
                 &mut u64_to_fr_safe(0),
-                &mut u64_to_fr_safe(124),
+                124,
                 (*resp).proof_ptr,
                 (*resp).proof_len,
             );
@@ -437,15 +435,22 @@ unsafe fn sector_builder_lifecycle(use_live_store: bool) -> Result<(), Box<dyn E
 
         let sealed_sector_metadata: sector_builder_ffi_FFISealedSectorMetadata =
             from_raw_parts((*resp).sectors_ptr, (*resp).sectors_len)[0];
+
+        let sector_ids = vec![sealed_sector_metadata.sector_id];
+
         let sealed_sector_replica_commitment: [u8; 32] = sealed_sector_metadata.comm_r;
-        // FIXME: for some reason bindgen generates *mut instead of *const.
+
         let mut challenge_seed: [u8; 32] = [0; 32];
 
+        let faults = vec![];
+
         let resp = sector_builder_ffi_generate_post(
             sector_builder_b,
-            &sealed_sector_replica_commitment[0],
-            32,
+            sealed_sector_replica_commitment.as_ptr(),
+            sealed_sector_replica_commitment.len(),
             &mut challenge_seed,
+            faults.as_ptr(),
+            faults.len(),
         );
         defer!(sector_builder_ffi_destroy_generate_post_response(resp));
 
@@ -455,14 +460,15 @@ unsafe fn sector_builder_lifecycle(use_live_store: bool) -> Result<(), Box<dyn E
 
         let resp = sector_builder_ffi_verify_post(
             sizes.sector_class.sector_size,
-            sizes.sector_class.post_proof_partitions,
-            &sealed_sector_replica_commitment[0],
-            32,
             &mut challenge_seed,
-            (*resp).flattened_proofs_ptr,
-            (*resp).flattened_proofs_len,
-            (*resp).faults_ptr,
-            (*resp).faults_len,
+            sector_ids.as_ptr(),
+            sector_ids.len(),
+            faults.as_ptr(),
+            faults.len(),
+            sealed_sector_replica_commitment.as_ptr(),
+            sealed_sector_replica_commitment.len(),
+            (*resp).proof_ptr,
+            (*resp).proof_len,
         );
         defer!(sector_builder_ffi_destroy_verify_post_response(resp));
 

sector-builder-ffi/src/api.rs

@@ -6,6 +6,8 @@ use ffi_toolkit::rust_str_to_c_str;
 use ffi_toolkit::{c_str_to_rust_str, raw_ptr};
 use libc;
 use once_cell::sync::OnceCell;
+use storage_proofs::sector::SectorId;
+
 use sector_builder::{PieceMetadata, SealStatus, SecondsSinceEpoch, SectorBuilder};
 
 use crate::responses::{
@@ -16,7 +18,6 @@ use crate::responses::{
 pub struct FFISectorClass {
     sector_size: u64,
     porep_proof_partitions: u8,
-    post_proof_partitions: u8,
 }
 
 /// Writes user piece-bytes to a staged sector and returns the id of the sector
@@ -45,7 +46,7 @@ pub unsafe extern "C" fn sector_builder_ffi_add_piece(
     ) {
         Ok(sector_id) => {
             response.status_code = FCPResponseStatus::FCPNoError;
-            response.sector_id = sector_id;
+            response.sector_id = u64::from(sector_id);
         }
         Err(err) => {
             let (code, ptr) = err_code_and_msg(&err);
@@ -113,7 +114,7 @@ pub unsafe extern "C" fn sector_builder_ffi_get_seal_status(
 
     let mut response: responses::GetSealStatusResponse = Default::default();
 
-    match (*ptr).get_seal_status(sector_id) {
+    match (*ptr).get_seal_status(SectorId::from(sector_id)) {
         Ok(seal_status) => {
             response.status_code = FCPResponseStatus::FCPNoError;
 
@@ -136,7 +137,7 @@ pub unsafe extern "C" fn sector_builder_ffi_get_seal_status(
                     response.proof_ptr = meta.proof.as_ptr();
                     response.seal_status_code = FFISealStatus::Sealed;
                     response.sector_access = rust_str_to_c_str(meta.sector_access);
-                    response.sector_id = meta.sector_id;
+                    response.sector_id = u64::from(meta.sector_id);
 
                     mem::forget(meta.proof);
                     mem::forget(pieces);
@@ -194,7 +195,7 @@ pub unsafe extern "C" fn sector_builder_ffi_get_sealed_sectors(
                         proofs_len: snark_proof.len(),
                         proofs_ptr: snark_proof.as_ptr(),
                         sector_access: rust_str_to_c_str(meta.sector_access.clone()),
-                        sector_id: meta.sector_id,
+                        sector_id: u64::from(meta.sector_id),
                     };
 
                     mem::forget(snark_proof);
@@ -241,7 +242,7 @@ pub unsafe extern "C" fn sector_builder_ffi_get_staged_sectors(
 
                     let mut sector = responses::FFIStagedSectorMetadata {
                         sector_access: rust_str_to_c_str(meta.sector_access.clone()),
-                        sector_id: meta.sector_id,
+                        sector_id: u64::from(meta.sector_id),
                         pieces_len: pieces.len(),
                         pieces_ptr: pieces.as_ptr(),
                         seal_status_code: FFISealStatus::Pending,
@@ -293,41 +294,32 @@ pub unsafe extern "C" fn sector_builder_ffi_generate_post(
     flattened_comm_rs_ptr: *const u8,
     flattened_comm_rs_len: libc::size_t,
     challenge_seed: &[u8; 32],
+    faults_ptr: *const u64,
+    faults_len: libc::size_t,
 ) -> *mut responses::GeneratePoStResponse {
     init_log();
 
     info!("generate_post: {}", "start");
 
     let comm_rs = into_commitments(flattened_comm_rs_ptr, flattened_comm_rs_len);
+    let faults = from_raw_parts(faults_ptr, faults_len)
+        .iter()
+        .map(|x| SectorId::from(*x))
+        .collect();
 
-    let result = (*ptr).generate_post(&comm_rs, challenge_seed);
+    let result = (*ptr).generate_post(&comm_rs, challenge_seed, faults);
 
     let mut response = responses::GeneratePoStResponse::default();
 
     match result {
-        Ok(filecoin_proofs::post_adapter::GeneratePoStDynamicSectorsCountOutput {
-            proofs,
-            faults,
-        }) => {
+        Ok(proof) => {
             response.status_code = FCPResponseStatus::FCPNoError;
 
-            let flattened_proofs: Vec<u8> = proofs.iter().flat_map(|x| x.iter().cloned()).collect();
-            response.flattened_proofs_len = flattened_proofs.len();
-            response.flattened_proofs_ptr = flattened_proofs.as_ptr();
-
-            let class = (*ptr).get_sector_class();
-            let filecoin_proofs::PoStProofPartitions(n) =
-                filecoin_proofs::PoStProofPartitions::from(filecoin_proofs::PoStConfig::from(
-                    class,
-                ));
-            response.proof_partitions = n;
-
-            response.faults_len = faults.len();
-            response.faults_ptr = faults.as_ptr();
+            response.proof_len = proof.len();
+            response.proof_ptr = proof.as_ptr();
 
             // we'll free this stuff when we free the GeneratePoSTResponse
-            mem::forget(flattened_proofs);
-            mem::forget(faults);
+            mem::forget(proof);
         }
         Err(err) => {
             let (code, ptr) = err_code_and_msg(&err);
@@ -357,7 +349,7 @@ pub unsafe extern "C" fn sector_builder_ffi_init_sector_builder(
 
     let result = SectorBuilder::init_from_metadata(
         from_ffi_sector_class(sector_class),
-        last_used_sector_id,
+        SectorId::from(last_used_sector_id),
         c_str_to_rust_str(metadata_dir).to_string(),
         *prover_id,
         c_str_to_rust_str(sealed_sector_dir).to_string(),
@@ -445,7 +437,7 @@ pub unsafe extern "C" fn sector_builder_ffi_verify_seal(
     comm_d: &[u8; 32],
     comm_r_star: &[u8; 32],
     prover_id: &[u8; 31],
-    sector_id: &[u8; 31],
+    sector_id: u64,
     proof_ptr: *const u8,
     proof_len: libc::size_t,
 ) -> *mut filecoin_proofs_ffi::responses::VerifySealResponse {
@@ -468,27 +460,29 @@ pub unsafe extern "C" fn sector_builder_ffi_verify_seal(
 #[no_mangle]
 pub unsafe extern "C" fn sector_builder_ffi_verify_post(
     sector_size: u64,
-    proof_partitions: u8,
+    challenge_seed: &[u8; 32],
+    sector_ids_ptr: *const u64,
+    sector_ids_len: libc::size_t,
+    faulty_sector_ids_ptr: *const u64,
+    faulty_sector_ids_len: libc::size_t,
     flattened_comm_rs_ptr: *const u8,
     flattened_comm_rs_len: libc::size_t,
-    challenge_seed: &[u8; 32],
-    flattened_proofs_ptr: *const u8,
-    flattened_proofs_len: libc::size_t,
-    faults_ptr: *const u64,
-    faults_len: libc::size_t,
+    proof_ptr: *const u8,
+    proof_len: libc::size_t,
 ) -> *mut filecoin_proofs_ffi::responses::VerifyPoStResponse {
     init_log();
 
     filecoin_proofs_ffi::api::verify_post(
         sector_size,
-        proof_partitions,
+        challenge_seed,
+        sector_ids_ptr,
+        sector_ids_len,
+        faulty_sector_ids_ptr,
+        faulty_sector_ids_len,
         flattened_comm_rs_ptr,
         flattened_comm_rs_len,
-        challenge_seed,
-        flattened_proofs_ptr,
-        flattened_proofs_len,
-        faults_ptr,
-        faults_len,
+        proof_ptr,
+        proof_len,
     )
 }
 
@@ -620,11 +614,9 @@ pub fn from_ffi_sector_class(fsc: FFISectorClass) -> filecoin_proofs::SectorClas
         FFISectorClass {
             sector_size,
             porep_proof_partitions,
-            post_proof_partitions,
         } => filecoin_proofs::SectorClass(
             filecoin_proofs::SectorSize(sector_size),
             filecoin_proofs::PoRepProofPartitions(porep_proof_partitions),
-            filecoin_proofs::PoStProofPartitions(post_proof_partitions),
         ),
     }
 }