From 73ce6a3fe6066e4fa51ff2599641ce875b2acdb1 Mon Sep 17 00:00:00 2001 From: Gursharan Singh <3442979+G8XSU@users.noreply.github.com> Date: Wed, 3 May 2023 16:22:07 -0700 Subject: [PATCH] Add Vss thin client protobuf generation & build/CI setup --- .github/workflows/build.yml | 31 +++ .gitignore | 2 + Cargo.toml | 4 + rustfmt.toml | 6 + vss-accessor/Cargo.toml | 10 + vss-accessor/src/generated-src/org.vss.rs | 251 ++++++++++++++++++++++ vss-accessor/src/lib.rs | 1 + vss-accessor/tests/generate_protos.rs | 31 +++ 8 files changed, 336 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 rustfmt.toml create mode 100644 vss-accessor/Cargo.toml create mode 100644 vss-accessor/src/generated-src/org.vss.rs create mode 100644 vss-accessor/src/lib.rs create mode 100644 vss-accessor/tests/generate_protos.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..41afc3a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,31 @@ +name: Continuous Integration Checks + +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + toolchain: [ stable, beta ] + include: + - toolchain: stable + check-fmt: true + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v2 + - name: Install Protobuf compiler (protoc) + run: sudo apt-get update && sudo apt-get -y install protobuf-compiler + - name: Install Rust ${{ matrix.toolchain }} toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.toolchain }} + override: true + profile: minimal + - name: Build on Rust ${{ matrix.toolchain }} + run: cargo build --verbose --color always + - name: Check formatting + if: matrix.check-fmt + run: rustup component add rustfmt && cargo fmt --all -- --check + - name: Test on Rust ${{ matrix.toolchain }} + run: cargo test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b2dd75f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +/vss-accessor/src/proto/ diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..858b842 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = [ + "vss-accessor", +] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..e7437e4 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,6 @@ +hard_tabs = true # use tab characters for indentation, spaces for alignment +use_field_init_shorthand = true +max_width = 120 +use_small_heuristics = "Max" +chain_width = 80 +fn_args_layout = "Compressed" diff --git a/vss-accessor/Cargo.toml b/vss-accessor/Cargo.toml new file mode 100644 index 0000000..d00aefd --- /dev/null +++ b/vss-accessor/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "vss-accessor" +version = "0.1.0" +edition = "2021" + +[dependencies] + +[dev-dependencies] +prost-build = { version = "0.11.3" } +reqwest = { version = "0.11.13", features = ["blocking"] } diff --git a/vss-accessor/src/generated-src/org.vss.rs b/vss-accessor/src/generated-src/org.vss.rs new file mode 100644 index 0000000..acc4f69 --- /dev/null +++ b/vss-accessor/src/generated-src/org.vss.rs @@ -0,0 +1,251 @@ +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetObjectRequest { + /// store_id is a keyspace identifier. + /// Ref: ) + /// All APIs operate within a single store_id. + /// It is up to clients to use single or multiple stores for their use-case. + /// This can be used for client-isolation/ rate-limiting / throttling on the server-side. + /// Authorization and billing can also be performed at the store_id level. + #[prost(string, tag = "1")] + pub store_id: ::prost::alloc::string::String, + /// Key for which the value is to be fetched. + /// + /// Consistency Guarantee: + /// Get(read) operations against a key are consistent reads and will reflect all previous writes, + /// since Put/Write provides read-after-write and read-after-update consistency guarantees. + /// + /// Read Isolation: + /// Get/Read operations against a key are ensured to have read-committed isolation. + /// Ref: )#Read_committed + #[prost(string, tag = "2")] + pub key: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetObjectResponse { + /// Fetched value and version along with the corresponding key in the request. + #[prost(message, optional, tag = "2")] + pub value: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PutObjectRequest { + /// store_id is a keyspace identifier. + /// Ref: ) + /// All APIs operate within a single store_id. + /// It is up to clients to use single or multiple stores for their use-case. + /// This can be used for client-isolation/ rate-limiting / throttling on the server-side. + /// Authorization and billing can also be performed at the store_id level. + #[prost(string, tag = "1")] + pub store_id: ::prost::alloc::string::String, + /// global_version is a sequence-number/version of the whole store. This can be used for versioning + /// and ensures that multiple updates in case of multiple devices can only be done linearly, even + /// if those updates did not directly conflict with each other based on keys/transaction_items. + /// + /// If present, the write will only succeed if the current server-side global_version against + /// the store_id is same as in the request. + /// Clients are expected to store (client-side) the global version against store_id. + /// The request must contain their client-side value of global_version if global versioning and + /// conflict detection is desired. + /// + /// For the first write of the store, global version should be '0'. If the write succeeds, clients + /// must increment their global version (client-side) by 1. + /// The server increments global_version (server-side) for every successful write, hence this + /// client-side increment is required to ensure matching versions. This updated global version + /// should be used in subsequent PutObjectRequests for the store. + /// + /// Requests with a conflicting version will fail with `CONFLICT_EXCEPTION` as ErrorCode. + #[prost(int64, optional, tag = "2")] + pub global_version: ::core::option::Option, + /// Items to be written as a result of this PutObjectRequest. + /// + /// In an item, each key is supplied with its corresponding value and version. + /// Clients can choose to encrypt the keys client-side in order to obfuscate their usage patterns. + /// If the write is successful, the previous value corresponding to the key will be overwritten. + /// + /// Multiple items in transaction_items of a single PutObjectRequest are written in + /// a database-transaction in an all-or-nothing fashion. + /// Items in a single PutObjectRequest must have distinct keys. + /// + /// Clients are expected to store a version against every key. + /// The write will succeed if the current DB version against the key is the same as in the request. + /// When initiating a PutObjectRequest, the request should contain their client-side version for + /// that key-value. + /// + /// For the first write of any key, the version should be '0'. If the write succeeds, the client + /// must increment their corresponding key versions (client-side) by 1. + /// The server increments key versions (server-side) for every successful write, hence this + /// client-side increment is required to ensure matching versions. These updated key versions should + /// be used in subsequent PutObjectRequests for the keys. + /// + /// Requests with a conflicting version will fail with `CONFLICT_EXCEPTION` as ErrorCode. + /// + /// Considerations for transactions: + /// Transaction writes of multiple items have a performance overhead, hence it is recommended to use + /// them only if required by the client application to ensure logic/code correctness. + /// That is, transaction_items are not a substitute for batch-write of multiple unrelated items. + /// When a write of multiple unrelated items is desired, it is recommended to use separate + /// PutObjectRequests. + /// + /// Consistency guarantee: + /// All PutObjectRequests are strongly consistent i.e. they provide read-after-write and + /// read-after-update consistency guarantees. + #[prost(message, repeated, tag = "3")] + pub transaction_items: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PutObjectResponse {} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListKeyVersionsRequest { + /// store_id is a keyspace identifier. + /// Ref: ) + /// All APIs operate within a single store_id. + /// It is up to clients to use single or multiple stores for their use-case. + /// This can be used for client-isolation/ rate-limiting / throttling on the server-side. + /// Authorization and billing can also be performed at the store_id level. + #[prost(string, tag = "1")] + pub store_id: ::prost::alloc::string::String, + /// A key_prefix is a string of characters at the beginning of the key. Prefixes can be used as + /// a way to organize key-values in a similar way to directories. + /// + /// If key_prefix is specified, the response results will be limited to those keys that begin with + /// the specified prefix. + /// + /// If no key_prefix is specified or it is empty (""), all the keys are eligible to be returned in + /// the response. + #[prost(string, optional, tag = "2")] + pub key_prefix: ::core::option::Option<::prost::alloc::string::String>, + /// page_size is used by clients to specify the maximum number of results that can be returned by + /// the server. + /// The server may further constrain the maximum number of results returned in a single page. + /// If the page_size is 0 or not set, the server will decide the number of results to be returned. + #[prost(int32, optional, tag = "3")] + pub page_size: ::core::option::Option, + /// page_token is a pagination token. + /// + /// To query for the first page of ListKeyVersions, page_token must not be specified. + /// + /// For subsequent pages, use the value that was returned as `next_page_token` in the previous + /// page's ListKeyVersionsResponse. + #[prost(string, optional, tag = "4")] + pub page_token: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListKeyVersionsResponse { + /// Fetched keys and versions. + /// Even though this API reuses KeyValue struct, the value sub-field will not be set by the server. + #[prost(message, repeated, tag = "1")] + pub key_versions: ::prost::alloc::vec::Vec, + /// next_page_token is a pagination token, used to retrieve the next page of results. + /// Use this value to query for next_page of paginated ListKeyVersions operation, by specifying + /// this value as the `page_token` in the next request. + /// + /// If next_page_token is empty (""), then the "last page" of results has been processed and + /// there is no more data to be retrieved. + /// + /// If next_page_token is not empty, it does not necessarily mean that there is more data in the + /// result set. The only way to know when you have reached the end of the result set is when + /// next_page_token is empty. + /// + /// Caution: Clients must not assume a specific number of key_versions to be present in a page for + /// paginated response. + #[prost(string, optional, tag = "2")] + pub next_page_token: ::core::option::Option<::prost::alloc::string::String>, + /// global_version is a sequence-number/version of the whole store. + /// + /// global_version is only returned in response for the first page of the ListKeyVersionsResponse + /// and is guaranteed to be read before reading any key-versions. + /// + /// In case of refreshing the complete key-version view on the client-side, correct usage for + /// the returned global_version is as following: + /// 1. Read global_version from the first page of paginated response and save it as local variable. + /// 2. Update all the key_versions on client-side from all the pages of paginated response. + /// 3. Update global_version on client_side from the local variable saved in step-1. + /// This ensures that on client-side, all current key_versions were stored at global_version or later. + /// This guarantee is helpful for ensuring the versioning correctness if using the global_version + /// in PutObject API and can help avoid the race conditions related to it. + #[prost(int64, optional, tag = "3")] + pub global_version: ::core::option::Option, +} +/// When HttpStatusCode is not ok (200), the response `content` contains a serialized ErrorResponse +/// with the relevant ErrorCode and message +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ErrorResponse { + /// The error code uniquely identifying an error condition. + /// It is meant to be read and understood programmatically by code that detects/handles errors by + /// type. + #[prost(enumeration = "ErrorCode", tag = "1")] + pub error_code: i32, + /// The error message containing a generic description of the error condition in English. + /// It is intended for a human audience only and should not be parsed to extract any information + /// programmatically. Client-side code may use it for logging only. + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeyValue { + /// Key against which the value is stored. + #[prost(string, tag = "1")] + pub key: ::prost::alloc::string::String, + /// Version field is used for key-level versioning. + /// For first write of key, version should be '0'. If the write succeeds, clients must increment + /// their corresponding key version (client-side) by 1. + /// The server increments key version (server-side) for every successful write, hence this + /// client-side increment is required to ensure matching versions. These updated key versions should + /// be used in subsequent PutObjectRequests for the keys. + #[prost(int64, tag = "2")] + pub version: i64, + /// Object value in bytes which is stored (in put) and fetched (in get). + /// Clients must encrypt this blob client-side before sending it over the wire to server in order + /// to preserve privacy and security. + #[prost(bytes = "vec", tag = "3")] + pub value: ::prost::alloc::vec::Vec, +} +/// ErrorCodes to be used in ErrorResponse +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ErrorCode { + /// Default protobuf Enum value. Will not be used as ErrorCode by server. + Unknown = 0, + /// CONFLICT_EXCEPTION is used when the request contains mismatched version (either key or global) + /// in PutObjectRequest. For more info refer PutObjectRequest. + ConflictException = 1, + /// INVALID_REQUEST_EXCEPTION is used in the following cases: + /// - The request was missing a required argument. + /// - The specified argument was invalid, incomplete or in the wrong format. + /// - The request body of api cannot be deserialized into corresponding protobuf object. + InvalidRequestException = 2, + /// An internal server error occurred, client is probably at no fault and can safely retry this + /// error with exponential backoff. + InternalServerException = 3, +} +impl ErrorCode { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ErrorCode::Unknown => "UNKNOWN", + ErrorCode::ConflictException => "CONFLICT_EXCEPTION", + ErrorCode::InvalidRequestException => "INVALID_REQUEST_EXCEPTION", + ErrorCode::InternalServerException => "INTERNAL_SERVER_EXCEPTION", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN" => Some(Self::Unknown), + "CONFLICT_EXCEPTION" => Some(Self::ConflictException), + "INVALID_REQUEST_EXCEPTION" => Some(Self::InvalidRequestException), + "INTERNAL_SERVER_EXCEPTION" => Some(Self::InternalServerException), + _ => None, + } + } +} diff --git a/vss-accessor/src/lib.rs b/vss-accessor/src/lib.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/vss-accessor/src/lib.rs @@ -0,0 +1 @@ + diff --git a/vss-accessor/tests/generate_protos.rs b/vss-accessor/tests/generate_protos.rs new file mode 100644 index 0000000..195b4d3 --- /dev/null +++ b/vss-accessor/tests/generate_protos.rs @@ -0,0 +1,31 @@ +extern crate prost_build; + +use std::fs; +use std::fs::File; +use std::path::Path; + +// Use this test-case to generate rust-proto objects from proto file. +// 1. Enable test case by commenting/removing `#[ignore]` +// 2. Run command : +// ``` +// cargo build && OUT_DIR=../target/tmp/ cargo test generate_protos -- --exact +// ``` +#[test] +#[ignore] +fn generate_protos() { + download_file( + "https://raw.githubusercontent.com/lightningdevkit/vss-server/ff4b5fc6a079ed8719eb8be7ec35ca1d01c1cc55/app/src/main/proto/vss.proto", + "src/proto/vss.proto", + ).unwrap(); + + prost_build::compile_protos(&["src/proto/vss.proto"], &["src/"]).unwrap(); + fs::copy(concat!(env!("OUT_DIR"), "/org.vss.rs"), "src/generated-src/org.vss.rs").unwrap(); +} + +fn download_file(url: &str, save_to: &str) -> Result<(), Box> { + let mut response = reqwest::blocking::get(url)?; + fs::create_dir_all(Path::new(save_to).parent().unwrap())?; + let mut out_file = File::create(save_to)?; + response.copy_to(&mut out_file)?; + Ok(()) +}